feat(send): browser extension that clips pages into Readest as EPUBs (#4266)

Replaces the URL-only placeholder extension with a full MV3 page-clipper that builds a self-contained EPUB on the user's machine and uploads it to the inbox. - Captures the rendered DOM in a content script, then runs Readability, asset bundling, and EPUB build through the shared `convertToEpub({kind: 'page'})` pipeline inside a Chrome offscreen document (the SW lacks DOMParser). - Uploads the resulting EPUB directly from the offscreen page to the new `POST /api/send/inbox/file` endpoint — keeps the bytes in one realm because `runtime.sendMessage` JSON-serialises ArrayBuffer to `{}` between extension contexts. - Adds a long-lived Port + ping handshake between SW, offscreen, and the on-demand capture content script so neither idle-eviction nor load-order races can hang the popup. - Localised popup, badge feedback, key-as-content i18n (`_('English source')`) with an extract script that seeds locale stubs from i18n-langs.json and writes a static-imports map for the runtime. All 33 locales fully translated. - Server: `pages/api/send/inbox/file.ts` accepts a raw EPUB body (Content-Type: application/epub+zip), enforces the inbox pending cap, stores to the existing send-inbox R2 bucket as `kind='file'`. `assetBundler` now sets `credentials: 'include'` in the non-Tauri branch so the extension SW carries paywalled-CDN cookies. - 47 vitest cases for the extension shell (upload, badge, auth, lazy, popup state machine, auth-bridge token sync) + 8 cases for the new server endpoint. CI's `test_web_app` invokes both via the extended `test:pr:web` plus a `build-browser-ext` step that catches webpack alias / Tauri-stub regressions. Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-22 20:45:29 +08:00
parent 9ad43aa8b2
commit f6dfd09d82
115 changed files with 5187 additions and 215 deletions
@@ -0,0 +1,226 @@
+import { describe, expect, test, vi, beforeEach, afterEach } from 'vitest';
+import { EventEmitter } from 'node:events';
+import type { NextApiRequest, NextApiResponse } from 'next';
+
+// Hoisted mocks — defined before the route module is imported so vitest
+// intercepts the imports inside the route.
+
+const validateUserMock = vi.fn();
+vi.mock('@/utils/access', () => ({
+  validateUserAndToken: (...args: unknown[]) => validateUserMock(...args),
+}));
+
+const corsMock = vi.fn(async () => undefined);
+vi.mock('@/utils/cors', () => ({
+  corsAllMethods: vi.fn(),
+  runMiddleware: corsMock,
+}));
+
+const putObjectMock = vi.fn();
+vi.mock('@/utils/object', () => ({
+  putObject: (...args: unknown[]) => putObjectMock(...args),
+}));
+
+const insertMock = vi.fn();
+const updateMock = vi.fn();
+const deleteMock = vi.fn();
+const countMock = vi.fn();
+vi.mock('@/utils/supabase', () => ({
+  createSupabaseAdminClient: () => ({
+    from: () => ({
+      select: () => ({
+        eq: () => ({
+          in: () => countMock(),
+        }),
+      }),
+      insert: (row: unknown) => ({
+        select: () => ({
+          single: () => insertMock(row),
+        }),
+      }),
+      update: (row: unknown) => ({
+        eq: () => updateMock(row),
+      }),
+      delete: () => ({
+        eq: () => deleteMock(),
+      }),
+    }),
+  }),
+}));
+
+// Import AFTER mocks are in place.
+const { default: handler } = await import('@/pages/api/send/inbox/file');
+
+interface MockRes {
+  status: ReturnType<typeof vi.fn>;
+  json: ReturnType<typeof vi.fn>;
+  _status: number;
+  _body: Record<string, unknown> | undefined;
+}
+
+function makeRes(): MockRes {
+  const res: MockRes = {
+    status: vi.fn(),
+    json: vi.fn(),
+    _status: 0,
+    _body: undefined,
+  };
+  res.status.mockImplementation((code: number) => {
+    res._status = code;
+    return res as unknown as NextApiResponse;
+  });
+  res.json.mockImplementation((body: Record<string, unknown>) => {
+    res._body = body;
+    return res as unknown as NextApiResponse;
+  });
+  return res;
+}
+
+function makeReq(opts: {
+  method?: string;
+  authorization?: string;
+  contentType?: string;
+  title?: string;
+  url?: string;
+  body?: Buffer;
+}): NextApiRequest {
+  const emitter = new EventEmitter() as EventEmitter & {
+    headers: Record<string, string>;
+    method: string;
+    destroy: () => void;
+  };
+  emitter.headers = {};
+  if (opts.authorization) emitter.headers['authorization'] = opts.authorization;
+  if (opts.contentType) emitter.headers['content-type'] = opts.contentType;
+  if (opts.title) emitter.headers['x-readest-title'] = opts.title;
+  if (opts.url) emitter.headers['x-readest-url'] = opts.url;
+  emitter.method = opts.method ?? 'POST';
+  emitter.destroy = vi.fn();
+
+  // Emit body asynchronously so handler awaits the stream.
+  setImmediate(() => {
+    if (opts.body) emitter.emit('data', opts.body);
+    emitter.emit('end');
+  });
+
+  return emitter as unknown as NextApiRequest;
+}
+
+const validUser = { id: 'user-123' };
+const VALID_HEADERS = {
+  authorization: 'Bearer abc',
+  contentType: 'application/epub+zip',
+};
+
+beforeEach(() => {
+  validateUserMock.mockReset().mockResolvedValue({ user: validUser });
+  putObjectMock.mockReset().mockResolvedValue(undefined);
+  countMock.mockReset().mockResolvedValue({ count: 0, error: null });
+  insertMock.mockReset().mockResolvedValue({ data: { id: 'inbox-1' }, error: null });
+  updateMock.mockReset().mockResolvedValue({ error: null });
+  deleteMock.mockReset().mockResolvedValue({ error: null });
+  corsMock.mockReset().mockResolvedValue(undefined);
+});
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+describe('POST /api/send/inbox/file', () => {
+  test('rejects unauthenticated requests with 403', async () => {
+    validateUserMock.mockResolvedValueOnce({ user: null });
+    const req = makeReq({ ...VALID_HEADERS, body: Buffer.from('PK\x03\x04...') });
+    const res = makeRes();
+    await handler(req, res as unknown as NextApiResponse);
+    expect(res._status).toBe(403);
+  });
+
+  test('rejects non-POST methods with 405', async () => {
+    const req = makeReq({ method: 'GET', ...VALID_HEADERS });
+    const res = makeRes();
+    await handler(req, res as unknown as NextApiResponse);
+    expect(res._status).toBe(405);
+  });
+
+  test('rejects unsupported content-types with 415', async () => {
+    const req = makeReq({
+      authorization: 'Bearer abc',
+      contentType: 'text/html',
+      body: Buffer.from('<html/>'),
+    });
+    const res = makeRes();
+    await handler(req, res as unknown as NextApiResponse);
+    expect(res._status).toBe(415);
+  });
+
+  test('rejects invalid source URL with 400', async () => {
+    const req = makeReq({
+      ...VALID_HEADERS,
+      url: 'javascript:alert(1)',
+      body: Buffer.from('PK\x03\x04'),
+    });
+    const res = makeRes();
+    await handler(req, res as unknown as NextApiResponse);
+    expect(res._status).toBe(400);
+  });
+
+  test('rejects when inbox is full with 429', async () => {
+    countMock.mockResolvedValueOnce({ count: 60, error: null });
+    const req = makeReq({ ...VALID_HEADERS, body: Buffer.from('PK\x03\x04') });
+    const res = makeRes();
+    await handler(req, res as unknown as NextApiResponse);
+    expect(res._status).toBe(429);
+  });
+
+  test('rejects empty file with 400', async () => {
+    const req = makeReq({ ...VALID_HEADERS, body: Buffer.alloc(0) });
+    const res = makeRes();
+    await handler(req, res as unknown as NextApiResponse);
+    expect(res._status).toBe(400);
+  });
+
+  test('stores the EPUB and returns the inbox row id on success', async () => {
+    const body = Buffer.from('PK\x03\x04epub-bytes');
+    const req = makeReq({
+      ...VALID_HEADERS,
+      url: 'https://example.com/article',
+      title: "UTF-8''Article%20%E2%9C%85",
+      body,
+    });
+    const res = makeRes();
+    await handler(req, res as unknown as NextApiResponse);
+
+    expect(res._status).toBe(200);
+    expect(res._body).toEqual({ id: 'inbox-1' });
+
+    expect(insertMock).toHaveBeenCalledTimes(1);
+    const insertArg = insertMock.mock.calls[0]![0];
+    expect(insertArg).toMatchObject({
+      user_id: validUser.id,
+      kind: 'file',
+      source: 'extension',
+      url: 'https://example.com/article',
+      filename: 'Article ✅',
+      byte_size: body.byteLength,
+    });
+
+    expect(putObjectMock).toHaveBeenCalledTimes(1);
+    expect(putObjectMock.mock.calls[0]![0]).toBe('inbox/user-123/inbox-1/clip.epub');
+    expect(putObjectMock.mock.calls[0]![2]).toBe('application/epub+zip');
+
+    expect(updateMock).toHaveBeenCalledTimes(1);
+    expect(updateMock.mock.calls[0]![0]).toEqual({
+      payload_key: 'inbox/user-123/inbox-1/clip.epub',
+    });
+  });
+
+  test('rolls back the inbox row when R2 put fails', async () => {
+    putObjectMock.mockRejectedValueOnce(new Error('R2 unavailable'));
+    const req = makeReq({ ...VALID_HEADERS, body: Buffer.from('PK\x03\x04') });
+    const res = makeRes();
+    await handler(req, res as unknown as NextApiResponse);
+
+    expect(res._status).toBe(500);
+    expect(deleteMock).toHaveBeenCalledTimes(1);
+  });
+});
@@ -0,0 +1,170 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { createSupabaseAdminClient } from '@/utils/supabase';
+import { corsAllMethods, runMiddleware } from '@/utils/cors';
+import { validateUserAndToken } from '@/utils/access';
+import { putObject } from '@/utils/object';
+import { parseSubjectTag } from '@/services/send/sendAddress';
+import {
+  SEND_INBOX_BUCKET,
+  SEND_INBOX_FILE_MAX_BYTES,
+  SEND_INBOX_PENDING_LIMIT,
+} from '@/services/constants';
+
+/**
+ * `kind='file'` inbox endpoint for the browser extension. The extension
+ * builds a self-contained EPUB on the user's machine (Readability +
+ * inlined images + bundled stylesheet) and uploads it as the request body.
+ *
+ * Lives at its own path — `pages/api/send/inbox.ts` keeps the JSON
+ * bodyParser, but a binary upload needs `bodyParser: false` and a manual
+ * stream read.
+ *
+ * Drainer behaviour matches the email-attachment path: the payload is
+ * stored verbatim in the inbox R2 bucket, the row carries `kind='file'`,
+ * and the drainer imports the EPUB on the next open without any further
+ * conversion.
+ */
+export const config = {
+  api: {
+    bodyParser: false,
+    responseLimit: false,
+  },
+};
+
+const MAX_TITLE_LENGTH = 500;
+const MAX_URL_LENGTH = 2000;
+const ALLOWED_MIME = new Set(['application/epub+zip', 'application/octet-stream']);
+
+function header(req: NextApiRequest, name: string): string | null {
+  const value = req.headers[name];
+  if (Array.isArray(value)) return value[0] ?? null;
+  return value ?? null;
+}
+
+function decodeRfc5987(value: string): string {
+  // `X-Readest-Title: UTF-8''Spa%C3%9F`. Used so non-ASCII titles survive
+  // the HTTP-header transport without arbitrary client encoding.
+  const m = value.match(/^UTF-8''(.+)$/i);
+  if (m) {
+    try {
+      return decodeURIComponent(m[1]!);
+    } catch {
+      return '';
+    }
+  }
+  return value;
+}
+
+async function readBody(req: NextApiRequest, max: number): Promise<Buffer> {
+  return new Promise<Buffer>((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    let total = 0;
+    req.on('data', (chunk: Buffer) => {
+      total += chunk.byteLength;
+      if (total > max) {
+        reject(Object.assign(new Error('Payload too large'), { code: 'payload_too_large' }));
+        req.destroy();
+        return;
+      }
+      chunks.push(chunk);
+    });
+    req.on('end', () => resolve(Buffer.concat(chunks)));
+    req.on('error', reject);
+  });
+}
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  await runMiddleware(req, res, corsAllMethods);
+
+  if (req.method !== 'POST') {
+    return res.status(405).json({ error: 'Method not allowed' });
+  }
+
+  const { user } = await validateUserAndToken(req.headers['authorization']);
+  if (!user) {
+    return res.status(403).json({ error: 'Not authenticated' });
+  }
+
+  const contentType = header(req, 'content-type') ?? '';
+  const baseType = contentType.split(';')[0]!.trim().toLowerCase();
+  if (!ALLOWED_MIME.has(baseType)) {
+    return res.status(415).json({ error: 'Unsupported content type' });
+  }
+
+  const titleRaw = header(req, 'x-readest-title');
+  const urlRaw = header(req, 'x-readest-url');
+  const title = titleRaw ? decodeRfc5987(titleRaw).slice(0, MAX_TITLE_LENGTH) : null;
+  const sourceUrl = urlRaw ? decodeRfc5987(urlRaw).slice(0, MAX_URL_LENGTH) : null;
+
+  if (sourceUrl && !/^https?:\/\//i.test(sourceUrl)) {
+    return res.status(400).json({ error: 'Invalid source URL' });
+  }
+
+  const supabase = createSupabaseAdminClient();
+
+  // Same anti-abuse cap as the JSON inbox endpoint: a leaked token can't
+  // flood R2 once the user has too many pending items.
+  const { count, error: countError } = await supabase
+    .from('send_inbox')
+    .select('id', { count: 'exact', head: true })
+    .eq('user_id', user.id)
+    .in('status', ['pending', 'claimed']);
+  if (countError) {
+    return res.status(500).json({ error: countError.message });
+  }
+  if ((count ?? 0) >= SEND_INBOX_PENDING_LIMIT) {
+    return res.status(429).json({ error: 'Inbox is full — open Readest to process pending items' });
+  }
+
+  let body: Buffer;
+  try {
+    body = await readBody(req, SEND_INBOX_FILE_MAX_BYTES);
+  } catch (err) {
+    if ((err as { code?: string }).code === 'payload_too_large') {
+      return res.status(413).json({ error: 'File is too large' });
+    }
+    return res.status(400).json({ error: 'Could not read request body' });
+  }
+  if (body.byteLength === 0) {
+    return res.status(400).json({ error: 'Empty file' });
+  }
+
+  const { data: row, error: rowError } = await supabase
+    .from('send_inbox')
+    .insert({
+      user_id: user.id,
+      kind: 'file',
+      source: 'extension',
+      url: sourceUrl,
+      filename: title,
+      byte_size: body.byteLength,
+      subject_tag: parseSubjectTag(title) ?? null,
+    })
+    .select('id')
+    .single<{ id: string }>();
+  if (rowError) return res.status(500).json({ error: rowError.message });
+
+  const payloadKey = `inbox/${user.id}/${row.id}/clip.epub`;
+  // Allocate a fresh ArrayBuffer so we don't accidentally hand a
+  // SharedArrayBuffer-typed view to the S3 client, which expects an
+  // owned ArrayBuffer.
+  const payloadBuffer = new ArrayBuffer(body.byteLength);
+  new Uint8Array(payloadBuffer).set(body);
+  try {
+    await putObject(payloadKey, payloadBuffer, 'application/epub+zip', SEND_INBOX_BUCKET);
+  } catch (err) {
+    // Roll back the row so we never leave a `pending` item the drainer
+    // would only fail on.
+    await supabase.from('send_inbox').delete().eq('id', row.id);
+    console.error('Inbox file upload failed:', err);
+    return res.status(500).json({ error: 'Could not store EPUB' });
+  }
+
+  const { error: updateError } = await supabase
+    .from('send_inbox')
+    .update({ payload_key: payloadKey })
+    .eq('id', row.id);
+  if (updateError) return res.status(500).json({ error: updateError.message });
+
+  return res.status(200).json({ id: row.id });
+}
@@ -753,6 +753,12 @@ export const SHARE_EXPIRATION_DAYS = [1, 3, 7] as const;
 export const SEND_EMAIL_DOMAIN = 'readest.com';
 export const SEND_INBOX_BUCKET = 'readest-send-inbox';
 export const SEND_INBOX_PENDING_LIMIT = 50;
+// Hard cap on the size of a single uploaded EPUB the browser extension can
+// drop into the inbox. 30 MB is the same total-asset cap the client-side
+// bundler enforces — plus a bit of head-room for chapter HTML / structural
+// overhead. Beyond this size a clipped article is almost certainly an
+// over-illustrated page that would never read well in the EPUB anyway.
+export const SEND_INBOX_FILE_MAX_BYTES = 40 * 1024 * 1024;
 export const SHARE_DEFAULT_EXPIRATION_DAYS = 3;
 export const SHARE_MAX_PER_USER = 50;
 export const SHARE_TOKEN_LENGTH = 22;
@@ -5,12 +5,18 @@ import type { EpubImage } from './types';

 // On Tauri we go through the Rust HTTP client (no browser-CORS); on web
 // we use the native fetch — the bundler is only ever invoked from a
-// CORS-free caller there (the future extension's content script). In
-// Tauri we also fold in the full image-fetch header set (UA + Sec-Ch-Ua +
-// Sec-Fetch-* + Referer) so CDNs that gate images on the browser shape
-// — NYT, WSJ, paywalled CDNs — cooperate.
+// CORS-free caller there (the browser extension's service worker, which
+// has broad `host_permissions`). In the non-Tauri path we set
+// `credentials: 'include'` so the browser sends the user's cookies for
+// paywalled / member-only CDN hosts — without that, an authenticated
+// Substack image returns a placeholder. In Tauri we also fold in the
+// full image-fetch header set (UA + Sec-Ch-Ua + Sec-Fetch-* + Referer)
+// so CDNs that gate images on the browser shape — NYT, WSJ, paywalled
+// CDNs — cooperate.
 const httpFetch = (url: string, referer: string | null, init?: RequestInit): Promise<Response> => {
-  if (!isTauriAppPlatform()) return globalThis.fetch(url, init);
+  if (!isTauriAppPlatform()) {
+    return globalThis.fetch(url, { credentials: 'include', ...init });
+  }
  const baseHeaders = imageFetchHeaders(referer);
  const headers = new Headers(init?.headers);
  for (const [k, v] of Object.entries(baseHeaders)) {