diff --git a/apps/readest-app/.claude/memory/MEMORY.md b/apps/readest-app/.claude/memory/MEMORY.md index 8f1a6bb7..78902181 100644 --- a/apps/readest-app/.claude/memory/MEMORY.md +++ b/apps/readest-app/.claude/memory/MEMORY.md @@ -27,6 +27,9 @@ - [KOSync CFI spine resolution](kosync-cfi-spine-resolution.md) — convert via the CFI's own spine (`getXPointerFromCFI`/`getCFIFromXPointer`), never `new XCFI(primaryDoc, primaryIndex)`; primaryIndex lags during scroll → spine-mismatch throw - [Empty-start CFI sync bug](empty-start-cfi-sync.md) — `epubcfi(/6/24!/4,,/20/1:58)` (empty-start range) from the cfi-inert skip-link transitional window; jumps to wrong section end; `isMalformedLocationCfi` → discard the synced value in `useProgressSync` (NOT the local open path); foliate fix doesn't repair already-synced values +## Build & Vendoring +- [pdfjs vendor wasm decoders](pdfjs-vendor-wasm-decoders.md) — scanned PDFs blank in CI build only (0.11.2 regression); pdfjs 5.7.x moved JBIG2 to `jbig2.wasm`, `copy-pdfjs-wasm` allow-list dropped it; `cpx` no-errors on empty glob; local stale `public/vendor` (gitignored, not refreshed by `tauri build`) masked it; fix = copy `wasm/*` + ## Feature Notes - [Manage Cache + iOS container layout](manage-cache-ios-layout.md) — `'Cache'` base = `Library/Caches/` only (not all of Caches); iOS `Documents/Inbox` cleared too; WebKit cache + tmp out of reach; never touch App Support - [D-pad Navigation](dpad-navigation.md) — Android TV remote / keyboard arrow navigation design, key files, and pitfalls diff --git a/apps/readest-app/.claude/memory/pdfjs-vendor-wasm-decoders.md b/apps/readest-app/.claude/memory/pdfjs-vendor-wasm-decoders.md new file mode 100644 index 00000000..62541e79 --- /dev/null +++ b/apps/readest-app/.claude/memory/pdfjs-vendor-wasm-decoders.md @@ -0,0 +1,21 @@ +--- +name: pdfjs-vendor-wasm-decoders +description: Scanned PDFs blank in CI builds but fine locally — pdfjs wasm decoders (jbig2.wasm) not copied to public/vendor/pdfjs +metadata: + node_type: memory + type: project + originSessionId: 9066c0b0-71a6-4db0-92c1-c3ccacf1ff0d +--- + +CI-built DMG rendered scanned PDFs as blank pages (could still turn pages); text PDFs and EPUBs fine. Local build of the same commit worked. Regression between 0.11.1 (fine) and 0.11.2 (broken). + +**Root cause:** `pdfjs-dist` was bumped 5.4.530 → 5.7.284 in #4143 (commit e8df651d5, between the 0.11.1 and 0.11.2 tags). pdfjs 5.7.x moved image decoders — notably **JBIG2** (the codec used by virtually every black-and-white *scanned* PDF) — from pure JS into WebAssembly modules the worker fetches at runtime from `wasmUrl` (`/vendor/pdfjs/`, set in `packages/foliate-js/pdf.js`: `wasmUrl: pdfjsPath('')`). The `copy-pdfjs-wasm` npm script only copied an allow-list `{openjpeg.wasm,qcms_bg.wasm}` and silently dropped `jbig2.wasm`. **`cpx` does not error when a glob matches nothing**, so the missing decoder was invisible: worker loads, pages turn, JBIG2 decode fails → blank. + +**Why local masked it:** `pnpm build` / `tauri build` do NOT run `setup-vendors`. Local builds reuse whatever stale `public/vendor/pdfjs/` is already on disk (gitignored — `/public/vendor`). The dev's local copy was the old 5.4.530 worker (pure-JS JBIG2) → worked. CI runs `setup-vendors` fresh (release.yml:192) → ships the new 5.7.284 worker that needs jbig2.wasm → broke. + +**Fix:** changed `copy-pdfjs-wasm` to copy the whole `wasm/*` dir instead of an allow-list (mirrors the `{cmaps,standard_fonts}/*` fonts pattern). Robust against future codecs moving to wasm; also ships the `*_nowasm_fallback.js` files for graceful degradation. Regression test: `src/__tests__/document/pdfjs-wasm-assets.test.ts` asserts every `.wasm` the bundled pdf.js references is covered by `copy-pdfjs-wasm`. + +**Gotchas for future:** +- Vendor assets in `public/vendor/` are gitignored and only refreshed by `setup-vendors`. Local stale vendor can mask CI breakage — `git status` won't show it. +- `cpx` allow-lists are fragile: any upstream-added required file is dropped silently. Prefer copying whole dirs. +- Related: [[platform-compat-fixes]], [[bug-patterns]]. diff --git a/apps/readest-app/package.json b/apps/readest-app/package.json index 71c3e038..b9855a2d 100644 --- a/apps/readest-app/package.json +++ b/apps/readest-app/package.json @@ -45,7 +45,7 @@ "format:check": "pnpm -w format:check", "prepare-public-vendor": "mkdirp ./public/vendor/pdfjs ./public/vendor/simplecc ./public/vendor/jieba", "copy-pdfjs-js": "cpx \"../../packages/foliate-js/node_modules/pdfjs-dist/legacy/build/{pdf.worker.min.mjs,pdf.min.mjs,pdf.d.mts}\" ./public/vendor/pdfjs", - "copy-pdfjs-wasm": "cpx \"../../packages/foliate-js/node_modules/pdfjs-dist/wasm/{openjpeg.wasm,qcms_bg.wasm}\" ./public/vendor/pdfjs", + "copy-pdfjs-wasm": "cpx \"../../packages/foliate-js/node_modules/pdfjs-dist/wasm/*\" ./public/vendor/pdfjs", "copy-pdfjs-fonts": "cpx \"../../packages/foliate-js/node_modules/pdfjs-dist/{cmaps,standard_fonts}/*\" ./public/vendor/pdfjs", "copy-flatten-pdfjs-annotation-layer-css": "npx postcss \"../../packages/foliate-js/vendor/pdfjs/annotation_layer_builder.css\" --no-map -u postcss-nested > ./public/vendor/pdfjs/annotation_layer_builder.css", "copy-flatten-pdfjs-text-layer-css": "npx postcss \"../../packages/foliate-js/vendor/pdfjs/text_layer_builder.css\" --no-map -u postcss-nested > ./public/vendor/pdfjs/text_layer_builder.css", diff --git a/apps/readest-app/src/__tests__/document/pdfjs-wasm-assets.test.ts b/apps/readest-app/src/__tests__/document/pdfjs-wasm-assets.test.ts new file mode 100644 index 00000000..4dae649a --- /dev/null +++ b/apps/readest-app/src/__tests__/document/pdfjs-wasm-assets.test.ts @@ -0,0 +1,90 @@ +import { describe, it, expect } from 'vitest'; +import { existsSync, readdirSync, readFileSync } from 'fs'; +import { isAbsolute, resolve } from 'path'; + +/** + * Regression guard for a build-packaging bug (scanned PDFs rendered blank in + * CI builds of 0.11.2, but fine locally). + * + * pdfjs-dist 5.7.x moved several image decoders (notably JBIG2 — the codec used + * by virtually every black-and-white *scanned* PDF) from pure JS into + * WebAssembly modules that the worker fetches at runtime from `wasmUrl` + * (`/vendor/pdfjs/` — see packages/foliate-js/pdf.js). The `copy-pdfjs-wasm` + * npm script only copied an explicit allow-list (`openjpeg.wasm`, `qcms_bg.wasm`) + * and silently dropped `jbig2.wasm`. `cpx` does not error when a glob matches + * nothing, so the missing decoder went unnoticed: the worker loaded, pages + * turned, but JBIG2 image decoding failed → blank pages. + * + * Invariant: every `.wasm` file the *bundled* pdf.js (main + worker) references + * and that actually exists in pdfjs-dist's `wasm/` directory must be copied into + * the public vendor folder by `copy-pdfjs-wasm`. + */ + +const appRoot = process.cwd(); + +/** Extract the first double-quoted argument from a cpx-based npm script. */ +const sourceGlobOf = (script: string): string => { + const match = script.match(/"([^"]+)"/); + if (!match) throw new Error(`No quoted source glob in script: ${script}`); + return match[1]!; +}; + +/** + * Resolve which files a cpx source glob of the form `/` copies, + * where `` is `*`, a `{a,b,c}` brace list, or a single filename. + * Returns the absolute directory and the concrete set of copied basenames. + */ +const resolveCpxGlob = (glob: string): { dir: string; files: Set } => { + const slash = glob.lastIndexOf('/'); + const dirPart = glob.slice(0, slash); + const token = glob.slice(slash + 1); + const dir = isAbsolute(dirPart) ? dirPart : resolve(appRoot, dirPart); + + let names: string[]; + if (token.includes('*')) { + names = readdirSync(dir); + } else if (token.startsWith('{') && token.endsWith('}')) { + names = token.slice(1, -1).split(','); + } else { + names = [token]; + } + return { dir, files: new Set(names) }; +}; + +describe('pdfjs vendor wasm assets', () => { + const pkg = JSON.parse(readFileSync(resolve(appRoot, 'package.json'), 'utf8')) as { + scripts: Record; + }; + + it('copies every wasm decoder the bundled pdf.js references', () => { + // Files the worker/main bundle are copied from (source of truth for CI). + const jsGlob = sourceGlobOf(pkg.scripts['copy-pdfjs-js']!); + const { dir: jsDir, files: jsFiles } = resolveCpxGlob(jsGlob); + + // The wasm modules available in pdfjs-dist and what the copy script ships. + const wasmGlob = sourceGlobOf(pkg.scripts['copy-pdfjs-wasm']!); + const { dir: wasmDir, files: copiedWasm } = resolveCpxGlob(wasmGlob); + + const availableWasm = new Set(readdirSync(wasmDir).filter((f) => f.endsWith('.wasm'))); + + // Scan the bundled JS for `*.wasm` references, keeping only ones that map to + // a real file (drops minified false positives like `e.wasm`/`t.wasm`). + const referenced = new Set(); + for (const file of jsFiles) { + const full = resolve(jsDir, file); + if (!existsSync(full)) continue; + const text = readFileSync(full, 'utf8'); + for (const m of text.matchAll(/[A-Za-z0-9_-]+\.wasm/g)) { + if (availableWasm.has(m[0])) referenced.add(m[0]); + } + } + + // Sanity: the bump that caused the bug must still ship jbig2 as wasm. + expect(referenced.has('jbig2.wasm'), 'expected pdf.js to reference jbig2.wasm').toBe(true); + + const missing = [...referenced].filter((w) => !copiedWasm.has(w)); + expect(missing, `copy-pdfjs-wasm must copy these referenced wasm files: ${missing}`).toEqual( + [], + ); + }); +}); diff --git a/packages/foliate-js b/packages/foliate-js index e8995dae..1fc763b8 160000 --- a/packages/foliate-js +++ b/packages/foliate-js @@ -1 +1 @@ -Subproject commit e8995daeb6f7743aae4ac617529a3ce2f076dfab +Subproject commit 1fc763b8e6afe10971043a6818dcb9e2d8c54f4a diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d5aa6341..28fdb340 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -659,7 +659,7 @@ importers: specifier: ^15.9.0 version: 15.15.0 pdfjs-dist: - specifier: ^5.4.530 + specifier: ^5.7.284 version: 5.7.284 rollup: specifier: '>=4.59.0'