From 6e9faaa8745fe5db5def8a993ac0220688e19e33 Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Sat, 20 Jun 2026 08:00:44 +0800 Subject: [PATCH] fix(pdf): throttle PDF range reads to fix large-file OOM on Android/iOS (#3470) (#4670) Large PDFs (50 MB+) crashed on import/open. pdf.js requests hundreds of byte ranges in a burst while parsing the document structure, and foliate-js makePDF dispatched them all concurrently. On Android each read is served through the WebView's rangefile custom scheme (shouldInterceptRequest); the flood of simultaneous native requests exhausts the WebView's Java heap (OutOfMemoryError in handleRequest). Bump foliate-js to cap in-flight range reads at 6 (the implicit per-host limit a real HTTP transport already gets), and add a regression test asserting makePDF keeps at most 6 range reads in flight. Verified live on a Xiaomi 13 (Android 16 / WebView 147) via CDP: max concurrent range reads drop from 753 to 6 with no change in open time. Co-authored-by: Claude Opus 4.8 (1M context) --- .../memory/pdf-oom-range-flood-3470.md | 47 ++++++++ .../.claude/memory/zindex-overlay-scale.md | 48 ++++++++ .../foliate-pdf-range-concurrency.test.ts | 113 ++++++++++++++++++ packages/foliate-js | 2 +- 4 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 apps/readest-app/.claude/memory/pdf-oom-range-flood-3470.md create mode 100644 apps/readest-app/.claude/memory/zindex-overlay-scale.md create mode 100644 apps/readest-app/src/__tests__/foliate-pdf-range-concurrency.test.ts diff --git a/apps/readest-app/.claude/memory/pdf-oom-range-flood-3470.md b/apps/readest-app/.claude/memory/pdf-oom-range-flood-3470.md new file mode 100644 index 00000000..4aa9797a --- /dev/null +++ b/apps/readest-app/.claude/memory/pdf-oom-range-flood-3470.md @@ -0,0 +1,47 @@ +--- +name: pdf-oom-range-flood-3470 +description: "Android/iOS large-PDF import/open OOM (#3470) = unthrottled pdf.js range-request flood, not whole-file load; fix = concurrency cap in foliate makePDF" +metadata: + node_type: memory + type: project + originSessionId: 1f5ecad5-076c-4170-939a-c80438c37f64 +--- + +# Large-PDF OOM on Android/iOS (#3470) + +**Symptom:** importing/opening a 50 MB+ PDF crashes (no message) with +`java.lang.OutOfMemoryError ... target footprint 536870912` (512 MB Java heap) +at `RustWebViewClient.handleRequest` ← `shouldInterceptRequest`. Same file is +fine in the official pdf.js viewer on Android Chrome. Repro file: `100个句子记完7000个雅思单词.pdf` (67 MB, 970 pages). + +**Root cause (NOT whole-file load):** opening the PDF makes pdf.js fire ~759 +small **64 KB** range reads to parse scattered xref/object streams. foliate-js +`makePDF` fulfilled every `requestDataRange` with an **un-awaited** +`file.slice(begin,end).arrayBuffer()` → all dispatched at once (measured +**maxInFlight 753**). On Android each read is a `fetch()` to the `rangefile` +scheme → `shouldInterceptRequest` allocates a Rust `Vec` + a Java `byte[]` +per request; ~750 simultaneous intercepted requests exhaust the 512 MB Java +heap. The official pdf.js viewer survives because the **browser caps ~6 +connections/host**; the custom `rangefile` (and iOS native-file) scheme has no +such cap. Explains "50 MB+" (bigger PDF → more scattered objects → bigger +flood) and "crashes on some devices only" (heap/WebView threshold). + +**Fix (PR TODO):** `packages/foliate-js/pdf.js` `makePDF` — queue + pump bounding +range reads to `MAX_CONCURRENT_RANGES = 6` (mimics the browser's per-host +limit). One spot covers Android `RemoteFile`, iOS `NativeFile`, web `File`. +Throttling is **free** on speed (6 parallel fetches saturate throughput). foliate-js +is a **git submodule** → commit + push to readest fork, then bump pointer. +Test: `src/__tests__/foliate-pdf-range-concurrency.test.ts` — `vi.mock('@pdfjs/pdf.min.mjs')` installs a fake `globalThis.pdfjsLib` whose `getDocument` fires a 200-call flood; asserts `maxInFlight ≤ 6` and all served. Fails (200) before, passes after. + +## On-device CDP verification recipe (no rebuild) +Release Readest 0.11.10 ships a debuggable WebView (socket +`webview_devtools_remote_`), so CDP attaches without `run-as`. +- `adb forward tcp:9222 localabstract:webview_devtools_remote_$PID`; page WS from `curl :9222/json`. +- Push file where asset scope allows: `/sdcard/Readest/Books/` matches scope glob `**/Readest/**/*`; app has MANAGE_EXTERNAL_STORAGE → readable. Canonical path `/storage/emulated/0/Readest/Books/x.pdf`. +- rangefile URL: `http://rangefile.localhost/?path=&start=&end=` (end **inclusive**, omit=EOF, 8 MB cap, returns 200 + `X-Total-Size`). +- Faithfully replicate `makePDF`: `await import('http://tauri.localhost/vendor/pdfjs/pdf.min.mjs')` (sets `globalThis.pdfjsLib`, same vendored 5.7.284), a file-like `{size, slice(b,e)→{arrayBuffer:()=>fetchRangePart(b,e-1)}}`, `new pdfjsLib.PDFDataRangeTransport(size,[])`, instrument `requestDataRange`, `getDocument({range,wasmUrl:'/vendor/pdfjs/',cMapUrl,standardFontDataUrl,isEvalSupported:false})` then `getPage(1)/getViewport/getMetadata`. +- Java heap via `adb shell dumpsys meminfo com.bilingify.readest` (Dalvik Heap line). + +**Verified on Xiaomi 13 (fuxi) / Android 16 / WebView 147 / 8 GB:** this device does NOT OOM (newer WebView; Dalvik only +9 MB) but the flood reproduces: **753 → 6** concurrent, open time **1446 → 1479 ms** (no penalty), 970 pages/title/viewport identical. Gotcha: package installs (`installPackageLI` in logcat) kill the app mid-session → re-discover the devtools socket PID. The makePDF flood alone did NOT crash this device — can't get a live OOM here; rely on the user's WebView-145 log + the bounded-concurrency proof. + +Related: [[android-nativefile-remotefile-io]] (rangefile vs asset-protocol Range bug), [[webtoon-mode-3647]] (foliate-js submodule fork-push). diff --git a/apps/readest-app/.claude/memory/zindex-overlay-scale.md b/apps/readest-app/.claude/memory/zindex-overlay-scale.md new file mode 100644 index 00000000..4aec2057 --- /dev/null +++ b/apps/readest-app/.claude/memory/zindex-overlay-scale.md @@ -0,0 +1,48 @@ +--- +name: zindex-overlay-scale +description: Global overlay z-index scale; why Add-Catalog-behind-Settings was mobile-only (window-border trap); RSVP de-escalated from 10000 +metadata: + node_type: memory + type: project + originSessionId: e7590344-aa6d-4bec-9b6d-6f3b93b18c87 +--- + +RESOLVED — PR #4669 (merged 2026-06-19). Redesigned the overlay z-index scale (was: RSVP `z-[10000]`, Settings `!z-[10050]`, +ModalPortal `z-[100]`). Compact scale now, all clearing the desktop `.window-border` +page frame (`z-99` in `globals.css`): + +- `100` RSVP immersive overlay (`RSVPOverlay.tsx`) +- `101` RSVP controls — start dialog + lookup chip (`RSVPStartDialog.tsx`, `RSVPOverlay.tsx`) +- `110` Settings dialog (`SettingsDialog.tsx`, `!z-[110]`) +- `120` modal / command palette (`ModalPortal.tsx`, `CommandPalette.tsx`) +- `130` toast (`Alert.tsx`) +- `200` app-lock (`AppLockScreen.tsx`, unchanged) + +**Bug fixed:** "Add OPDS Catalog" dialog (a `ModalPortal`, opened via `CatalogManager +inSubPage` inside Settings → Integrations) rendered BEHIND the Settings sheet on mobile. +Root cause = regression from #3235, which raised Settings to `!z-[10050]` (to beat the +RSVP `z-[10000]` overlay for in-overlay dictionary mgmt) — that jumped Settings above the +`ModalPortal` layer (`z-[100]`), so any modal opened from inside Settings was buried. + +**Why MOBILE-ONLY (non-obvious):** `Dialog` does NOT portal — `SettingsDialog` renders +inline inside `.reader-page` (`ReaderContent.tsx:273`). On desktop rounded-window, +`.reader-page` has `.window-border` (`z-99`, `position:relative`) = a stacking context +that TRAPS Settings at the `z-99` layer. `ModalPortal` uses `createPortal(document.body)` +→ escapes to body `z-100+` → already wins on desktop. On mobile `hasRoundedWindow` is +false → no `.window-border` → Settings' `z-10050` competes at body level and buries the +modal. So RSVP must stay **≥100** to cover the `z-99` frame on desktop (an early `z-[70]` +idea would have broken desktop RSVP). + +**Invariant lock:** `src/__tests__/styles/zIndexScale.test.ts` reads the z values straight +from source and asserts MODAL>SETTINGS>RSVP_CTRL>RSVP>99, APP_LOCK>MODAL, and all <1000. +This static test would have caught #3235. Scale also documented in `DESIGN.md` §6 and a +comment block in `ModalPortal.tsx`. + +**On-device verify recipe (Xiaomi fuxi):** release build has WebView debugging OFF (no CDP +socket). `pnpm dev-android` builds the RELEASE-signed APK with `--features devtools` and +`adb install -r` — same signing key (`65:2D:..`), replaces in place, KEEPS data, enables +`@webview_devtools_remote_`. Proof of the bug = `document.elementFromPoint(cx,cy)` at +the Add-Catalog modal-box center returns a Settings `

` (topInsideSettings:true); after +fix returns the Add-Catalog `FORM`. Drove UI via `adb shell input tap` (logical*3 = physical +on this 1080×2400/360×800 device) + stdlib-only CDP ws client at `/tmp/cdp.py`. +Related: [[android-cdp-e2e-lane]], [[cdp-android-webview-profiling]], [[tts-sync-paragraph-rsvp-3235]]. diff --git a/apps/readest-app/src/__tests__/foliate-pdf-range-concurrency.test.ts b/apps/readest-app/src/__tests__/foliate-pdf-range-concurrency.test.ts new file mode 100644 index 00000000..bfc62f35 --- /dev/null +++ b/apps/readest-app/src/__tests__/foliate-pdf-range-concurrency.test.ts @@ -0,0 +1,113 @@ +// Regression test for readest/readest issue #3470. +// +// Opening a large PDF makes pdf.js request hundreds of small byte ranges at +// once while it parses the cross-reference and object streams. foliate-js' +// `makePDF` used to fulfil every `requestDataRange` immediately via an +// un-awaited `file.slice(begin, end).arrayBuffer()`, so all of those reads ran +// concurrently. On Android each read is a `fetch()` to the custom `rangefile` +// scheme that the WebView serves through `shouldInterceptRequest`; firing +// hundreds at once floods that native handler and exhausts the 512 MB Java +// heap, crashing the app on 50 MB+ PDFs. +// +// A real HTTP transport is implicitly throttled by the browser's per-host +// connection limit (~6); the custom file scheme bypasses that. `makePDF` must +// therefore throttle the concurrent range reads itself. This test drives a +// flood of `requestDataRange` calls and asserts the number of simultaneous +// `file.slice()` reads stays bounded while every requested range is still +// served. + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +// The number of range requests pdf.js fires in the simulated parse burst. +const FLOOD = 200; + +// Captured by the @pdfjs mock so the test can drive requestDataRange. +let rangeTransport: { + requestDataRange: (b: number, e: number) => void; + onDataRange: (b: number, c: ArrayBuffer) => void; +}; + +// Minimal stand-in for the vendored pdf.js build. foliate-js/pdf.js imports it +// only for the side effect of setting globalThis.pdfjsLib, then reads from +// that global — so the mock installs a controllable fake there. +vi.mock('@pdfjs/pdf.min.mjs', () => { + class PDFDataRangeTransport { + requestDataRange!: (begin: number, end: number) => void; + onDataRange = vi.fn(); + constructor( + public length: number, + public initialData: unknown, + ) {} + } + const fakePdf = { + numPages: 100, + getPage: vi.fn(async () => ({ + getViewport: () => ({ width: 600, height: 800 }), + cleanup: vi.fn(), + })), + getMetadata: vi.fn(async () => ({ metadata: undefined, info: {} })), + getOutline: vi.fn(async () => null), + getDestination: vi.fn(), + getPageIndex: vi.fn(), + destroy: vi.fn(), + }; + const getDocument = vi.fn(({ range }: { range: typeof rangeTransport }) => { + rangeTransport = range; + const promise = (async () => { + // pdf.js fires a burst of range requests as it parses the PDF structure. + for (let i = 0; i < FLOOD; i++) range.requestDataRange(i * 1000, i * 1000 + 999); + await new Promise((r) => setTimeout(r, 0)); + return fakePdf; + })(); + return { promise }; + }); + (globalThis as unknown as { pdfjsLib: unknown }).pdfjsLib = { + GlobalWorkerOptions: {}, + PDFDataRangeTransport, + getDocument, + }; + return {}; +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('makePDF range-read concurrency (#3470)', () => { + it('bounds simultaneous range reads and still serves every request', async () => { + const { makePDF } = await import('foliate-js/pdf.js'); + + let inFlight = 0; + let maxInFlight = 0; + let served = 0; + const file = { + size: FLOOD * 1000, + slice(begin: number, end: number) { + return { + async arrayBuffer() { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((r) => setTimeout(r, 5)); + inFlight--; + served++; + return new ArrayBuffer(end - begin); + }, + }; + }, + }; + + await makePDF(file as unknown as File); + + // Wait for the throttled queue to drain every requested range. + const start = Date.now(); + while (served < FLOOD && Date.now() - start < 5000) { + await new Promise((r) => setTimeout(r, 5)); + } + + // Every range pdf.js asked for must still be delivered… + expect(served).toBe(FLOOD); + // …but never more than a browser-like per-host connection limit at once. + expect(maxInFlight).toBeGreaterThan(0); + expect(maxInFlight).toBeLessThanOrEqual(6); + }); +}); diff --git a/packages/foliate-js b/packages/foliate-js index 1cb26e3f..e098bc3e 160000 --- a/packages/foliate-js +++ b/packages/foliate-js @@ -1 +1 @@ -Subproject commit 1cb26e3fe2c4b9612918b4d2fa66467a97bf3734 +Subproject commit e098bc3ef522ed0b6d07a9878eb3a743af440c5f