From 3b348c8f35960adc90a47e2ed9e1a905839a1519 Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Wed, 6 May 2026 21:50:15 +0800 Subject: [PATCH] feat(sync): CRDT replica sync foundation (#4075) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(sync): foundation for CRDT-based cross-device replica sync (Phase 1+2) Adds the primitives and orchestration layer for syncing user-imported assets (dictionaries, fonts, textures, OPDS catalogs, dict settings) across devices via a polymorphic `replicas` table with field-level LWW under HLC ordering. Phase 1 ships the foundation (CRDT, crypto, server schemas, SQL migrations, push/pull endpoint); Phase 2 adds the adapter registry, HTTP client, and sync manager. No modifications to existing book sync — additive only. Phase 1: - src/libs/crdt.ts — HlcGenerator (monotonic + remote-absorption + clock-regression-safe), per-field LWW with deviceId tiebreak, remove-wins tombstones, reincarnation token revival. - src/libs/crypto/{derive,encrypt,envelope,passphrase}.ts — PBKDF2-600k key derivation (OWASP 2024), AES-GCM round-trip, envelope {c,i,s,alg,h} with SHA-256 sidecar integrity check, passphrase storage abstraction (web ephemeral; Tauri keychain stub). - src/libs/replica-schemas.ts — Zod-backed allowlist (dictionary only in PR 1), 64KiB row cap, 64-field cap, schemaVersion bounds, filename validator. - src/libs/replica-sync-server.ts — push batch validation (auth + allowlist + schema + HLC ±60s skew clamp). - src/pages/api/sync/replicas.ts — POST/GET endpoint wrapping the Postgres crdt_merge_replica function via RPC. - docker/volumes/db/migrations/003_add_replicas.sql — replicas table + replica_keys table + RLS. - docker/volumes/db/migrations/004_crdt_merge_replica_fn.sql — atomic per-field LWW merge function (forwards-compat preserves unknown fields). Phase 2: - src/services/sync/replicaRegistry.ts — adapter contract (core + optional BinaryCapability + LifecycleHooks per eng review). - src/libs/replica-sync-client.ts — HTTP wrapper mapping status codes to typed SyncError codes. - src/services/sync/replicaSyncManager.ts — 5s debounced push, immediate flush on visibilitychange/online, per-kind pull cursor, remote HLC absorption. Tests: 125 new (crdt 26, crypto 32, schemas 21, server 16, client 12, registry 6, manager 12). Full suite 3656 passing, lint clean. Existing book/config/note sync paths untouched. Plan: ~/.claude/plans/vivid-orbiting-thimble.md CEO plan: ~/.gstack/projects/readest-readest/ceo-plans/2026-05-06-replica-sync-cathedral.md Co-Authored-By: Claude Opus 4.7 (1M context) * feat(sync): add kind="replica" path through TransferManager (Phase 3) Adds the replica branch to the existing book-shaped transfer infrastructure so dictionary (and future kinds) bundles can flow through the same queue, retry, and progress UI as book uploads. Existing book transfer paths remain unchanged. The book-side regression suite (37 tests in transfer-store.test.ts, 37 in transfer-manager.test.ts) all stay green. Store (src/store/transferStore.ts): - TransferItem gains kind: 'book' | 'replica' (default 'book' on legacy persisted rows), replicaKind, replicaId, replicaFiles, replicaBase. - New addReplicaTransfer(replicaKind, replicaId, displayTitle, type, opts) with files + base in opts; auto-computes totalBytes from file sizes. - New getReplicaTransfer(replicaKind, replicaId, type) lookup. - getTransferByBookHash filters to kind === 'book' (defensive against bookHash="" collisions on replica items). - restoreTransfers fills kind: 'book' for legacy persisted rows. Manager (src/services/transferManager.ts): - queueReplicaUpload / queueReplicaDownload / queueReplicaDelete. - executeTransfer dispatches by kind to the new executeReplicaTransfer (iterates files, calls appService.uploadReplicaFile per file with per-file progress aggregation) or the existing executeBookTransfer (refactored out, byte-identical behavior). - Dispatches replica-transfer-complete event on success so stores can react (e.g., commit manifest_jsonb to the replica row). Storage / cloud (src/libs/storage.ts, src/services/cloudService.ts): - uploadReplicaFile bypasses the book-only File.name smuggling and takes an explicit cfp (cloud file path). - uploadReplicaFileToCloud / downloadReplicaFileFromCloud / deleteReplicaBundleFromCloud orchestrate per-file operations under ${userId}/Readest/replicas///. - replicaCloudKey() centralizes the path-construction rule. - New CLOUD_REPLICAS_SUBDIR constant. App service (src/services/appService.ts, src/types/system.ts): - AppService gains uploadReplicaFile, downloadReplicaFile, deleteReplicaBundle (file-level operations; orchestration lives in TransferManager). Tests: 18 new (12 in transfer-store.test.ts, 5 in transfer-manager.test.ts, 1 fixture). Full suite 3674 passing, lint clean. Existing book regression clean. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(sync): dictionary replica adapter + bootstrap (Phase 4, partial) Lands the safe-to-ship foundation of Phase 4 — adapter logic, registry bootstrap, and SystemSettings hooks for replica sync. The on-disk migration of legacy customDictionaries (bundleDir → content-hash id), the live store wiring, and the Settings → Sync UI are deferred to a follow-up PR so they can land with real-device QA. Adapter (src/services/sync/adapters/dictionary.ts): - dictionaryAdapter: kind='dictionary', schemaVersion=1. - pack/unpack — only synced subset (name, kind, lang, addedAt, unsupported{,Reason}). bundleDir / files / unavailable / deletedAt stay per-device or are handled by the tombstone mechanism. - BinaryCapability.enumerateFiles dispatches by bundle kind: - mdict: mdx + mdd[] + css[] - stardict: ifo + idx + dict + syn (skips .idx.offsets / .syn.offsets sidecars — those are device-local indices) - dict: dict + index - slob: single .slob file - primaryDictionaryFile() picks the anchor file per kind for partialMD5 hashing. - computeDictionaryReplicaId(partialMd5, byteSize, sortedFilenames) produces a deterministic 32-hex content-hash id used at import time. - 23 tests cover pack/unpack identity, kind dispatch, file enumeration, id determinism, and per-device-field exclusion. Bootstrap (src/services/sync/replicaBootstrap.ts): - bootstrapReplicaAdapters() registers all known adapters once at app start. Idempotent (safe to call multiple times). Wired into EnvContext.tsx so the registry populates on app mount. - 3 tests cover registration, idempotency, and the PR-1 allowlist. SystemSettings (src/types/settings.ts, src/services/constants.ts): - +SyncCategory = 'book' | 'progress' | 'note' | 'dictionary' — typed union for the user-facing sync toggles. 'progress' gates the existing book-config sync (reading progress); 'note' gates annotations; 'book' gates book binaries + metadata; 'dictionary' gates the new replica sync. Future replica kinds extend the union. - +SYNC_CATEGORIES readonly array for UI iteration. - +syncCategories: Partial> — per- category opt-in toggles in DEFAULT_SYSTEM_SETTINGS (default ON for all four). UI panel ships in the follow-up. - +lastSyncedAtReplicas: Record — per-kind HLC pull cursors (matches replicaSyncManager's CursorStore contract). Registry type cleanup (src/services/sync/replicaRegistry.ts): - BinaryCapability.enumerateFiles return shape: localRelPath → lfp to match the existing TransferStore.ReplicaTransferFile convention. Tests: 28 new (23 dict + 3 bootstrap + 2 syncCategories defaults). Full suite 3702 passing, lint clean. Existing book/config/note sync paths untouched. Deferred to PR 1 follow-up (with real-device QA): - customDictionaryStore migration: rehash legacy uniqueId() bundleDir to content-hash id; preserve providerOrder mapping; staged .legacy// backup. - Wire customDictionaryStore mutations through replicaSyncManager (markDirty on add/rename/delete; pull on init). - Settings → Sync panel: per-category toggles + last-sync timestamps. - Sync passphrase modal: set / change / forgot flow (lazy first prompt on encrypted-field push/pull). - in CustomDictionaries.tsx for "Download from cloud (X MB)" affordance. - Tauri keychain backend for sync passphrase storage. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(sync): replicaSync singleton + content-hash id at dict import (Phase 4b) Two foundation pieces that everything UI-side will sit on top of, both purely additive — no live store wiring, no behavior change for existing dictionary imports. replicaSync singleton (src/services/sync/replicaSync.ts): - initReplicaSync({deviceId, cursorStore, hlcStore?, client?}) builds one ReplicaSyncManager backed by an HlcGenerator with persistence wrapped around .next()/.observe(). Idempotent (second init returns the existing instance). - LocalStorageHlcStore (src/libs/hlc-store.ts) snapshots the HLC counter under 'readest_replica_hlc' so it survives restart. Falls back silently when localStorage is unavailable (private mode, SSR); client re-derives via the existing remote max(updated_at_ts) repair path. InMemoryHlcStore is the test backend. - 16 tests (9 hlc-store, 7 replicaSync) covering snapshot persistence, restore-on-init, and idempotency. - Wiring into EnvContext for production deferred to the follow-up that also adds the cursor store backed by useSettingsStore. Content-hash id at dictionary import (src/services/dictionaries/contentId.ts): - computeDictionaryContentId(primaryFile, filenames) wraps computeDictionaryReplicaId(partialMd5(primary), byteSize, sortedFilenames) — the cross-device id used as the replica_id when the dict actually pushes/pulls. - Wired into all four import paths in dictionaryService.ts: - stardict primary = .ifo (small text, partialMD5 ≈ full hash) - mdict primary = .mdx (body) - dict primary = .dict.dz (gzipped body) - slob primary = .slob (single-file bundle) ImportedDictionary gains contentId?: string. Optional for backwards compat; legacy bundles without contentId are flagged as "needs rehash before sync" by the upcoming store-wiring follow-up. - 6 tests cover identity determinism, byteSize sensitivity, filename- set sensitivity, and order-independence. Tests: 22 new (9 + 7 + 6). Full suite 3724 passing, lint clean. Existing dictionary import flow unchanged for users — contentId is an additional field, not a replacement for the bundleDir-based id. Deferred to follow-up (with on-device QA): - Production cursor store backed by useSettingsStore + appService.saveSettings. - EnvContext call to initReplicaSync after appService boot. - customDictionaryStore mutation hooks → replicaSyncManager.markDirty. - Legacy bundleDir → contentId migration with .legacy/ backup. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(sync): land replica-sync design plan in repo Moves the plan document that drove this PR's foundation work (`~/.claude/plans/vivid-orbiting-thimble.md`) into the project tree at `apps/readest-app/.claude/plans/` so reviewers and future contributors can read it alongside the code without leaving the repo. The plan went through three review passes — Codex (19 findings, all absorbed), CEO/scope review (mode SCOPE EXPANSION; encrypted secrets pulled forward to v1, "private-only forever" posture lock), and eng review (FULL_REVIEW mode, 16 findings absorbed). The full review trail lives in the file's `## GSTACK REVIEW REPORT` section at the bottom. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(crdt): point README at in-repo plan path Now that vivid-orbiting-thimble.md lives at apps/readest-app/.claude/plans/, the README link should point there rather than at the home-dir copy that no longer exists. Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(sync): rename src/libs files to camelCase per project convention Test files renamed in lockstep. Imports + comment references updated across cloudService, storage, transferManager, replicaSync, replicaSyncManager, /api/sync/replicas, and all four test files. No behavior change. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .../.claude/plans/vivid-orbiting-thimble.md | 1066 +++++++++++++++++ .../src/__tests__/libs/crdt.test.ts | 267 +++++ .../src/__tests__/libs/crypto/derive.test.ts | 60 + .../src/__tests__/libs/crypto/encrypt.test.ts | 91 ++ .../__tests__/libs/crypto/envelope.test.ts | 89 ++ .../__tests__/libs/crypto/passphrase.test.ts | 55 + .../src/__tests__/libs/hlcStore.test.ts | 78 ++ .../src/__tests__/libs/replicaSchemas.test.ts | 178 +++ .../__tests__/libs/replicaSyncClient.test.ts | 153 +++ .../__tests__/libs/replicaSyncServer.test.ts | 155 +++ ...are-server.test.ts => shareServer.test.ts} | 4 +- .../src/__tests__/services/constants.test.ts | 14 + .../services/dictionaries/contentId.test.ts | 45 + .../services/sync/adapters/dictionary.test.ts | 205 ++++ .../services/sync/replicaBootstrap.test.ts | 35 + .../services/sync/replicaRegistry.test.ts | 62 + .../services/sync/replicaSync.test.ts | 125 ++ .../services/sync/replicaSyncManager.test.ts | 183 +++ .../services/transfer-manager.test.ts | 117 ++ .../__tests__/store/transfer-store.test.ts | 131 ++ .../src/app/api/share/[token]/cover/route.ts | 2 +- .../share/[token]/download/confirm/route.ts | 2 +- .../app/api/share/[token]/download/route.ts | 2 +- .../src/app/api/share/[token]/import/route.ts | 2 +- .../app/api/share/[token]/og.png/render.tsx | 2 +- .../src/app/api/share/[token]/revoke/route.ts | 2 +- .../src/app/api/share/[token]/route.ts | 2 +- .../src/app/api/share/create/route.ts | 2 +- apps/readest-app/src/app/s/page.tsx | 2 +- apps/readest-app/src/context/EnvContext.tsx | 2 + apps/readest-app/src/libs/crdt.README.md | 103 ++ apps/readest-app/src/libs/crdt.ts | 192 +++ apps/readest-app/src/libs/crypto/derive.ts | 41 + apps/readest-app/src/libs/crypto/encrypt.ts | 46 + apps/readest-app/src/libs/crypto/envelope.ts | 83 ++ .../readest-app/src/libs/crypto/passphrase.ts | 49 + apps/readest-app/src/libs/errors.ts | 48 + apps/readest-app/src/libs/hlcStore.ts | 49 + apps/readest-app/src/libs/replicaSchemas.ts | 172 +++ .../readest-app/src/libs/replicaSyncClient.ts | 94 ++ .../readest-app/src/libs/replicaSyncServer.ts | 121 ++ .../libs/{share-server.ts => shareServer.ts} | 0 apps/readest-app/src/libs/storage.ts | 37 + .../src/pages/api/sync/replicas.ts | 146 +++ apps/readest-app/src/services/appService.ts | 38 + apps/readest-app/src/services/cloudService.ts | 68 +- apps/readest-app/src/services/constants.ts | 8 + .../src/services/dictionaries/contentId.ts | 24 + .../dictionaries/dictionaryService.ts | 25 + .../src/services/dictionaries/types.ts | 8 + .../src/services/sync/adapters/dictionary.ts | 115 ++ .../src/services/sync/replicaBootstrap.ts | 22 + .../src/services/sync/replicaRegistry.ts | 39 + .../src/services/sync/replicaSync.ts | 67 ++ .../src/services/sync/replicaSyncManager.ts | 122 ++ .../src/services/transferManager.ts | 219 +++- apps/readest-app/src/store/transferStore.ts | 87 +- apps/readest-app/src/types/replica.ts | 54 + apps/readest-app/src/types/settings.ts | 28 + apps/readest-app/src/types/system.ts | 16 + .../db/migrations/003_add_replicas.sql | 84 ++ .../migrations/004_crdt_merge_replica_fn.sql | 189 +++ 62 files changed, 5489 insertions(+), 38 deletions(-) create mode 100644 apps/readest-app/.claude/plans/vivid-orbiting-thimble.md create mode 100644 apps/readest-app/src/__tests__/libs/crdt.test.ts create mode 100644 apps/readest-app/src/__tests__/libs/crypto/derive.test.ts create mode 100644 apps/readest-app/src/__tests__/libs/crypto/encrypt.test.ts create mode 100644 apps/readest-app/src/__tests__/libs/crypto/envelope.test.ts create mode 100644 apps/readest-app/src/__tests__/libs/crypto/passphrase.test.ts create mode 100644 apps/readest-app/src/__tests__/libs/hlcStore.test.ts create mode 100644 apps/readest-app/src/__tests__/libs/replicaSchemas.test.ts create mode 100644 apps/readest-app/src/__tests__/libs/replicaSyncClient.test.ts create mode 100644 apps/readest-app/src/__tests__/libs/replicaSyncServer.test.ts rename apps/readest-app/src/__tests__/libs/{share-server.test.ts => shareServer.test.ts} (97%) create mode 100644 apps/readest-app/src/__tests__/services/dictionaries/contentId.test.ts create mode 100644 apps/readest-app/src/__tests__/services/sync/adapters/dictionary.test.ts create mode 100644 apps/readest-app/src/__tests__/services/sync/replicaBootstrap.test.ts create mode 100644 apps/readest-app/src/__tests__/services/sync/replicaRegistry.test.ts create mode 100644 apps/readest-app/src/__tests__/services/sync/replicaSync.test.ts create mode 100644 apps/readest-app/src/__tests__/services/sync/replicaSyncManager.test.ts create mode 100644 apps/readest-app/src/libs/crdt.README.md create mode 100644 apps/readest-app/src/libs/crdt.ts create mode 100644 apps/readest-app/src/libs/crypto/derive.ts create mode 100644 apps/readest-app/src/libs/crypto/encrypt.ts create mode 100644 apps/readest-app/src/libs/crypto/envelope.ts create mode 100644 apps/readest-app/src/libs/crypto/passphrase.ts create mode 100644 apps/readest-app/src/libs/errors.ts create mode 100644 apps/readest-app/src/libs/hlcStore.ts create mode 100644 apps/readest-app/src/libs/replicaSchemas.ts create mode 100644 apps/readest-app/src/libs/replicaSyncClient.ts create mode 100644 apps/readest-app/src/libs/replicaSyncServer.ts rename apps/readest-app/src/libs/{share-server.ts => shareServer.ts} (100%) create mode 100644 apps/readest-app/src/pages/api/sync/replicas.ts create mode 100644 apps/readest-app/src/services/dictionaries/contentId.ts create mode 100644 apps/readest-app/src/services/sync/adapters/dictionary.ts create mode 100644 apps/readest-app/src/services/sync/replicaBootstrap.ts create mode 100644 apps/readest-app/src/services/sync/replicaRegistry.ts create mode 100644 apps/readest-app/src/services/sync/replicaSync.ts create mode 100644 apps/readest-app/src/services/sync/replicaSyncManager.ts create mode 100644 apps/readest-app/src/types/replica.ts create mode 100644 docker/volumes/db/migrations/003_add_replicas.sql create mode 100644 docker/volumes/db/migrations/004_crdt_merge_replica_fn.sql diff --git a/apps/readest-app/.claude/plans/vivid-orbiting-thimble.md b/apps/readest-app/.claude/plans/vivid-orbiting-thimble.md new file mode 100644 index 00000000..8adacf03 --- /dev/null +++ b/apps/readest-app/.claude/plans/vivid-orbiting-thimble.md @@ -0,0 +1,1066 @@ +# Plan: CRDT-based user-replica sync (server-allowlisted kinds) + +## Context + +User-imported data and configuration live scattered through `SystemSettings` +and never leave the device: + +- `customDictionaries: ImportedDictionary[]` (multi-file bundles under + `'Dictionaries'//`, identity `uniqueId()` — random per device) +- `customFonts: CustomFont[]` (single file under `'Fonts'/`, id `md5(name)`) +- `customTextures: CustomTexture[]` (single file under `'Images'/`, id + `md5(name)`) +- `opdsCatalogs: OPDSCatalog[]` (URL + credentials, no binaries) +- `dictionarySettings.{providerOrder, providerEnabled, defaultProviderId, + webSearches}` + +`SystemSettings` is written via `safeSaveJSON` and not synced anywhere — the +"syncs to cloud" comment in `customDictionaryStore.ts:81` is misleading. A +user signed in on two devices has to re-import everything on the second +device, and any settings tweak on one device is invisible on the other. + +Books already sync along two rails: per-record LWW metadata via the sync-DB +protocol (`src/libs/sync.ts` ↔ `src/pages/api/sync.ts`) and binaries via +`TransferManager` → S3/R2 storage. We want **the same plumbing** for user +replicas, with **CRDT semantics** so concurrent offline edits on two devices +converge automatically when they reconnect. + +This plan was reviewed by Codex and substantially revised. Major shifts from +the first draft: kinds are now server-allowlisted (not free-form), the merge +is atomic on the server (not API-level fetch-then-upsert), deletion is +remove-wins (not "edit beats tombstone"), and OPDS credentials are NOT +synced in plaintext. + +After Codex revision the plan went through `/plan-ceo-review` (mode: SCOPE +EXPANSION). The user picked Approach A (the cathedral) over per-kind LWW +tables and manual share+import, and accepted **encrypted secrets in v1** +(was deferred to v2 in the Codex-revised draft). The CEO review also +locked a permanent strategic posture: **Readest stays private-only**. +Cross-user content sharing (public/unlisted dictionary library) is not +deferred — it is explicitly out of scope forever. See +`~/.gstack/projects/readest-readest/ceo-plans/2026-05-06-replica-sync-cathedral.md` +for the full decision capture. + +## Design tenets + +1. **`kind` is from a server-managed allowlist.** The DB has a CHECK + constraint; the server validates every push against a per-kind schema + (allowed field names, max JSON size, max field count, filename rules, + per-user quota). Clients use adapter modules for dispatch ergonomics, but + adding a new kind is a small **coordinated** client+server change. No + "drop one file, done." +2. **One polymorphic DB table** holds every allowlisted kind. Kind-agnostic + columns; per-kind data in a JSONB blob whose shape the server validates. +3. **One CRDT merge function** runs atomically on the server, regardless of + kind. Field-level LWW with HLC timestamps. Atomic via Postgres RPC or + `INSERT … ON CONFLICT … DO UPDATE` with a SQL-side merge call. +4. **One transport.** Push/pull, the TransferManager refactor, the cloud + key scheme, the auth check — all kind-agnostic but kind-validated. +5. **User opts in per-category.** A settings panel lets the user enable or + disable sync for each category (books, progress, annotations, fonts, + dictionaries, textures, OPDS catalogs, ...). Disabled categories don't + push or pull. +6. **Sensitive fields are never synced in plaintext.** A per-account sync + passphrase (separate from auth password) derives a key via PBKDF2-600k + (OWASP 2024). Sensitive fields encrypt with AES-GCM before push. + Encryption lives in each adapter's `pack`/`unpack` (the adapter decides + which fields are sensitive; HLC stamps the envelope, not plaintext). + Encryption ships in v1, not v2. +7. **Private-only forever.** Cross-user content sharing (public/unlisted + replicas, community dictionary library) is explicitly out of scope. + The polymorphic `replicas` table is internal infrastructure, not a + future API. Adding any cross-user-visibility feature requires a fresh + architecture review and is not authorized by this plan. + +## Architecture overview + +### Layer 1 — Identity & on-disk storage + +- Each replica has an `id: string` stable across devices. Identity = `H(primaryFile partial bytes ‖ byteSize ‖ filename list)` — `partialMD5` alone + is too collision-prone for adversarial inputs (Codex finding). For + binary-backed kinds the primary file is the natural anchor (`.mdx` for + MDict, `.dict.dz` for StarDict, `.slob` for Slob, the font/image bytes + for fonts/textures). For metadata-only kinds (OPDS catalog, web search, + provider position/pref), the adapter computes id from natural keys + (URL+username, web-search uuid, provider id). +- For 1+ GB MDDs we may also schedule a background full streaming hash and + store it as `strongHash` for tamper detection on re-download. v1 ships + with `partialMD5` + size + filename mix; full hash is a follow-up. +- Local on-disk paths are per-adapter. Existing `BaseDir`s + (`'Dictionaries'`, `'Fonts'`, `'Images'`) reused for the current + kinds. +- Cloud key for binary files: `${userId}/Readest/replicas///`. + Filenames are server-validated: no `..`, no `/`, no `\`, length ≤ 255, + charset restricted. +- A `manifest.json` lists `{filename, byteSize, partialMd5}[]`. The + upload state machine writes the manifest LAST. A row is only marked + `downloadable: true` once `manifestUploadedAt` is set. (Codex finding — + prevents partial-upload races.) + +### Layer 2 — CRDT-backed metadata sync + +Single Postgres table `replicas`: + +``` +user_id uuid NOT NULL +kind text NOT NULL CHECK (kind IN (... server allowlist ...)) +replica_id text NOT NULL +fields_jsonb jsonb NOT NULL -- per-field CRDT register; max 64 KiB; max 64 fields +manifest_jsonb jsonb NULL -- populated only after binary upload completes +deleted_at_ts text NULL -- HLC string of remove-wins tombstone (never resurrected) +reincarnation text NULL -- new-id token for explicit re-import revival +updated_at_ts text NOT NULL -- max(field ts, deleted_at_ts) +PRIMARY KEY (user_id, kind, replica_id) +INDEX (user_id, kind, updated_at_ts) +RLS: auth.uid() = user_id +CHECK pg_column_size(fields_jsonb) <= 65536 +CHECK jsonb_object_keys count <= 64 +``` + +Per-user, per-kind row quotas enforced server-side. + +`fields_jsonb` shape (CRDT envelope per field): + +``` +{ + name: { v: , t: '', s: '' }, + enabled: { v: true, t: '...', s: '...' }, + ... +} +``` + +CRDT type: **LWW-Element-Set with per-field LWW-Register and remove-wins +tombstones.** + +- **HLC** (Hybrid Logical Clock) — `(physical_ms, logical_counter, + device_id)`, packed as a sortable string. Clock skew handled by the + physical component; same-ms ties handled by the counter. **Server clamps + incoming HLCs**: physical_ms must be within ±60s of server time + (configurable). Out-of-range writes are rejected with a clock-skew error + the client can recover from. (Codex finding — prevents far-future HLC + freezing future writes.) +- **HLC persistence** lives in IndexedDB or the Tauri keyring (a small + per-account key), not just `localStorage`. Multi-window, storage-clear, + and reinstall are handled by re-deriving from the remote `max(updated_at_ts)` + on next pull. (Codex finding.) +- **Per-field LWW-Register** — for each scalar field, keep the entry with + the largest HLC. Same-field concurrent edits on offline devices: the + loser's value drops. Documented per-kind (acceptable for `name`, + `enabled`, `defaultProviderId`; explicitly NOT acceptable for any + `Map`/`Set`/array — those split into separate rows, see below). +- **Remove-wins tombstones** — `deleted_at_ts` is the record's tombstone + HLC. **A field write does NOT revive a tombstoned record by HLC alone** + (the original draft's rule was unsafe — Codex finding). Revival happens + only via a fresh **reincarnation token**: when an import on Device B + produces an id that matches a tombstoned row, the importer writes a new + `reincarnation` value and a fresh row; the tombstone stays put as + history. The client merges `(deleted=true, reincarnation=null)` rows by + hiding them; rows with `reincarnation != prior_value` are surfaced as + alive. + +Merge function `mergeReplica(local, remote)` is **deterministic, +commutative, associative, idempotent** — the four CRDT properties — and +lives in `src/libs/crdt.ts`. **The server runs this merge atomically** via +either: + +- a Postgres function `crdt_merge_replica(...)` invoked through `INSERT + … ON CONFLICT (user_id, kind, replica_id) DO UPDATE SET fields_jsonb = + crdt_merge_replica(replicas.fields_jsonb, EXCLUDED.fields_jsonb), …`, + OR +- an explicit RPC `rpc_push_replicas(...)` with row locking + (`SELECT … FOR UPDATE`) in a single transaction. + +Either way the merge happens **inside one SQL statement / transaction**, so +two concurrent pushes can't interleave a fetch-then-upsert race. (Codex +finding — the existing `src/pages/api/sync.ts` has separate fetch and +upsert calls; this is an honest protocol upgrade.) + +Helpers in `src/libs/crdt.ts`: + +- `Hlc.next()` — bumps local counter; persists to IndexedDB. +- `setField(replica, fieldName, value)` — writes `{v, t: Hlc.next(), s}`. +- `removeReplica(replica)` — sets `deleted_at_ts = Hlc.next()`. Permanent. +- `mergeFields(local, remote)` / `mergeReplica(local, remote)` — pure. + +### Layer 3 — Sync transport & protocol changes + +`src/libs/sync.ts` extension is **not** a one-line change (Codex finding). +`SyncType`, `SyncResult`, and `SyncData` are hard-coded today. The honest +protocol upgrade: + +- `SyncType` extends from `'books' | 'configs' | 'notes'` to add `'replicas'`. +- `SyncData` and `SyncResult` add a `replicas: ReplicaRow[]` arm. +- A new RPC route `/api/sync/replicas` (or extend the existing `/api/sync` + with an `op=replicas` switch) accepts a batch of `(kind, replica_id, fields_jsonb, + deleted_at_ts, manifest_jsonb)` rows from the client and returns the + merged authoritative result. The server validates `kind` against the + allowlist and `fields_jsonb` against the per-kind schema before + invoking `crdt_merge_replica`. + +`src/services/sync/replicaSyncManager.ts`: + +- `pull(kind, since?)` — `SELECT … WHERE user_id = $u AND kind = $k AND + updated_at_ts > $since`. Per-kind cursor (`lastSyncedAtReplicas[kind]` + in `SystemSettings`). +- `push(rows)` — calls the atomic merge endpoint. Batches up to 100 + rows per request (configurable; benchmark in PR 1). +- `subscribeToLocalMutations()` — listens for `setField` / + `removeReplica` on the dirty-set; **5-second debounced push**, with + **immediate flush** on `document.visibilitychange` (tab/app blur) + and `window.online` events. Industry-standard cadence (Notion, + Figma). +- HLC counter persists to **IndexedDB always** (high write rate, low + security). Sync passphrase persists separately: + - Tauri (native): keychain via `tauri-plugin-keyring` (macOS, + Windows, Linux libsecret, iOS/Android keystore). + - Web: ephemeral non-extractable Web Crypto `CryptoKey` held in + memory; dropped on tab close. Never `localStorage` / + `IndexedDB` (XSS risk). + +### Layer 4 — Generic binary transfer (real refactor) + +`TransferManager` and `cloudService` are book-shaped today (Codex finding): + +- `TransferItem.bookHash` / `bookTitle` are required (`transferStore.ts:6`). +- `executeTransfer` calls `appService.uploadBook` / `downloadBook` and + finishes with `updateBook(...)` (`transferManager.ts:202`). +- `uploadFile` (`storage.ts:42`) takes `bookHash`/`fileSize`/`fileName` + and nothing else. +- `cloudService.uploadFileToCloud` smuggles the cloud path through + `File.name` (`cloudService.ts:57`). + +Honest refactor: + +- Generalize `TransferItem` to `{kind: 'book' | 'replica', target: {hash: + string, title: string, recordKind?: string, recordId?: string}}`. Keep + the existing book path working unchanged. +- Add a thin `appService.uploadReplica(kind, id, onProgress)` / + `downloadReplica(kind, id, onProgress)` / + `deleteReplicaFiles(kind, id)`. +- Add `storage.uploadReplicaFile(kind, recordId, filename, file, + onProgress)` to the storage wrapper, with explicit cloud-path support + (no `File.name` smuggling). The book path can migrate later. +- Cloud subdir: a single new constant `CLOUD_REPLICAS_SUBDIR = + '${DATA_SUBDIR}/replicas'`. + +Upload state machine for binary-backed kinds: + +1. Adapter writes local files; CRDT row gets created locally with + `manifest_jsonb = null`. +2. `transferManager.queueReplicaUpload(kind, id)` runs. +3. On each file upload success, the manifest is built incrementally + (in-memory). +4. **Last step** writes `manifest.json` to the cloud and updates the DB + row's `manifest_jsonb`. Only then is the row visible to other devices + as `downloadable`. +5. On partial failure, the row stays `manifest_jsonb = null`; other + devices see it as "exists in metadata but binaries pending". The + transfer queue retries; idempotent. + +Receivers download with: `manifest_jsonb` defines the file list + +expected sizes/hashes. Mismatches abort. + +### Layer 5 — Adapter registry (the open-extension seam, with limits) + +`src/services/sync/replicaRegistry.ts` (shape revised after eng review — +core + capability composition, not 9 flat fields): + +``` +interface ReplicaAdapter { + kind: string; // must be in the server allowlist + schemaVersion: number; // bumps when the kind's field shape changes + pack(replica: T): Record; // → fields object (encrypts sensitive fields) + unpack(fields: Record): T; // ← merged fields → typed replica (decrypts) + computeId(input: ...): Promise; + binary?: BinaryCapability; // present only for binary-backed kinds + lifecycle?: LifecycleHooks; // optional load/save hooks +} + +interface BinaryCapability { + localBaseDir: BaseDir; + enumerateFiles(replica: T): { logical: string; localRelPath: string; byteSize: number }[]; +} + +interface LifecycleHooks { + postDownload?(replica: T, fs: FileSystem): Promise; + validateOnLoad?(replica: T, fs: FileSystem): Promise<{ unavailable?: boolean }>; +} + +const registry = new Map(); +export function registerReplicaAdapter(adapter: ReplicaAdapter): void; + // throws on duplicate kind registration (defensive — guards against + // doubly-imported adapter modules during dev hot-reload) +export function getReplicaAdapter(kind: string): ReplicaAdapter | undefined; +``` + +Self-registering adapters are a client-side dispatch convenience — the +server is the source of truth for what kinds are valid. Clients tolerate +unknown kinds in pull responses (skip + log) rather than crashing, since +older clients may receive newer kinds during a phased rollout. + +The server-side allowlist + per-kind JSON schema lives in code: +`packages/server-schema/replicaSchemas.ts` (or similar) — TypeScript +schemas validated at runtime via Zod or equivalent. Adding a kind = client +adapter PR + server schema PR + DB migration if `CHECK (kind IN …)` +changes (we use a `kinds_allowlist` reference table to avoid migrations +on every add). + +## Per-kind initial allowlist (ship in this order) + +| kind | binary | id source | files | +|---|---|---|---| +| `dictionary` | yes | `partialMD5(primaryFile) + size + filenames` | mdx + mdd[] + css[] (skip `.idx.offsets`/`.syn.offsets`) | +| `font` | yes | `partialMD5(file) + size + filename` | single .ttf/.otf/.woff[2] | +| `texture` | yes | `partialMD5(file) + size + filename` | single image | +| `opds_catalog` | no | `md5(url + username)` | — (URL + name + headers + **password ENCRYPTED**) | +| `dict_provider_pref` | no | provider id | — (one boolean field `enabled`) | +| `dict_provider_position` | no | provider id | — (one position string + actor id) | +| `dict_web_search` | no | existing `WebSearchEntry.id` | — | + +Future kinds require a server PR (schema + allowlist + migration if +needed). + +## User-selectable sync categories + +A new settings panel under **Account → Sync** with per-category toggles. +Default: book/config/note/dictionary/font/texture/opds_catalog all ON; +internal kinds (`dict_provider_pref`, `dict_provider_position`, +`dict_web_search`) follow the parent (dictionary) toggle. + +`SystemSettings.syncCategories: Record` stores the +preferences. The category map itself syncs through the existing +`config`-style settings JSON sync (or, more cleanly, becomes its own kind +`sync_pref` once the primitive is built). Per-category gates apply at the +sync manager: + +- `pull(kind)` no-ops if `syncCategories[kind] === false`. +- `push(rows)` filters out rows whose kind is disabled. +- Disabling a category doesn't delete remote rows — it just stops the + device from sending or receiving them. Re-enabling resumes from the + current `updated_at_ts` cursor. + +UI surfaces existing book/config/note sync toggles too (today they're +implicitly always-on). Per-device override is a v2 concern; v1 is +per-account. + +## Encrypted secrets sync (v1 — per CEO review) + +Sensitive fields across any kind sync encrypted from day 1. Initial +sensitive field: `opds_catalog.password` (and `username` if the user +prefers). Future kinds' secrets (AI API keys, etc.) reuse the same +machinery. + +### Sync passphrase + +- User sets a **sync passphrase** (separate from Readest auth password) + via Settings → Sync → "Set sync passphrase" inline modal. +- **Lazy first prompt:** the passphrase modal first appears when the + user attempts to push or pull an encrypted-field replica (e.g., first + OPDS import with credentials, or first sign-in on Device B that pulls + an encrypted row). Users who never use sensitive features never see + the prompt. +- Passphrase storage on device: + - **Tauri (desktop + mobile):** native keychain via Tauri keyring + plugin (macOS Keychain, Windows Credential Manager, iOS Keychain, + Android Keystore). Survives app restart. + - **Web:** prompt **per session**. Hold passphrase-derived key as a + non-extractable Web Crypto `CryptoKey`; drop on tab close. Never + persist to `localStorage` or `IndexedDB` (XSS exposure). +- Per-account salt: server stores a per-user `salt_v1: bytea` (random, + immutable) in a `replica_keys(user_id, salt, alg, created_at)` table. + Salt itself is useless without the passphrase. Salt rotation is a v2 + concern. + +### Key derivation & encryption + +``` +key = PBKDF2( + passphrase, + salt = server.replica_keys.salt_v1, + iterations = 600_000, // OWASP 2024 for SHA-256 PBKDF2 + hash = 'SHA-256', + length = 256 +) + +ciphertext, iv, tag = AES-GCM(plaintext, key, randomIV) +hashSidecar = SHA-256(plaintext) +``` + +### Encrypted-field envelope + +Stored in `fields_jsonb` alongside plain fields. The adapter's `pack` +decides which fields encrypt: + +``` +{ + url: { v: '...', t: '', s: '' }, // plaintext + username: { v: 'alice', t: '...', s: '...' }, // plaintext + password: { + v: { c: '', + i: '', + s: 'salt_v1', + alg: 'aes-gcm/pbkdf2-600k-sha256', + h: '' }, + t: '', s: '' + } +} +``` + +- **`alg`** lets us bump iterations or change cipher later (alg + registry: `aes-gcm/pbkdf2-600k-sha256` initially). +- **`h`** is the SHA-256 sidecar — client verifies hash after decrypt + to detect post-merge corruption without revealing plaintext to the + server. +- HLC stamps the **envelope**, not the plaintext. The CRDT merge sees + opaque ciphertext. Per-field LWW works on encrypted fields exactly + the same as plaintext fields. + +### Encryption sits inside the adapter + +`pack(replica)` and `unpack(fields)` per adapter own the +encrypt/decrypt boundary. The OPDS adapter calls `encryptField()` for +`password`; the dictionary adapter encrypts nothing. The sync manager, +CRDT merge, and Postgres function are encryption-agnostic. This keeps +the threat model per-adapter. + +### Passphrase rotation + +Changing the passphrase re-encrypts every encrypted field locally (with +the new key) and pushes the new envelopes — but does **not** bump HLC +on those fields (no semantic mutation, just envelope swap). Server runs +the merge as usual; receiving devices re-decrypt. + +If the user changes passphrase on Device A and Device B comes online +later, B's old passphrase fails to decrypt → B is prompted "passphrase +changed on another device — re-enter." + +### Forgot passphrase + +A "Forgot passphrase" CTA in Settings → Sync triggers a confirmation +dialog ("This will permanently delete your synced encrypted fields. You +can re-enter them on each device. Continue?"). On confirm: + +1. Server deletes (or NULLs) every `fields_jsonb` envelope where + `alg LIKE 'aes-gcm/%'` for that user. +2. New per-user salt is generated server-side (`replica_keys.salt_v2`). +3. User re-enters affected secrets per device. + +Plain-text fields untouched. Recoverable. + +### Wrong-passphrase handling + +Decryption failure (auth tag fail or SHA256 sidecar mismatch) raises +`DecryptError` / `IntegrityError`. The replicaSyncManager catches and: + +- Surfaces a one-time toast: "Sync passphrase incorrect" or "A synced + field couldn't be verified. Reverting to local copy." +- Refuses to overwrite local cache with the failed remote value. +- Logs replica id + field name for diagnostics. **Local plaintext copy + is preserved** so the user is never locked out of their own data. + +## Stores + +Each store's mutations route through CRDT helpers +(`replicaSyncManager.setField(kind, id, field, value)` / +`removeReplica(kind, id)`) instead of writing `SystemSettings` directly. +On load: + +1. Hydrate from `SystemSettings` (the local cache). +2. If `syncCategories[kind] !== false`, run + `replicaSyncManager.pull(kind, sinceCursor)`. +3. Apply merged fields back to the cache via the adapter's `unpack`. +4. Run `validateOnLoad` per record (sets `unavailable` if local files + missing). + +Store CRUD methods bump `updatedAt_ts` and enqueue a push. Removals call +`removeReplica` (HLC tombstone). Re-imports use the **explicit +reincarnation path**: importer computes new id, checks the local cache — +if a tombstoned row exists with the same id-input but no +`reincarnation`, write a new `reincarnation` token (random) and +re-create the local record under the new logical entity. This avoids the +"stale offline edit revives a tombstone" foot-gun. + +## Phasing (revised) + +Codex flagged the original phasing as wrong: "PR 1 creates a broad +generic platform before proving one kind." Revised sequence: + +### PR 1 — dictionary-only sync, fixed schema (with encryption infra) + +- `replicas` table + RLS migration with **only `'dictionary'` in the + CHECK constraint allowlist** initially. +- `replica_keys(user_id, salt, alg, created_at)` table for per-user + PBKDF2 salt (encryption infra ships in PR 1 even though dictionary + has no encrypted fields — having the infra ready avoids a data + migration when encrypted-field kinds arrive). +- `crdt.ts` (HLC + merges + tests). +- `crypto.ts` (PBKDF2-600k key derivation, AES-GCM encrypt/decrypt, + envelope helpers, SHA-256 sidecar verification, passphrase storage + abstraction with Tauri keychain + web per-session backends). +- Atomic merge via Postgres function `crdt_merge_replica` with + `SECURITY DEFINER` and explicit `auth.uid() = user_id` guard inside + the function body (RLS alone is insufficient for SECURITY DEFINER). +- A new `/api/sync/replicas` endpoint (kind=dictionary only). Server + enforces `kind` allowlist, JSON size cap (64 KiB / row), field count + cap (64 / row), filename validation, per-user-per-kind row quota, + `schemaVersion` bounds (`minSupported ≤ x ≤ maxKnown`), and HLC + ±60s skew clamp. +- Replicas reuse the **existing book signed-URL upload pattern** for + binary uploads (bypasses CF Workers body limit; supports 1+ GB). +- `dictionaryAdapter.ts` registered. +- `customDictionaryStore` rewires through the new manager. +- Migration: rehash legacy `bundleDir` ids in a staged path with backup + (write new metadata alongside old, validate, then swap; preserve + `providerOrder` mapping). +- TransferManager refactor for `kind: 'replica'` (book path unchanged). +- Settings → Sync panel with the dictionaries toggle (and the + pre-existing book/config/note toggles surfaced). +- UI: `` in `CustomDictionaries.tsx` for "Download from + cloud (X MB)". +- Feature flag `ENABLE_REPLICA_SYNC` (default off in production for the + first 2 weeks; on for staging and dev). +- README at `src/libs/crdt.ts` with HLC + merge worked examples + (knowledge concentration mitigation). + +This PR proves the entire stack against one real kind, with the +encryption infrastructure ready for PR 4 (OPDS catalogs). After it +ships and stabilizes, decide whether to extract generic primitives +(Codex's recommendation: extract only after the second kind validates +the abstraction). + +### PR 2 — extract primitives + add `font` + +Refactor any dictionary-specific bits in `replicaSyncManager` / +`TransferManager` to be kind-agnostic now that we have a real second +example. Add `'font'` to the allowlist + schema + adapter. + +### PR 3 — `texture` + +Similar shape to fonts. + +### PR 4 — `opds_catalog` (with encrypted password) + +Adapter syncs `id`, `name`, `url`, `description`, `icon`, +`customHeaders`, `autoDownload` as plaintext fields, and `password` +(and optionally `username`) as **encrypted-field envelopes** using the +crypto infra shipped in PR 1. Lazy passphrase prompt: first OPDS +catalog import with credentials triggers the "Set sync passphrase" +modal. + +### PR 5 — internal dict-settings kinds + +Migrate `dictionarySettings.providerOrder` (per-position rows with +`(position, actorId, replicaId)` total order) / +`dictionarySettings.providerEnabled` (per-pref rows) / +`dictionarySettings.webSearches`. This lets concurrent rename + reorder +both survive on dictionaries — the original goal — but only after the +single-array model is fully retired. + +### PR 6+ — anything else + +`pref`, `theme`, `shortcut`, `annotation_rule`, future AI keys +(another encrypted-field kind), etc. Each is a coordinated client+server +PR (adapter + schema + allowlist). + +## Migration + +For dictionaries (PR 1): + +1. On first launch after upgrade, for each entry without a content-hash + id: + - Identify primary file by kind. + - Compute `id = partialMD5(primary) + size + filenames`. + - Write a NEW row alongside the old (don't delete the legacy + `bundleDir` until validated). + - Move `providerOrder` / `providerEnabled` / `defaultProviderId` + references from old id → new id. + - Validate: re-load and look up a known word — if it works, soft- + remove the legacy entry (move bundleDir to `.legacy//`, + keep for one release cycle as backup). +2. If bundle missing on disk and `id` is absent — surface "needs + re-import to enable cloud sync" and skip until re-import. +3. Idempotent: a second pass sees `id` already populated and skips. + +For other kinds: similar staged backup pattern in their respective +PRs. + +## Edge cases + +- **Concurrent rename + reorder** on offline devices: only post-PR-5 + (when `providerOrder` is split). Pre-PR-5: the array-shaped order + field is per-replica LWW; the loser's reorder drops. Document. +- **Concurrent rename**: HLC tiebreak. Loser's name drops. Acceptable. +- **Same dict imported on two devices**: same content → same id. Both + push idempotently; binary uploads overwrite same key with identical + bytes. Manifest writes are last-wins identical. +- **Re-import producing different bytes** (user upgraded the dict file): + new id. Old row tombstoned; new row inserts. Manifest of old row stays + pointing at old binaries until cleanup (run a daily server-side + reaper for tombstoned rows older than N days; delete cloud files). +- **Tombstone resurrection**: requires explicit `reincarnation` token. + Stale offline edits do NOT revive deleted records. +- **Mixed-version clients**: old clients ignore unknown kinds. New + clients tolerate unknown kinds in pull. Schema bumps via + `schemaVersion` on the adapter; server validates + `minSupported ≤ schemaVersion ≤ maxKnown` (both bounds, per CEO + review — prevents a malicious client claiming arbitrary version). +- **Bundle 1+ GB**: reuses the existing book signed-URL path (bypasses + CF Workers body limit). Future: streaming + resumable uploads as + separate work. +- **Quota exhaustion mid-upload**: existing TransferManager toast/retry + copy applies. Server-side per-user-per-kind row count limit returns + 402 / 507 on push. +- **Free-form filenames**: server validates (no `..`, no path + separators, length cap, charset). Cloud key escape lives there. +- **Far-future HLC**: server clamps; client repairs by re-deriving from + remote `max(updated_at_ts)`. +- **HLC persistence failure** (IndexedDB / Tauri keychain unavailable): + fall back to `serverMax + 1` per push; accept potential same-ms + collisions; document. +- **Manifest commit failure** (binary uploads succeed but + `manifest_jsonb` write fails): TransferManager retries 3x with + backoff. After max retries, surface "Replica X stuck syncing — retry + from Settings" toast with action button; row stays + `manifest_jsonb = null` until success. +- **SHA-256 sidecar mismatch on encrypted field** (corruption or + tamper): refuse remote, surface one-time toast "A synced field + couldn't be verified. Reverting to local copy.", log replica id + + field. Local plaintext preserved. +- **Wrong sync passphrase** on Device B: decrypt throws `DecryptError` + cleanly, no corrupt result. UI prompts re-entry. +- **Sync passphrase changed on another device**: this device's old + passphrase fails; UI prompts "passphrase changed on another device — + re-enter." +- **Forgot sync passphrase**: confirm-dialog → server wipes encrypted + envelopes (`fields_jsonb` rows where any field has `alg LIKE + 'aes-gcm/%'`); per-user salt rotated; user re-enters affected + secrets per device. Plaintext fields untouched. + +## Error & rescue map (consolidated) + +| Exception class | Rescued | Action | User sees | +|---|---|---|---| +| `TimeoutError` | yes | retry 2x backoff | (transparent) | +| `AuthError` (401) | yes | refresh token; re-auth flow | "Sign in again" | +| `QuotaExceededError` (402/507) | yes | halt push; toast | "Storage full — manage in Settings"| +| `ClockSkewError` (409) | yes | re-derive HLC from server max; retry once | (transparent) | +| `ValidationError` (422) | yes | halt push; log payload loudly | (transparent — devs see logs) | +| `ServerError` (5xx) | yes | retry 3x backoff | "Sync paused — retrying" | +| `DecryptError` | yes | prompt for passphrase | "Sync passphrase incorrect" | +| `IntegrityError` | yes | refuse remote; toast; preserve local | "A synced field couldn't be verified" | +| `UnsupportedAlgError` | yes | skip + log + suggest update | "Update Readest to read this data" | +| `SaltNotFoundError` | yes | re-fetch salt list from server | (transparent) | +| `CryptoUnavailableError` | yes | disable encrypted-field sync | "Browser doesn't support encryption" | +| `NoPassphraseError` | yes | prompt for passphrase | "Set sync passphrase" | +| `LocalFileMissingError` | yes | mark replica unavailable; log | "Replica X needs re-import" | +| `TransferError` | yes | retry per existing pattern | (existing UI) | +| `StorageError` | yes | retry with backoff | "Sync paused" | +| `ManifestCommitError` | yes | retry 3x; then surface "stuck" toast | "Dictionary X stuck syncing — retry" | +| `UnknownKindError` | yes | skip + log | (transparent) | +| `SchemaTooNewError` | yes | skip + log + offer update | "Update Readest" | +| `LegacyMigrationSkipError` | yes | skip; surface "needs re-import" | "Re-import to enable cloud sync" | +| `HlcPersistError` | yes | fall back to `serverMax + 1` | (transparent) | + +## Observability + +PR 1 ships with the full metric set from day 1: + +**Metrics:** +- `replicas_pushed_total{kind, outcome}` — counter (success / error / + rejected_quota / rejected_skew / rejected_validation) +- `replicas_pulled_total{kind}` — counter +- `crdt_merge_duration_seconds{kind}` — histogram (Postgres function + side; emit via pg_stat or RPC return) +- `encryption_failures_total{reason}` — counter (decrypt_error, + integrity_error, no_passphrase, unsupported_alg) +- `manifest_commit_failures_total{kind}` — counter +- `hlc_skew_rejections_total` — counter +- `replica_quota_rejections_total{user_id}` — counter (sampled) + +**Structured logs:** entry/exit per merge, per-error at warn/error +with `replica_id`, `kind`, `field` (no plaintext values), `user_id` +hash. + +**Day-1 alerts:** +- `encryption_failures_total` rate spike > 5x baseline (1h window) → + page (suggests passphrase or salt issue, or attack) +- `hlc_skew_rejections_total` rate spike > 10x baseline → page + (suggests bad client or NTP drift across users) +- `manifest_commit_failures_total` rate spike → page (suggests R2 + outage) + +## Deployment & rollout + +- **Feature flag** `ENABLE_REPLICA_SYNC` (server-side per-user, default + off in production for the first 2 weeks; on for staging and dev). + Rollout: dev → 5% prod → 25% → 50% → 100% over ~3 weeks. +- **Migrations** are additive (new tables, new function, new RLS + policies). Zero-downtime, no table locks. +- **Backwards compat:** old clients (no replica-sync awareness) keep + working; `/api/sync/replicas` 404 to them is fine. New clients + (replica-sync aware) seeing 404 disable replica sync, log, continue + to use existing book/config/note sync. +- **Smoke test post-deploy:** push+pull a synthetic replica end-to-end + per environment. Verify echo, manifest commit, decrypt round-trip + (in the encryption follow-up PR). +- **Rollback posture:** feature flag off cuts client traffic instantly. + Server data persists harmlessly. Schema migrations are forward-only + (CRDT/HLC/encryption can't easily un-merge); document this as + Reversibility = 3/5. + +## Critical files + +New: + +- `src/libs/crdt.ts` — HLC + merge + tests + README. HLC string format: + `${physicalMs.toString(16).padStart(13, '0')}-${counter.toString(16).padStart(8, '0')}-${deviceId}`. + Lexicographic order matches temporal order (invariant test). +- `src/libs/crypto/derive.ts` — PBKDF2-600k (OWASP 2024) key derivation. +- `src/libs/crypto/encrypt.ts` — AES-GCM encrypt/decrypt round-trip. +- `src/libs/crypto/passphrase.ts` — passphrase storage abstraction: + Tauri keychain backend + web per-session ephemeral backend. Set, + change, forget operations. +- `src/libs/crypto/envelope.ts` — envelope shape `{c, i, s, alg, h}`, + encode/decode helpers, SHA-256 sidecar verification. +- `src/libs/errors.ts` — `class SyncError extends Error { code: + SyncErrorCode }` with a `SyncErrorCode` union covering all 19 paths + in the error map. Single `instanceof` check; switch on `.code` for + handling. (Replaces the 19-class hierarchy in the prior plan.) +- `src/services/sync/replicaRegistry.ts` — adapter map. +- `src/services/sync/replicaSyncManager.ts` — pull/push/merge orchestration. +- `src/services/sync/adapters/dictionary.ts` (PR 1), then + `font.ts`, `texture.ts`, `opdsCatalog.ts`, + `dictProviderPref.ts`, `dictProviderPosition.ts`, + `dictWebSearch.ts` (later PRs). +- `src/services/sync/adapters/index.ts` — barrel. +- `src/components/settings/CloudReplicaRow.tsx` — shared cloud-state row. +- `src/components/settings/SyncCategoriesPanel.tsx` — per-category toggles. +- DB migrations: `replicas` table + `replica_keys` table + + `crdt_merge_replica()` Postgres function (with `SECURITY DEFINER` and + explicit `auth.uid() = user_id` guard). +- Server-side schema definitions: `src/server/replicaSchemas.ts` (Zod + schemas; allowlist; per-kind quotas; filename validators; + `schemaVersion` bounds). +- `src/components/settings/SyncPassphrasePanel.tsx` — set/change/forgot + passphrase UX; lazy-prompted on first encrypted-field operation. + +Modified: + +- `src/types/replica.ts` (new) — `ReplicaRow`, `SyncableReplica` base. +- `src/libs/sync.ts` — add `'replicas'` to `SyncType`, extend + `SyncResult`/`SyncData`. +- `src/pages/api/sync.ts` — wire to `crdt_merge_replica` Postgres + function via `INSERT … ON CONFLICT … DO UPDATE` or RPC. +- `src/services/transferManager.ts` — `kind: 'book' | 'replica'` + discriminator; new `queueReplica*` methods. +- `src/store/transferStore.ts` — `kind` on `TransferItem`; book fields + optional. +- `src/services/cloudService.ts` — explicit `cfp` argument support + (drop `File.name` smuggling). +- `src/libs/storage.ts` — `uploadReplicaFile` API alongside `uploadFile`. +- `src/services/appService.ts` — `uploadReplica` / `downloadReplica` / + `deleteReplicaFiles`. +- `src/types/system.ts` — declare new methods. +- `src/services/constants.ts` — `CLOUD_REPLICAS_SUBDIR`. +- `src/services/dictionaries/dictionaryService.ts` — id from new hash + recipe (partial + size + filenames). +- `src/store/customDictionaryStore.ts` — replica-sync integration, + staged migration, reincarnation revival. +- `src/types/settings.ts` — add `syncCategories` field. +- `src/components/settings/CustomDictionaries.tsx` — ``. + +## Verification + +Manual end-to-end: + +1. Sign in on Devices A and B. Take both offline. +2. On A: rename dict X. On B: disable X. Reconnect → both survive (per- + field LWW). +3. On A: rename dict X to "Foo". On B: rename dict X to "Bar". + Reconnect → HLC tiebreak picks one; loser's rename gone (acceptable; + document). +4. On A: delete dict X. On B: re-import same content. Reconnect → A + sees the re-imported entry (reincarnation token; new logical record). +5. Disable "fonts" sync in Settings → Sync. Verify no font records pull + or push from that device. Re-enable → resumes. +6. Lazy-download large dict on a fresh device → manifest verifies file + list + sizes; partial download retries cleanly. +7. OPDS catalog with credentials imported on A → appears on B; B + prompts to set sync passphrase if not set, then auto-decrypts the + password. +8. Far-future HLC injected by a synthetic client → server rejects with + clock-skew error; client repairs. +9. Set sync passphrase on A → push encrypted OPDS catalog → sign in on + B → enter passphrase → password decrypts and OPDS catalog works. +10. Forgot-passphrase flow on A → server wipes encrypted envelopes → + salt rotates → A re-enters credentials → B sees old envelopes + gone, prompts re-enter on its side. +11. Wrong passphrase on B → `DecryptError` toast; local plaintext copy + preserved; user not locked out. +12. Tamper test: corrupt one byte of an encrypted envelope's `c` → + `IntegrityError` toast; remote refused; local preserved. + +Automated: + +- `crdt.ts` unit tests: HLC monotonicity; field-level LWW commutativity, + associativity, idempotence; remove-wins tombstone rules; reincarnation + token logic. +- `crypto.ts` unit tests: + - encrypt → decrypt round-trip = identity. + - SHA-256 sidecar verification catches tampered ciphertext. + - Wrong passphrase raises `DecryptError`, not corrupt result. + - PBKDF2 600k key matches reference vector. + - Cross-platform key-derivation parity (Tauri keychain vs web Crypto + subtle): same passphrase + salt → same key bytes. + - Forgot-passphrase wipe: encrypted envelopes nulled; plaintext + fields preserved. +- Server merge test: two concurrent pushes against the same replica with + different field updates both land (no lost field). +- Adapter contract test: every adapter satisfies `pack ∘ unpack = + identity`; `kind` is in the allowlist; `computeId` is deterministic. +- Schema validation tests: invalid filenames rejected, oversize JSON + rejected, unknown kind rejected, `schemaVersion` out of bounds + rejected. +- Migration test: legacy `uniqueId()` dict → new content-hash id; + `providerOrder` references rewritten; idempotent rerun. +- E2E sync category toggle test: disable/enable per kind. +- Manifest atomicity test: kill upload mid-flight, verify + `manifest_jsonb = null` on the row, verify recovery on next launch + retries cleanly. +- 1+ GB upload regression: replica path must use the same signed-URL + pattern books use; assert request body never goes through a Worker. +- RLS test: `crdt_merge_replica()` with `SECURITY DEFINER` correctly + rejects writes where `auth.uid() != user_id` even when caller has + service role. +- **Push trigger:** debounce of 5s plus immediate flush on + `visibilitychange` and `online` events behaves correctly under rapid + successive mutations. +- **HLC packing format invariant:** for any pair of HLCs `a`, `b` with + `a.physicalMs < b.physicalMs`, lexicographic comparison of packed + strings preserves the order; same for counter ties. +- **Postgres `crdt_merge_replica()` preserves unknown fields:** when + the incoming row carries a field not present locally (or vice versa), + the merged row keeps both. Forwards-compat with future schemaVersion + bumps. +- **`replicaRegistry` double-registration:** calling + `registerReplicaAdapter(adapter)` twice with the same `kind` throws + (defensive guard against doubly-imported modules during dev + hot-reload). +- **Error code → UX mapping:** every `SyncError.code` maps to the + expected user-visible behavior (toast, log only, prompt, retry, etc.) + per the error/rescue map. Drives the integration test for each row of + the table. +- **TransferManager regression:** existing book upload path + (`type: 'upload', kind: 'book'`) behaves identically before and after + the `kind` discriminator is added. No regression in book sync. +- **`assertNever` exhaustiveness:** every `switch` over `SyncType` + ends in `default: assertNever(_)` so a future fifth value is caught + at compile time. +- **Per-kind quota enforcement:** push 1 row beyond the per-user-per- + kind limit returns 402/507; in-flight rows already accepted are not + rolled back. +- **Concurrent same-field tiebreak:** Device A writes + `name='Foo' @t1`, Device B writes `name='Bar' @t2 (t2 > t1)`; both + push; both pull; both converge to 'Bar' (HLC tiebreak). + +## Manifest schema (locked) + +``` +type Manifest = { + files: { + filename: string; // server-validated: no `..`, no `/`, no `\`, + // length ≤ 255, charset restricted + byteSize: number; + partialMd5: string; // 32-hex; matches client-side computation + mtime?: number; // optional; clock-skew tolerant; never trusted + // for ordering, only display + }[]; + schemaVersion: number; // future-proof for manifest format changes +}; +``` + +`manifest_jsonb` in the `replicas` row stores this exact shape. + +## Performance baselines (PR 1 SLOs) + +- Push p95 latency < 500ms for batches ≤ 50 rows on broadband. +- Pull p95 latency < 1s for ≤ 1000 rows. +- `crdt_merge_replica()` p99 < 50ms per row at the 100-row batch cap. +- 1+ GB MDict upload sustained throughput ≥ 80% of network capacity + (signed-URL direct-to-R2 path, no Worker proxy). +- PBKDF2-600k derivation < 2s on iPhone SE-class hardware. Web Worker + offload deferred to v2 if real-device telemetry shows jank. + +## Open decisions + +1. **Atomic merge mechanism** — **DECIDED:** inline UPSERT via + `INSERT … ON CONFLICT … DO UPDATE SET fields_jsonb = crdt_merge_replica(...)`. + One round-trip, less code than explicit RPC. +2. **Per-device sync override**: per-account toggles only (this plan) + vs also per-device override. Recommend per-account in v1; revisit if + users ask for laptop-only sync. +3. **Strong hash**: skip `strongHash` (this plan) vs background full + streaming hash for tamper detection. Recommend skip in v1. +4. **External CRDT library** — **DECIDED:** roll-our-own. Yjs and Loro + impose JSON-incompatible wire formats; field-level LWW is the only + primitive we need. +5. **OPDS encryption phase** — **DECIDED (CEO review):** ship encrypted + sync in **v1**, not v2. Crypto infra (`crypto.ts`, + `replica_keys` table, sync passphrase UX) ships in PR 1; the OPDS + adapter that uses it ships in PR 4. + +## GSTACK REVIEW REPORT + +| Review | Trigger | Why | Runs | Status | Findings | +|--------|---------|-----|------|--------|----------| +| CEO Review | `/plan-ceo-review` | Scope & strategy | 1 | issues_found | Mode SCOPE EXPANSION; 6 expansions proposed, 1 accepted (encrypted secrets v1), 5 skipped, 1 strategic posture lock (private-only forever); 8 architecture/security/test findings absorbed | +| Codex Review | `/codex review` | Independent 2nd opinion | 1 | issues_found | 19 issues raised; revisions absorbed | +| Eng Review | `/plan-eng-review` | Architecture & tests (required) | 1 | issues_found | Mode FULL_REVIEW; complexity-check confirmed cathedral; 16 findings (4 arch, 5 quality, 9 test gaps, 1 perf); adapter contract restructured, 5s debounce push trigger, crypto split into 4 files, single SyncError + code, 9 test additions, manifest schema locked, HLC packing format spec'd, perf SLOs added | +| Design Review | `/plan-design-review` | UI/UX gaps | 0 | — | — | +| DX Review | `/plan-devex-review` | Developer experience gaps | 0 | — | — | + +**CEO REVIEW — accepted scope changes:** + +- **Encrypted secrets in v1** (was deferred to v2). Per-account sync + passphrase, PBKDF2-600k (OWASP 2024), AES-GCM, encrypted-fields + envelope `{c, i, s, alg, h}` with SHA-256 sidecar. Encryption sits in + per-adapter `pack`/`unpack`. Tauri keychain native; web per-session + Web Crypto subtle; never localStorage. Lazy first-prompt on first + encrypted-field push/pull. Forgot-passphrase wipes server-side + envelopes + rotates salt; user re-enters secrets per device. Wrong + passphrase preserves local plaintext, surfaces toast. +- **Replicas reuse signed-URL upload pattern** (matches book path); + bypasses CF Workers body limit; supports 1+ GB. +- **`crdt_merge_replica()` SECURITY DEFINER** must explicitly assert + `auth.uid() = user_id` inside the function body (not just RLS). +- **`schemaVersion` bounds** are two-sided: `minSupported ≤ x ≤ maxKnown`. +- **Day-1 observability** ships with the full metric set (push/pull + totals by outcome, crdt_merge_duration, encryption_failures, + manifest_commit_failures, hlc_skew_rejections); alerts on encryption + and skew spikes. +- **Feature flag** `ENABLE_REPLICA_SYNC` gates v1; staged rollout + dev → 5% → 25% → 50% → 100% over ~3 weeks. +- **Test additions:** crypto round-trip, SHA-256 sidecar, wrong + passphrase, forgot-passphrase wipe, cross-platform key-derivation + parity, manifest atomicity, 1+ GB upload regression, RLS guard test. +- **`crdt.ts` README** — knowledge concentration mitigation. +- **Error & rescue map** consolidated as a first-class section + (previously implicit). + +**CEO REVIEW — strategic posture locks (permanent, not deferrals):** + +- **Private-only forever.** No cross-user content sharing. The + polymorphic `replicas` table is internal infrastructure, not a public + API. Any future cross-user-visibility feature requires a fresh + architecture review. + +**CEO REVIEW — explicitly skipped (not in scope):** + +- First-run "Migrate from another device" wizard (deferred to follow-up). +- Per-device naming + Manage Devices UI. +- Bandwidth-aware sync (Wi-Fi-only, charging-only). +- Sync status indicator (breathing dot). +- Audit log for sensitive ops (deferred to v2). + +**CODEX:** 19 substantive findings, all incorporated: + +- Free-form `kind` → server-managed allowlist with per-kind JSON schemas, + size/field limits, filename validation, per-user quotas. Tenet 1 rewritten. +- `kind` enum → server CHECK constraint and `kinds_allowlist` table. +- API-level fetch-then-upsert merge → atomic SQL UPSERT with `crdt_merge_replica()` + Postgres function, one statement. +- Deletion-revival via "field write > tombstone" → remove-wins tombstones with + explicit reincarnation tokens for re-import. +- Single-position fractional index → per-position rows with `(position, + actorId, replicaId)` deterministic tie-break. +- Trusted client HLC → server-side ±60s skew clamp + client repair. +- HLC in localStorage only → IndexedDB / Tauri keyring with re-derivation + fallback from remote `max(updated_at_ts)`. +- "Add one branch in sync.ts" understatement → honest protocol upgrade + acknowledged; new `/api/sync/replicas` endpoint with per-kind validation. +- TransferManager "discriminator" → real refactor of `TransferItem`, + `cloudService.uploadFileToCloud` (no more `File.name` smuggling), new + `storage.uploadReplicaFile` API. +- `partialMD5` weak identity → `partialMD5 + byteSize + filename mix` in v1; + optional background `strongHash` deferred. +- Manifest atomicity gap → upload state machine with `manifest_jsonb` + populated last; row only `downloadable` after manifest commits. +- "Rehash legacy entries" hand-wavy → staged migration with `.legacy//` + backup directory, reference rewrites, validate-then-swap. +- Unavailable/reimport revival → explicit reincarnation path described. +- Phasing inverted → dictionary-only end-to-end first (PR 1); generic + primitives extracted in PR 2 after a second kind proves the abstraction. +- "One-file change for new kinds" tenet **dropped** — adding a kind is now a + coordinated client+server PR. +- OPDS encryption was deferred to v2 in the Codex revision; the **CEO + review pulled it forward to v1** because shipping plaintext OPDS + metadata + per-device password re-entry was deemed worse UX than + shipping the crypto infra in PR 1. + +**CROSS-MODEL:** Codex and the CEO review converge on "atomic Postgres +merge", "remove-wins tombstones", "server-allowlisted kinds", +"manifest atomicity", "TransferManager refactor scope is real, not +discriminator-trivial". They diverge on encryption phase: Codex +recommended deferring v1 encryption to avoid coupling sync foundations +with crypto UX; CEO review judged the OPDS-plaintext-or-password-prompt-per-device +UX cost too high and pulled encryption into v1. Net: ship encryption +infra in PR 1; OPDS adapter that consumes it ships in PR 4. + +**ENG REVIEW — accepted refinements:** + +- **`ReplicaAdapter` restructured** into core + optional + `BinaryCapability` + optional `LifecycleHooks`. Reduces decision points + per adapter; metadata-only adapters (`opds_catalog`, + `dict_provider_pref`, `dict_web_search`) omit `binary` entirely. +- **Push trigger:** 5s debounce + immediate flush on + `visibilitychange` and `online` events. Notion/Figma-style cadence. +- **Crypto module split** into 4 files (`derive`, `encrypt`, + `passphrase`, `envelope`) under `src/libs/crypto/`. Each ≤ 100 LOC, + single responsibility. +- **Sync subscription extracted** as `useReplicaSubscription(kind, + store)` hook, not inlined into `customDictionaryStore.ts`. Reusable + by future kinds' stores. +- **Single `SyncError` base class** with `code: SyncErrorCode` enum; + replaces the 19-class hierarchy. Single `instanceof` check; switch on + `.code` for handling. +- **HLC always in IndexedDB**, separated from passphrase storage + (Tauri keychain native; web ephemeral non-extractable Web Crypto + key). High-write-rate counter does NOT live in keychain. +- **Forgot-passphrase + offline Device B:** B preserves local plaintext + when it later sees wiped envelopes; offers to push under the new + passphrase after re-prompt. +- **Manifest schema locked:** + `{filename: string, byteSize: number, partialMd5: string, mtime?: + number}[]` plus a `schemaVersion`. Filename server-validated. +- **HLC packing format specified:** + `${physicalMs.toString(16).padStart(13, '0')}-${counter.toString(16).padStart(8, '0')}-${deviceId}`. + Lexicographic = temporal (invariant test). +- **Postgres `crdt_merge_replica()` preserves unknown fields** — + forwards-compat for schemaVersion bumps. +- **Per-kind quotas live in `replicaSchemas.ts`** (Zod) — single source + of truth; server reads at request time. +- **9 additional tests** added to Verification: push trigger, HLC + packing invariant, unknown-field preservation, registry + double-registration, error code → UX mapping, TransferManager book + regression, `assertNever` exhaustiveness, per-kind quota enforcement, + concurrent same-field tiebreak. +- **Performance SLOs documented:** push p95 < 500ms (≤50 rows + broadband); pull p95 < 1s (≤1000 rows); merge fn p99 < 50ms/row at + 100-row batch; 1+GB upload ≥ 80% network capacity. PBKDF2 Web Worker + offload deferred to v2 if real-device telemetry shows jank. + +**UNRESOLVED:** + +- 2 open decisions remain (per-device sync override; strong full-stream + hash). Items 1, 4, 5 resolved via CEO review; pre-existing items 2, 3 + remain. +- `/plan-design-review` not yet run (recommended after PR 1 has visual + surfaces to review). +- `/codex review` re-run on the revised plan would catch any drift + introduced by CEO + eng review changes; recommended but not required. + +**VERDICT:** CEO + ENG CLEARED — plan is implementation-ready. Codex +review reshaped the architecture; CEO review reshaped the scope; eng +review locked implementation details (adapter shape, push trigger, +file layout, error model, test coverage, manifest schema, perf SLOs). +Next step: implement PR 1. diff --git a/apps/readest-app/src/__tests__/libs/crdt.test.ts b/apps/readest-app/src/__tests__/libs/crdt.test.ts new file mode 100644 index 00000000..164b7527 --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/crdt.test.ts @@ -0,0 +1,267 @@ +import { describe, expect, test, vi } from 'vitest'; +import { + HlcGenerator, + hlcCompare, + hlcPack, + hlcParse, + mergeFields, + mergeReplica, + removeReplica, + setField, + withReincarnation, +} from '@/libs/crdt'; +import type { FieldsObject, Hlc, ReplicaRow } from '@/types/replica'; + +const DEV_A = 'dev-a'; +const DEV_B = 'dev-b'; + +const hlc = (ms: number, counter = 0, dev = DEV_A): Hlc => hlcPack(ms, counter, dev); + +const emptyRow = (overrides: Partial = {}): ReplicaRow => ({ + user_id: 'u1', + kind: 'dictionary', + replica_id: 'r1', + fields_jsonb: {}, + manifest_jsonb: null, + deleted_at_ts: null, + reincarnation: null, + updated_at_ts: hlc(0), + schema_version: 1, + ...overrides, +}); + +describe('HLC pack/parse', () => { + test('roundtrips physicalMs + counter + deviceId', () => { + const packed = hlcPack(1700000000000, 7, 'device-xyz'); + const parsed = hlcParse(packed); + expect(parsed.physicalMs).toBe(1700000000000); + expect(parsed.counter).toBe(7); + expect(parsed.deviceId).toBe('device-xyz'); + }); + + test('format is 13-hex-ms - 8-hex-counter - deviceId', () => { + const packed = hlcPack(0, 0, 'd'); + expect(packed).toBe('0000000000000-00000000-d'); + const max = hlcPack(0xfffffffffffff, 0xffffffff, 'd'); + expect(max).toBe('fffffffffffff-ffffffff-d'); + }); + + test('lexicographic order matches temporal order across 1000 random HLCs', () => { + const samples: { ms: number; counter: number; packed: Hlc }[] = []; + for (let i = 0; i < 1000; i++) { + const ms = Math.floor(Math.random() * 0x100000000); + const counter = Math.floor(Math.random() * 0x10000); + samples.push({ ms, counter, packed: hlcPack(ms, counter, DEV_A) }); + } + const lex = [...samples].sort((a, b) => + a.packed < b.packed ? -1 : a.packed > b.packed ? 1 : 0, + ); + const temporal = [...samples].sort((a, b) => a.ms - b.ms || a.counter - b.counter); + expect(lex.map((s) => s.packed)).toEqual(temporal.map((s) => s.packed)); + }); + + test('compare returns -1, 0, 1', () => { + expect(hlcCompare(hlc(1), hlc(2))).toBe(-1); + expect(hlcCompare(hlc(2), hlc(2))).toBe(0); + expect(hlcCompare(hlc(3), hlc(2))).toBe(1); + }); +}); + +describe('HlcGenerator', () => { + test('strictly monotonic across calls in the same ms', () => { + const now = vi.fn(() => 1000); + const gen = new HlcGenerator(DEV_A, now); + const a = gen.next(); + const b = gen.next(); + const c = gen.next(); + expect(hlcCompare(a, b)).toBe(-1); + expect(hlcCompare(b, c)).toBe(-1); + expect(hlcParse(a).counter).toBe(0); + expect(hlcParse(b).counter).toBe(1); + expect(hlcParse(c).counter).toBe(2); + }); + + test('counter resets when physical clock advances', () => { + let t = 1000; + const now = () => t; + const gen = new HlcGenerator(DEV_A, now); + gen.next(); + gen.next(); + expect(hlcParse(gen.next()).counter).toBe(2); + t = 2000; + expect(hlcParse(gen.next()).counter).toBe(0); + }); + + test('absorbs remote HLC: next() > any observed remote', () => { + const t = 1000; + const gen = new HlcGenerator(DEV_A, () => t); + const remote = hlcPack(5000, 0, DEV_B); + gen.observe(remote); + const next = gen.next(); + expect(hlcCompare(remote, next)).toBe(-1); + }); + + test('survives clock regression by holding the higher physical time', () => { + let t = 5000; + const gen = new HlcGenerator(DEV_A, () => t); + const a = gen.next(); + t = 3000; + const b = gen.next(); + expect(hlcCompare(a, b)).toBe(-1); + }); + + test('serialize/restore preserves state', () => { + const t = 1000; + const gen = new HlcGenerator(DEV_A, () => t); + gen.next(); + gen.next(); + const snapshot = gen.serialize(); + const gen2 = HlcGenerator.restore(snapshot, DEV_A, () => t); + expect(hlcParse(gen2.next()).counter).toBe(2); + }); +}); + +describe('setField', () => { + test('writes envelope with v, t, s', () => { + const fields = setField({}, 'name', 'Foo', hlc(100), DEV_A); + expect(fields['name']).toEqual({ v: 'Foo', t: hlc(100), s: DEV_A }); + }); + + test('replaces an existing field with a newer HLC', () => { + const old = setField({}, 'name', 'Old', hlc(100), DEV_A); + const next = setField(old, 'name', 'New', hlc(200), DEV_A); + expect(next['name']).toEqual({ v: 'New', t: hlc(200), s: DEV_A }); + }); + + test('returns a new object (immutable)', () => { + const a: FieldsObject = {}; + const b = setField(a, 'x', 1, hlc(1), DEV_A); + expect(a).not.toBe(b); + expect(a).toEqual({}); + }); +}); + +describe('mergeFields (CRDT properties)', () => { + test('commutativity: merge(a, b) === merge(b, a)', () => { + const a = setField({}, 'name', 'Foo', hlc(100), DEV_A); + const b = setField({}, 'enabled', true, hlc(150), DEV_B); + expect(mergeFields(a, b)).toEqual(mergeFields(b, a)); + }); + + test('associativity: merge(merge(a, b), c) === merge(a, merge(b, c))', () => { + const a = setField({}, 'x', 1, hlc(100), DEV_A); + const b = setField({}, 'y', 2, hlc(150), DEV_B); + const c = setField({}, 'z', 3, hlc(200), DEV_A); + expect(mergeFields(mergeFields(a, b), c)).toEqual(mergeFields(a, mergeFields(b, c))); + }); + + test('idempotence: merge(a, a) === a', () => { + const a = setField({}, 'name', 'Foo', hlc(100), DEV_A); + expect(mergeFields(a, a)).toEqual(a); + }); + + test('preserves fields unique to each side', () => { + const a = setField({}, 'name', 'Foo', hlc(100), DEV_A); + const b = setField({}, 'enabled', true, hlc(150), DEV_B); + const merged = mergeFields(a, b); + expect(merged['name']?.v).toBe('Foo'); + expect(merged['enabled']?.v).toBe(true); + }); + + test('larger HLC wins on same-field collision', () => { + const a = setField({}, 'name', 'Foo', hlc(100), DEV_A); + const b = setField({}, 'name', 'Bar', hlc(200), DEV_B); + expect(mergeFields(a, b)['name']?.v).toBe('Bar'); + expect(mergeFields(b, a)['name']?.v).toBe('Bar'); + }); + + test('ties on HLC: deterministic deviceId tiebreak', () => { + const a = setField({}, 'name', 'Foo', hlcPack(100, 0, 'aaa'), 'aaa'); + const b = setField({}, 'name', 'Bar', hlcPack(100, 0, 'bbb'), 'bbb'); + expect(mergeFields(a, b)).toEqual(mergeFields(b, a)); + }); +}); + +describe('removeReplica + mergeReplica (tombstones)', () => { + test('removeReplica sets deleted_at_ts and bumps updated_at_ts', () => { + const row = emptyRow({ + fields_jsonb: setField({}, 'name', 'Foo', hlc(100), DEV_A), + updated_at_ts: hlc(100), + }); + const tombstoned = removeReplica(row, hlc(200)); + expect(tombstoned.deleted_at_ts).toBe(hlc(200)); + expect(tombstoned.updated_at_ts).toBe(hlc(200)); + }); + + test('field write does NOT revive a tombstoned row (remove-wins)', () => { + const tombstoned = emptyRow({ + deleted_at_ts: hlc(100), + updated_at_ts: hlc(100), + }); + const fieldWrite = emptyRow({ + fields_jsonb: setField({}, 'name', 'Resurrected!', hlc(200), DEV_B), + updated_at_ts: hlc(200), + }); + const merged = mergeReplica(tombstoned, fieldWrite); + expect(merged.deleted_at_ts).toBe(hlc(100)); + expect(merged.fields_jsonb['name']?.v).toBe('Resurrected!'); + }); + + test('reincarnation token swaps the row to alive', () => { + const tombstoned = emptyRow({ + deleted_at_ts: hlc(100), + updated_at_ts: hlc(100), + }); + const reborn = withReincarnation(tombstoned, 'epoch-1'); + expect(reborn.reincarnation).toBe('epoch-1'); + expect(reborn.deleted_at_ts).toBe(null); + }); + + test('mergeReplica updated_at_ts = max(field HLCs, tombstone HLC)', () => { + const a = emptyRow({ + fields_jsonb: setField({}, 'name', 'Foo', hlc(100), DEV_A), + updated_at_ts: hlc(100), + }); + const b = emptyRow({ + fields_jsonb: setField({}, 'enabled', true, hlc(300), DEV_B), + updated_at_ts: hlc(300), + }); + const merged = mergeReplica(a, b); + expect(merged.updated_at_ts).toBe(hlc(300)); + }); + + test('two tombstones: keep the larger HLC', () => { + const a = emptyRow({ deleted_at_ts: hlc(100), updated_at_ts: hlc(100) }); + const b = emptyRow({ deleted_at_ts: hlc(200), updated_at_ts: hlc(200) }); + expect(mergeReplica(a, b).deleted_at_ts).toBe(hlc(200)); + expect(mergeReplica(b, a).deleted_at_ts).toBe(hlc(200)); + }); + + test('mergeReplica is commutative', () => { + const a = emptyRow({ + fields_jsonb: setField({}, 'name', 'Foo', hlc(100), DEV_A), + updated_at_ts: hlc(100), + }); + const b = emptyRow({ + fields_jsonb: setField({}, 'enabled', true, hlc(200), DEV_B), + updated_at_ts: hlc(200), + }); + expect(mergeReplica(a, b)).toEqual(mergeReplica(b, a)); + }); + + test('mergeReplica is idempotent', () => { + const a = emptyRow({ + fields_jsonb: setField({}, 'name', 'Foo', hlc(100), DEV_A), + updated_at_ts: hlc(100), + }); + expect(mergeReplica(a, a)).toEqual(a); + }); +}); + +describe('mergeReplica reincarnation interactions', () => { + test('reincarnation field merges per-field LWW (later epoch wins)', () => { + const a = emptyRow({ reincarnation: 'epoch-1', deleted_at_ts: null, updated_at_ts: hlc(100) }); + const b = emptyRow({ reincarnation: 'epoch-2', deleted_at_ts: null, updated_at_ts: hlc(200) }); + expect(mergeReplica(a, b).reincarnation).toBe('epoch-2'); + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/crypto/derive.test.ts b/apps/readest-app/src/__tests__/libs/crypto/derive.test.ts new file mode 100644 index 00000000..76407437 --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/crypto/derive.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from 'vitest'; +import { derivePbkdf2Key, exportRawKey } from '@/libs/crypto/derive'; + +const enc = new TextEncoder(); +const toHex = (b: Uint8Array): string => + Array.from(b) + .map((x) => x.toString(16).padStart(2, '0')) + .join(''); + +describe('derivePbkdf2Key (PBKDF2-HMAC-SHA-256)', () => { + test('reference vector: password="password", salt="salt", c=1', async () => { + const key = await derivePbkdf2Key('password', enc.encode('salt'), 1); + const raw = await exportRawKey(key); + expect(toHex(raw)).toBe('120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b'); + }); + + test('reference vector: password="password", salt="salt", c=4096', async () => { + const key = await derivePbkdf2Key('password', enc.encode('salt'), 4096); + const raw = await exportRawKey(key); + expect(toHex(raw)).toBe('c5e478d59288c841aa530db6845c4c8d962893a001ce4e11a4963873aa98134a'); + }); + + test('different passphrases produce different keys', async () => { + const salt = enc.encode('salt'); + const a = await derivePbkdf2Key('passwordA', salt, 1000); + const b = await derivePbkdf2Key('passwordB', salt, 1000); + const rawA = toHex(await exportRawKey(a)); + const rawB = toHex(await exportRawKey(b)); + expect(rawA).not.toBe(rawB); + }); + + test('different salts produce different keys', async () => { + const a = await derivePbkdf2Key('password', enc.encode('saltA'), 1000); + const b = await derivePbkdf2Key('password', enc.encode('saltB'), 1000); + const rawA = toHex(await exportRawKey(a)); + const rawB = toHex(await exportRawKey(b)); + expect(rawA).not.toBe(rawB); + }); + + test('different iteration counts produce different keys', async () => { + const salt = enc.encode('salt'); + const a = await derivePbkdf2Key('password', salt, 1000); + const b = await derivePbkdf2Key('password', salt, 2000); + const rawA = toHex(await exportRawKey(a)); + const rawB = toHex(await exportRawKey(b)); + expect(rawA).not.toBe(rawB); + }); + + test('default iteration count is 600_000 (OWASP 2024 for PBKDF2-SHA-256)', async () => { + const key = await derivePbkdf2Key('password', enc.encode('salt')); + const raw = await exportRawKey(key); + expect(raw.length).toBe(32); + }, 30_000); + + test('key is usable with AES-GCM (256-bit)', async () => { + const key = await derivePbkdf2Key('password', enc.encode('salt'), 1000); + const raw = await exportRawKey(key); + expect(raw.length).toBe(32); + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/crypto/encrypt.test.ts b/apps/readest-app/src/__tests__/libs/crypto/encrypt.test.ts new file mode 100644 index 00000000..77113f06 --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/crypto/encrypt.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, test } from 'vitest'; +import { decryptCiphertext, encryptPlaintext } from '@/libs/crypto/encrypt'; +import { derivePbkdf2Key } from '@/libs/crypto/derive'; +import { SyncError } from '@/libs/errors'; + +const enc = new TextEncoder(); +const dec = new TextDecoder(); +const ITER = 1000; + +const makeKey = () => derivePbkdf2Key('test-passphrase', enc.encode('test-salt'), ITER); + +describe('encryptPlaintext / decryptCiphertext', () => { + test('round-trip identity for ASCII string', async () => { + const key = await makeKey(); + const plain = enc.encode('hello world'); + const payload = await encryptPlaintext(plain, key); + const recovered = await decryptCiphertext(payload, key); + expect(dec.decode(recovered)).toBe('hello world'); + }); + + test('round-trip identity for UTF-8 / emoji', async () => { + const key = await makeKey(); + const original = '密码🔐 password — multi-byte'; + const payload = await encryptPlaintext(enc.encode(original), key); + const recovered = await decryptCiphertext(payload, key); + expect(dec.decode(recovered)).toBe(original); + }); + + test('round-trip for empty plaintext', async () => { + const key = await makeKey(); + const payload = await encryptPlaintext(new Uint8Array(0), key); + const recovered = await decryptCiphertext(payload, key); + expect(recovered.length).toBe(0); + }); + + test('round-trip for large plaintext (10 KiB)', async () => { + const key = await makeKey(); + const plain = new Uint8Array(10 * 1024); + for (let i = 0; i < plain.length; i++) plain[i] = i & 0xff; + const payload = await encryptPlaintext(plain, key); + const recovered = await decryptCiphertext(payload, key); + expect(recovered).toEqual(plain); + }); + + test('random IV per encryption: same plaintext produces different ciphertext', async () => { + const key = await makeKey(); + const plain = enc.encode('same plaintext'); + const a = await encryptPlaintext(plain, key); + const b = await encryptPlaintext(plain, key); + expect(a.iv).not.toEqual(b.iv); + expect(a.ciphertext).not.toEqual(b.ciphertext); + }); + + test('IV is 12 bytes (AES-GCM standard)', async () => { + const key = await makeKey(); + const payload = await encryptPlaintext(enc.encode('x'), key); + expect(payload.iv.length).toBe(12); + }); + + test('wrong key throws SyncError DECRYPT', async () => { + const keyA = await makeKey(); + const keyB = await derivePbkdf2Key('different', enc.encode('test-salt'), ITER); + const payload = await encryptPlaintext(enc.encode('secret'), keyA); + await expect(decryptCiphertext(payload, keyB)).rejects.toMatchObject({ + name: 'SyncError', + code: 'DECRYPT', + }); + }); + + test('tampered ciphertext throws SyncError DECRYPT (auth tag fails)', async () => { + const key = await makeKey(); + const payload = await encryptPlaintext(enc.encode('secret'), key); + const tampered = { + iv: payload.iv, + ciphertext: new Uint8Array(payload.ciphertext), + }; + tampered.ciphertext[0] = tampered.ciphertext[0]! ^ 0xff; + await expect(decryptCiphertext(tampered, key)).rejects.toBeInstanceOf(SyncError); + }); + + test('tampered IV throws SyncError DECRYPT', async () => { + const key = await makeKey(); + const payload = await encryptPlaintext(enc.encode('secret'), key); + const tampered = { + iv: new Uint8Array(payload.iv), + ciphertext: payload.ciphertext, + }; + tampered.iv[0] = tampered.iv[0]! ^ 0xff; + await expect(decryptCiphertext(tampered, key)).rejects.toBeInstanceOf(SyncError); + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/crypto/envelope.test.ts b/apps/readest-app/src/__tests__/libs/crypto/envelope.test.ts new file mode 100644 index 00000000..558dec3f --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/crypto/envelope.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, test } from 'vitest'; +import { CURRENT_ALG, decryptFromEnvelope, encryptToEnvelope } from '@/libs/crypto/envelope'; +import { derivePbkdf2Key } from '@/libs/crypto/derive'; +import { isCipherEnvelope } from '@/types/replica'; + +const enc = new TextEncoder(); +const ITER = 1000; + +const makeKey = () => derivePbkdf2Key('p', enc.encode('s'), ITER); + +describe('encryptToEnvelope / decryptFromEnvelope', () => { + test('round-trip identity for string plaintext', async () => { + const key = await makeKey(); + const env = await encryptToEnvelope('hello', key, 'salt-id-1'); + expect(isCipherEnvelope(env)).toBe(true); + const recovered = await decryptFromEnvelope(env, key); + expect(recovered).toBe('hello'); + }); + + test('envelope shape: c, i, s, alg, h all present and base64', async () => { + const key = await makeKey(); + const env = await encryptToEnvelope('x', key, 'salt-v1'); + expect(env.alg).toBe(CURRENT_ALG); + expect(env.s).toBe('salt-v1'); + expect(env.c).toMatch(/^[A-Za-z0-9+/=]+$/); + expect(env.i).toMatch(/^[A-Za-z0-9+/=]+$/); + expect(env.h).toMatch(/^[A-Za-z0-9+/=]+$/); + }); + + test('alg field is exactly aes-gcm/pbkdf2-600k-sha256', async () => { + const key = await makeKey(); + const env = await encryptToEnvelope('x', key, 'salt-v1'); + expect(env.alg).toBe('aes-gcm/pbkdf2-600k-sha256'); + }); + + test('SHA-256 sidecar mismatch (tampered c) raises INTEGRITY', async () => { + const key = await makeKey(); + const env = await encryptToEnvelope('original', key, 'salt-v1'); + const goodPlain = await decryptFromEnvelope(env, key); + expect(goodPlain).toBe('original'); + const otherEnv = await encryptToEnvelope('different', key, 'salt-v1'); + const tampered = { ...env, h: otherEnv.h }; + await expect(decryptFromEnvelope(tampered, key)).rejects.toMatchObject({ + name: 'SyncError', + code: 'INTEGRITY', + }); + }); + + test('tampered ciphertext raises DECRYPT (auth tag fail)', async () => { + const key = await makeKey(); + const env = await encryptToEnvelope('secret', key, 'salt-v1'); + const corrupted = atob(env.c); + const bytes = new Uint8Array(corrupted.length); + for (let i = 0; i < corrupted.length; i++) bytes[i] = corrupted.charCodeAt(i); + bytes[0] = bytes[0]! ^ 0xff; + const tampered = { ...env, c: btoa(String.fromCharCode(...bytes)) }; + await expect(decryptFromEnvelope(tampered, key)).rejects.toMatchObject({ + name: 'SyncError', + code: 'DECRYPT', + }); + }); + + test('unknown alg raises UNSUPPORTED_ALG', async () => { + const key = await makeKey(); + const env = await encryptToEnvelope('x', key, 'salt-v1'); + const futureAlg = { ...env, alg: 'aes-gcm/pbkdf2-2m-sha512' }; + await expect(decryptFromEnvelope(futureAlg, key)).rejects.toMatchObject({ + name: 'SyncError', + code: 'UNSUPPORTED_ALG', + }); + }); + + test('UTF-8 plaintext round-trip', async () => { + const key = await makeKey(); + const original = '密码 password — 🔐'; + const env = await encryptToEnvelope(original, key, 'salt-v1'); + const recovered = await decryptFromEnvelope(env, key); + expect(recovered).toBe(original); + }); + + test('two envelopes of same plaintext have different c/i but same h', async () => { + const key = await makeKey(); + const a = await encryptToEnvelope('same', key, 'salt-v1'); + const b = await encryptToEnvelope('same', key, 'salt-v1'); + expect(a.c).not.toBe(b.c); + expect(a.i).not.toBe(b.i); + expect(a.h).toBe(b.h); + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/crypto/passphrase.test.ts b/apps/readest-app/src/__tests__/libs/crypto/passphrase.test.ts new file mode 100644 index 00000000..17c0cb7b --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/crypto/passphrase.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, test } from 'vitest'; +import { EphemeralPassphraseStore, TauriPassphraseStore } from '@/libs/crypto/passphrase'; + +describe('EphemeralPassphraseStore', () => { + test('set then get returns the same passphrase', async () => { + const store = new EphemeralPassphraseStore(); + await store.set('correct horse battery staple'); + expect(await store.get()).toBe('correct horse battery staple'); + }); + + test('initial state: get returns null', async () => { + const store = new EphemeralPassphraseStore(); + expect(await store.get()).toBe(null); + }); + + test('clear empties the store', async () => { + const store = new EphemeralPassphraseStore(); + await store.set('abc'); + await store.clear(); + expect(await store.get()).toBe(null); + }); + + test('isAvailable returns true (always)', () => { + const store = new EphemeralPassphraseStore(); + expect(store.isAvailable()).toBe(true); + }); + + test('two instances are independent (per-tab semantic)', async () => { + const a = new EphemeralPassphraseStore(); + const b = new EphemeralPassphraseStore(); + await a.set('alpha'); + await b.set('beta'); + expect(await a.get()).toBe('alpha'); + expect(await b.get()).toBe('beta'); + }); + + test('set replaces previous value', async () => { + const store = new EphemeralPassphraseStore(); + await store.set('first'); + await store.set('second'); + expect(await store.get()).toBe('second'); + }); +}); + +describe('TauriPassphraseStore (stub until plugin lands)', () => { + test('stub: set throws NOT_IMPLEMENTED for v1', async () => { + const store = new TauriPassphraseStore(); + await expect(store.set('x')).rejects.toThrow(/Tauri keychain backend not yet wired/); + }); + + test('stub: isAvailable returns false', () => { + const store = new TauriPassphraseStore(); + expect(store.isAvailable()).toBe(false); + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/hlcStore.test.ts b/apps/readest-app/src/__tests__/libs/hlcStore.test.ts new file mode 100644 index 00000000..04d18698 --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/hlcStore.test.ts @@ -0,0 +1,78 @@ +import { afterEach, beforeEach, describe, expect, test } from 'vitest'; +import { InMemoryHlcStore, LocalStorageHlcStore, HLC_LOCAL_STORAGE_KEY } from '@/libs/hlcStore'; + +describe('InMemoryHlcStore', () => { + test('initial load returns null', () => { + const store = new InMemoryHlcStore(); + expect(store.load()).toBe(null); + }); + + test('save then load roundtrips', () => { + const store = new InMemoryHlcStore(); + store.save({ physicalMs: 1700, counter: 7 }); + expect(store.load()).toEqual({ physicalMs: 1700, counter: 7 }); + }); + + test('save replaces previous snapshot', () => { + const store = new InMemoryHlcStore(); + store.save({ physicalMs: 100, counter: 0 }); + store.save({ physicalMs: 200, counter: 5 }); + expect(store.load()).toEqual({ physicalMs: 200, counter: 5 }); + }); +}); + +describe('LocalStorageHlcStore', () => { + beforeEach(() => { + localStorage.clear(); + }); + + afterEach(() => { + localStorage.clear(); + }); + + test('initial load returns null', () => { + expect(new LocalStorageHlcStore().load()).toBe(null); + }); + + test('save persists to localStorage under the canonical key', () => { + new LocalStorageHlcStore().save({ physicalMs: 1700, counter: 3 }); + const raw = localStorage.getItem(HLC_LOCAL_STORAGE_KEY); + expect(raw).not.toBe(null); + expect(JSON.parse(raw!)).toEqual({ physicalMs: 1700, counter: 3 }); + }); + + test('save then load roundtrips', () => { + const store = new LocalStorageHlcStore(); + store.save({ physicalMs: 1700, counter: 3 }); + expect(store.load()).toEqual({ physicalMs: 1700, counter: 3 }); + }); + + test('survives across instances (different store reads same key)', () => { + new LocalStorageHlcStore().save({ physicalMs: 999, counter: 11 }); + expect(new LocalStorageHlcStore().load()).toEqual({ physicalMs: 999, counter: 11 }); + }); + + test('returns null on corrupted JSON instead of throwing', () => { + localStorage.setItem(HLC_LOCAL_STORAGE_KEY, '{not-valid-json'); + expect(new LocalStorageHlcStore().load()).toBe(null); + }); + + test('returns null when localStorage unavailable (server-side)', () => { + const original = globalThis.localStorage; + Object.defineProperty(globalThis, 'localStorage', { + configurable: true, + get() { + throw new Error('localStorage not available'); + }, + }); + try { + expect(new LocalStorageHlcStore().load()).toBe(null); + expect(() => new LocalStorageHlcStore().save({ physicalMs: 1, counter: 0 })).not.toThrow(); + } finally { + Object.defineProperty(globalThis, 'localStorage', { + configurable: true, + value: original, + }); + } + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/replicaSchemas.test.ts b/apps/readest-app/src/__tests__/libs/replicaSchemas.test.ts new file mode 100644 index 00000000..777ea871 --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/replicaSchemas.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, test } from 'vitest'; +import { + KIND_ALLOWLIST, + isAllowedKind, + validateFilename, + validateRow, + MAX_JSON_BYTES, + MAX_FIELD_COUNT, +} from '@/libs/replicaSchemas'; +import type { ReplicaRow, Hlc } from '@/types/replica'; + +const HLC_A = '0000000000064-00000000-dev-a' as Hlc; + +const baseRow = (overrides: Partial = {}): ReplicaRow => ({ + user_id: 'u1', + kind: 'dictionary', + replica_id: 'r1', + fields_jsonb: { + name: { v: 'Webster', t: HLC_A, s: 'dev-a' }, + enabled: { v: true, t: HLC_A, s: 'dev-a' }, + }, + manifest_jsonb: null, + deleted_at_ts: null, + reincarnation: null, + updated_at_ts: HLC_A, + schema_version: 1, + ...overrides, +}); + +describe('isAllowedKind', () => { + test('PR 1 allowlist contains only dictionary', () => { + expect(isAllowedKind('dictionary')).toBe(true); + expect(isAllowedKind('font')).toBe(false); + expect(isAllowedKind('texture')).toBe(false); + expect(isAllowedKind('opds_catalog')).toBe(false); + }); + + test('rejects arbitrary strings', () => { + expect(isAllowedKind('arbitrary')).toBe(false); + expect(isAllowedKind('')).toBe(false); + expect(isAllowedKind('../etc/passwd')).toBe(false); + }); + + test('KIND_ALLOWLIST keys match isAllowedKind', () => { + for (const k of Object.keys(KIND_ALLOWLIST)) { + expect(isAllowedKind(k)).toBe(true); + } + }); +}); + +describe('validateFilename', () => { + test('accepts plain filenames', () => { + expect(validateFilename('webster.mdx').ok).toBe(true); + expect(validateFilename('webster.mdd').ok).toBe(true); + expect(validateFilename('manifest.json').ok).toBe(true); + expect(validateFilename('foo-bar_baz.css').ok).toBe(true); + }); + + test('rejects path traversal', () => { + expect(validateFilename('../etc/passwd').ok).toBe(false); + expect(validateFilename('..').ok).toBe(false); + expect(validateFilename('foo/../bar').ok).toBe(false); + }); + + test('rejects path separators', () => { + expect(validateFilename('foo/bar').ok).toBe(false); + expect(validateFilename('foo\\bar').ok).toBe(false); + }); + + test('rejects empty / too-long names', () => { + expect(validateFilename('').ok).toBe(false); + expect(validateFilename('a'.repeat(256)).ok).toBe(false); + }); + + test('accepts up to 255 chars', () => { + expect(validateFilename('a'.repeat(255)).ok).toBe(true); + }); + + test('rejects null bytes', () => { + expect(validateFilename('foo\u0000bar').ok).toBe(false); + }); + + test('rejects control characters', () => { + expect(validateFilename('foo\u0001bar').ok).toBe(false); + expect(validateFilename('foo\u001fbar').ok).toBe(false); + expect(validateFilename('foo\u007fbar').ok).toBe(false); + }); + + test('accepts unicode (utf-8 dictionary names common in CJK locales)', () => { + expect(validateFilename('韦氏高阶英汉双解词典.mdx').ok).toBe(true); + }); +}); + +describe('validateRow', () => { + test('accepts a valid dictionary row', () => { + const result = validateRow(baseRow()); + expect(result.ok).toBe(true); + }); + + test('rejects unknown kind', () => { + const result = validateRow(baseRow({ kind: 'arbitrary' })); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.code).toBe('UNKNOWN_KIND'); + }); + + test('rejects fields_jsonb beyond MAX_JSON_BYTES (64 KiB)', () => { + const huge = { ...baseRow().fields_jsonb }; + huge['blob'] = { v: 'x'.repeat(MAX_JSON_BYTES), t: HLC_A, s: 'dev-a' }; + const result = validateRow(baseRow({ fields_jsonb: huge })); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.code).toBe('VALIDATION'); + }); + + test('rejects fields_jsonb with too many fields (> 64)', () => { + const fields: Record = {}; + for (let i = 0; i < MAX_FIELD_COUNT + 1; i++) { + fields[`field${i}`] = { v: i, t: HLC_A, s: 'dev-a' }; + } + const result = validateRow(baseRow({ fields_jsonb: fields as never })); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.code).toBe('VALIDATION'); + }); + + test('rejects schemaVersion below minSupported', () => { + const result = validateRow(baseRow({ schema_version: 0 })); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.code).toBe('SCHEMA_TOO_NEW'); + }); + + test('rejects schemaVersion above maxKnown', () => { + const result = validateRow(baseRow({ schema_version: 999 })); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.code).toBe('SCHEMA_TOO_NEW'); + }); + + test('rejects manifest with invalid filename', () => { + const result = validateRow( + baseRow({ + manifest_jsonb: { + schemaVersion: 1, + files: [{ filename: '../etc/passwd', byteSize: 1, partialMd5: 'abc' }], + }, + }), + ); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.code).toBe('VALIDATION'); + }); + + test('accepts manifest with valid filenames', () => { + const result = validateRow( + baseRow({ + manifest_jsonb: { + schemaVersion: 1, + files: [ + { filename: 'webster.mdx', byteSize: 1024, partialMd5: 'abcdef' }, + { filename: 'webster.mdd', byteSize: 2048, partialMd5: 'abcdef' }, + ], + }, + }), + ); + expect(result.ok).toBe(true); + }); + + test('preserves unknown fields (forwards-compat)', () => { + const row = baseRow(); + row.fields_jsonb['future_field'] = { v: 'unknown', t: HLC_A, s: 'dev-a' }; + const result = validateRow(row); + expect(result.ok).toBe(true); + }); + + test('rejects fields with malformed envelope (missing v/t/s)', () => { + const row = baseRow(); + row.fields_jsonb['broken'] = { v: 'x' } as never; + const result = validateRow(row); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.code).toBe('VALIDATION'); + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/replicaSyncClient.test.ts b/apps/readest-app/src/__tests__/libs/replicaSyncClient.test.ts new file mode 100644 index 00000000..2dd94e02 --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/replicaSyncClient.test.ts @@ -0,0 +1,153 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +vi.mock('@/utils/access', () => ({ + getAccessToken: vi.fn(async () => 'fake-token'), +})); +vi.mock('@/services/environment', () => ({ + getAPIBaseUrl: () => 'https://example.test', +})); + +import { ReplicaSyncClient } from '@/libs/replicaSyncClient'; +import { hlcPack } from '@/libs/crdt'; +import type { Hlc, ReplicaRow } from '@/types/replica'; +import { SyncError } from '@/libs/errors'; + +const HLC = hlcPack(1_700_000_000_000, 0, 'd') as Hlc; + +const sampleRow: ReplicaRow = { + user_id: 'u1', + kind: 'dictionary', + replica_id: 'r1', + fields_jsonb: { name: { v: 'Webster', t: HLC, s: 'd' } }, + manifest_jsonb: null, + deleted_at_ts: null, + reincarnation: null, + updated_at_ts: HLC, + schema_version: 1, +}; + +const mockFetch = vi.fn(); + +beforeEach(() => { + mockFetch.mockReset(); + globalThis.fetch = mockFetch as unknown as typeof fetch; +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('ReplicaSyncClient.push', () => { + test('POSTs rows to /sync/replicas with bearer token', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ rows: [sampleRow] }), { status: 200 }), + ); + const client = new ReplicaSyncClient(); + const result = await client.push([sampleRow]); + expect(mockFetch).toHaveBeenCalledOnce(); + const [url, init] = mockFetch.mock.calls[0]!; + expect(url).toBe('https://example.test/sync/replicas'); + expect(init.method).toBe('POST'); + expect(init.headers.Authorization).toBe('Bearer fake-token'); + expect(init.headers['Content-Type']).toBe('application/json'); + expect(JSON.parse(init.body)).toEqual({ rows: [sampleRow] }); + expect(result).toEqual([sampleRow]); + }); + + test('400 / VALIDATION → SyncError VALIDATION', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'bad', code: 'VALIDATION' }), { status: 400 }), + ); + const client = new ReplicaSyncClient(); + await expect(client.push([sampleRow])).rejects.toMatchObject({ + name: 'SyncError', + code: 'VALIDATION', + }); + }); + + test('401 → SyncError AUTH', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'unauth', code: 'AUTH' }), { status: 401 }), + ); + const client = new ReplicaSyncClient(); + await expect(client.push([sampleRow])).rejects.toMatchObject({ + code: 'AUTH', + }); + }); + + test('409 / CLOCK_SKEW → SyncError CLOCK_SKEW', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'skew', code: 'CLOCK_SKEW' }), { status: 409 }), + ); + const client = new ReplicaSyncClient(); + await expect(client.push([sampleRow])).rejects.toMatchObject({ code: 'CLOCK_SKEW' }); + }); + + test('413 / batch too large → SyncError VALIDATION', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'batch', code: 'VALIDATION' }), { status: 413 }), + ); + const client = new ReplicaSyncClient(); + await expect(client.push([sampleRow])).rejects.toMatchObject({ code: 'VALIDATION' }); + }); + + test('422 / UNKNOWN_KIND → SyncError UNKNOWN_KIND', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'unknown', code: 'UNKNOWN_KIND' }), { status: 422 }), + ); + const client = new ReplicaSyncClient(); + await expect(client.push([sampleRow])).rejects.toMatchObject({ code: 'UNKNOWN_KIND' }); + }); + + test('5xx → SyncError SERVER', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ error: 'oops' }), { status: 500 }), + ); + const client = new ReplicaSyncClient(); + await expect(client.push([sampleRow])).rejects.toMatchObject({ code: 'SERVER' }); + }); + + test('network error → SyncError TIMEOUT/SERVER', async () => { + mockFetch.mockRejectedValueOnce(new TypeError('Failed to fetch')); + const client = new ReplicaSyncClient(); + await expect(client.push([sampleRow])).rejects.toBeInstanceOf(SyncError); + }); + + test('empty rows is a no-op (no fetch call)', async () => { + const client = new ReplicaSyncClient(); + const result = await client.push([]); + expect(result).toEqual([]); + expect(mockFetch).not.toHaveBeenCalled(); + }); +}); + +describe('ReplicaSyncClient.pull', () => { + test('GETs with kind + since query params', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ rows: [sampleRow] }), { status: 200 }), + ); + const client = new ReplicaSyncClient(); + const rows = await client.pull('dictionary', HLC); + const [url, init] = mockFetch.mock.calls[0]!; + expect(url).toBe( + `https://example.test/sync/replicas?kind=dictionary&since=${encodeURIComponent(HLC)}`, + ); + expect(init.method).toBe('GET'); + expect(rows).toEqual([sampleRow]); + }); + + test('GET without since cursor', async () => { + mockFetch.mockResolvedValueOnce(new Response(JSON.stringify({ rows: [] }), { status: 200 })); + const client = new ReplicaSyncClient(); + await client.pull('dictionary', null); + const [url] = mockFetch.mock.calls[0]!; + expect(url).toBe('https://example.test/sync/replicas?kind=dictionary'); + }); + + test('404 → empty array (server lacks /api/sync/replicas; old backend)', async () => { + mockFetch.mockResolvedValueOnce(new Response('not found', { status: 404 })); + const client = new ReplicaSyncClient(); + const rows = await client.pull('dictionary', null); + expect(rows).toEqual([]); + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/replicaSyncServer.test.ts b/apps/readest-app/src/__tests__/libs/replicaSyncServer.test.ts new file mode 100644 index 00000000..79e872b4 --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/replicaSyncServer.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, test } from 'vitest'; +import { + HLC_SKEW_TOLERANCE_MS, + MAX_PUSH_BATCH, + clampHlcSkew, + validatePullParams, + validatePushBatch, +} from '@/libs/replicaSyncServer'; +import { hlcPack } from '@/libs/crdt'; +import type { Hlc, ReplicaRow } from '@/types/replica'; + +const USER = 'u1'; +const NOW = 1_700_000_000_000; +const HLC_NOW = hlcPack(NOW, 0, 'dev-a') as Hlc; + +const baseRow = (overrides: Partial = {}): ReplicaRow => ({ + user_id: USER, + kind: 'dictionary', + replica_id: 'r1', + fields_jsonb: { + name: { v: 'Webster', t: HLC_NOW, s: 'dev-a' }, + }, + manifest_jsonb: null, + deleted_at_ts: null, + reincarnation: null, + updated_at_ts: HLC_NOW, + schema_version: 1, + ...overrides, +}); + +describe('clampHlcSkew', () => { + test('accepts HLC within tolerance', () => { + expect(clampHlcSkew(hlcPack(NOW + 1000, 0, 'd') as Hlc, NOW)).toBe(true); + expect(clampHlcSkew(hlcPack(NOW - 1000, 0, 'd') as Hlc, NOW)).toBe(true); + expect(clampHlcSkew(hlcPack(NOW + HLC_SKEW_TOLERANCE_MS, 0, 'd') as Hlc, NOW)).toBe(true); + }); + + test('rejects HLC beyond tolerance', () => { + expect(clampHlcSkew(hlcPack(NOW + HLC_SKEW_TOLERANCE_MS + 1, 0, 'd') as Hlc, NOW)).toBe(false); + expect(clampHlcSkew(hlcPack(NOW - HLC_SKEW_TOLERANCE_MS - 1, 0, 'd') as Hlc, NOW)).toBe(false); + }); + + test('far-future HLC is rejected', () => { + expect(clampHlcSkew(hlcPack(NOW + 1_000_000_000, 0, 'd') as Hlc, NOW)).toBe(false); + }); +}); + +describe('validatePushBatch', () => { + test('accepts an empty batch', () => { + const result = validatePushBatch({ rows: [] }, USER, NOW); + expect(result.ok).toBe(true); + if (result.ok) expect(result.rows).toEqual([]); + }); + + test('accepts a single valid row', () => { + const result = validatePushBatch({ rows: [baseRow()] }, USER, NOW); + expect(result.ok).toBe(true); + }); + + test('rejects body that is not an object', () => { + const result = validatePushBatch(null, USER, NOW); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.status).toBe(400); + expect(result.code).toBe('VALIDATION'); + } + }); + + test('rejects body without rows array', () => { + const result = validatePushBatch({ wrong: 'shape' }, USER, NOW); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.status).toBe(400); + }); + + test('rejects batch above MAX_PUSH_BATCH', () => { + const rows = Array.from({ length: MAX_PUSH_BATCH + 1 }, (_, i) => + baseRow({ replica_id: `r${i}` }), + ); + const result = validatePushBatch({ rows }, USER, NOW); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.status).toBe(413); + expect(result.code).toBe('VALIDATION'); + } + }); + + test('rejects row with mismatched user_id (cross-account write attempt)', () => { + const result = validatePushBatch({ rows: [baseRow({ user_id: 'attacker' })] }, USER, NOW); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.status).toBe(403); + expect(result.code).toBe('AUTH'); + expect(result.offendingIndex).toBe(0); + } + }); + + test('rejects row with kind not in allowlist', () => { + const result = validatePushBatch({ rows: [baseRow({ kind: 'evil' })] }, USER, NOW); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.status).toBe(422); + expect(result.code).toBe('UNKNOWN_KIND'); + } + }); + + test('rejects row with HLC outside skew tolerance', () => { + const farFuture = hlcPack(NOW + 1_000_000, 0, 'd') as Hlc; + const result = validatePushBatch({ rows: [baseRow({ updated_at_ts: farFuture })] }, USER, NOW); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.status).toBe(409); + expect(result.code).toBe('CLOCK_SKEW'); + } + }); + + test('reports the offending index for downstream telemetry', () => { + const rows = [ + baseRow({ replica_id: 'r0' }), + baseRow({ replica_id: 'r1', kind: 'evil' }), + baseRow({ replica_id: 'r2' }), + ]; + const result = validatePushBatch({ rows }, USER, NOW); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.offendingIndex).toBe(1); + }); +}); + +describe('validatePullParams', () => { + test('accepts kind=dictionary with no since', () => { + const result = validatePullParams('dictionary', null); + expect(result.ok).toBe(true); + if (result.ok) expect(result.params.since).toBe(null); + }); + + test('accepts kind=dictionary with a since cursor', () => { + const result = validatePullParams('dictionary', '0000000000064-00000000-dev-a'); + expect(result.ok).toBe(true); + if (result.ok) expect(result.params.since).toBe('0000000000064-00000000-dev-a'); + }); + + test('rejects missing kind', () => { + const result = validatePullParams(null, null); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.status).toBe(400); + }); + + test('rejects unknown kind', () => { + const result = validatePullParams('font', null); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.status).toBe(422); + expect(result.code).toBe('UNKNOWN_KIND'); + } + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/share-server.test.ts b/apps/readest-app/src/__tests__/libs/shareServer.test.ts similarity index 97% rename from apps/readest-app/src/__tests__/libs/share-server.test.ts rename to apps/readest-app/src/__tests__/libs/shareServer.test.ts index 32e38095..f7760895 100644 --- a/apps/readest-app/src/__tests__/libs/share-server.test.ts +++ b/apps/readest-app/src/__tests__/libs/shareServer.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from 'vitest'; -import { generateShareToken, hashShareToken, isValidShareToken } from '@/libs/share-server'; +import { generateShareToken, hashShareToken, isValidShareToken } from '@/libs/shareServer'; -describe('share-server', () => { +describe('shareServer', () => { describe('generateShareToken', () => { it('produces a 22-char alphanumeric raw token', async () => { const { raw, hash } = await generateShareToken(); diff --git a/apps/readest-app/src/__tests__/services/constants.test.ts b/apps/readest-app/src/__tests__/services/constants.test.ts index 462c2c8b..88aa6398 100644 --- a/apps/readest-app/src/__tests__/services/constants.test.ts +++ b/apps/readest-app/src/__tests__/services/constants.test.ts @@ -243,6 +243,20 @@ describe('services/constants', () => { expect(DEFAULT_SYSTEM_SETTINGS.screenBrightness!).toBeLessThanOrEqual(100); }); + it('seeds syncCategories with all four SyncCategory keys', () => { + const cats = DEFAULT_SYSTEM_SETTINGS.syncCategories!; + expect(cats).toEqual({ + book: true, + progress: true, + note: true, + dictionary: true, + }); + }); + + it('seeds lastSyncedAtReplicas as an empty record', () => { + expect(DEFAULT_SYSTEM_SETTINGS.lastSyncedAtReplicas).toEqual({}); + }); + it('has library settings', () => { expect(DEFAULT_SYSTEM_SETTINGS.libraryViewMode).toBe('grid'); expect(typeof DEFAULT_SYSTEM_SETTINGS.librarySortBy).toBe('string'); diff --git a/apps/readest-app/src/__tests__/services/dictionaries/contentId.test.ts b/apps/readest-app/src/__tests__/services/dictionaries/contentId.test.ts new file mode 100644 index 00000000..77549a12 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/dictionaries/contentId.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, test } from 'vitest'; +import { computeDictionaryContentId } from '@/services/dictionaries/contentId'; + +const makeFile = (content: string, name: string): File => + new File([new TextEncoder().encode(content)], name); + +describe('computeDictionaryContentId', () => { + test('returns 32-hex md5', async () => { + const id = await computeDictionaryContentId(makeFile('hello world', 'a.ifo'), ['a.ifo']); + expect(id).toMatch(/^[0-9a-f]{32}$/); + }); + + test('same primary content + same filename list → same id', async () => { + const fileA = makeFile('webster contents', 'webster.mdx'); + const fileB = makeFile('webster contents', 'webster.mdx'); + const a = await computeDictionaryContentId(fileA, ['webster.mdx', 'webster.mdd']); + const b = await computeDictionaryContentId(fileB, ['webster.mdx', 'webster.mdd']); + expect(a).toBe(b); + }); + + test('different primary content → different id', async () => { + const a = await computeDictionaryContentId(makeFile('content A', 'x.mdx'), ['x.mdx']); + const b = await computeDictionaryContentId(makeFile('content B', 'x.mdx'), ['x.mdx']); + expect(a).not.toBe(b); + }); + + test('filename order does NOT change the id (sorted internally)', async () => { + const a = await computeDictionaryContentId(makeFile('x', 'd.mdx'), ['a.mdd', 'd.mdx']); + const b = await computeDictionaryContentId(makeFile('x', 'd.mdx'), ['d.mdx', 'a.mdd']); + expect(a).toBe(b); + }); + + test('different filename set → different id', async () => { + const a = await computeDictionaryContentId(makeFile('x', 'd.mdx'), ['d.mdx']); + const b = await computeDictionaryContentId(makeFile('x', 'd.mdx'), ['d.mdx', 'd.mdd']); + expect(a).not.toBe(b); + }); + + test('different primary file SIZE → different id (defends against partialMD5 collision)', async () => { + // Same first-byte content but different total length. + const a = await computeDictionaryContentId(makeFile('A', 'x.mdx'), ['x.mdx']); + const b = await computeDictionaryContentId(makeFile('AB', 'x.mdx'), ['x.mdx']); + expect(a).not.toBe(b); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/adapters/dictionary.test.ts b/apps/readest-app/src/__tests__/services/sync/adapters/dictionary.test.ts new file mode 100644 index 00000000..9ac8f0fa --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/adapters/dictionary.test.ts @@ -0,0 +1,205 @@ +import { describe, expect, test } from 'vitest'; +import { + computeDictionaryReplicaId, + dictionaryAdapter, + enumerateDictionaryFiles, + primaryDictionaryFile, +} from '@/services/sync/adapters/dictionary'; +import type { ImportedDictionary } from '@/services/dictionaries/types'; + +const baseDict = (overrides: Partial = {}): ImportedDictionary => ({ + id: 'placeholder', + kind: 'mdict', + name: 'Webster', + bundleDir: 'webster-bundle', + files: { mdx: 'webster.mdx', mdd: ['webster.mdd'] }, + addedAt: 1700000000000, + ...overrides, +}); + +describe('dictionaryAdapter contract', () => { + test('kind is "dictionary"', () => { + expect(dictionaryAdapter.kind).toBe('dictionary'); + }); + + test('schemaVersion is 1', () => { + expect(dictionaryAdapter.schemaVersion).toBe(1); + }); + + test('binary capability uses BaseDir "Dictionaries"', () => { + expect(dictionaryAdapter.binary?.localBaseDir).toBe('Dictionaries'); + }); + + test('computeId returns the record id (sync passthrough)', async () => { + const d = baseDict({ id: 'content-hash-xyz' }); + expect(await dictionaryAdapter.computeId(d)).toBe('content-hash-xyz'); + }); +}); + +describe('pack ∘ unpack = identity for the synced subset', () => { + test('mdict bundle: synced fields round-trip', () => { + const d = baseDict({ + kind: 'mdict', + name: 'Webster', + lang: 'en', + addedAt: 1700000000000, + }); + const packed = dictionaryAdapter.pack(d); + const unpacked = dictionaryAdapter.unpack(packed); + expect(unpacked.name).toBe('Webster'); + expect(unpacked.lang).toBe('en'); + expect(unpacked.kind).toBe('mdict'); + expect(unpacked.addedAt).toBe(1700000000000); + }); + + test('per-device fields (bundleDir, files) are NOT in the synced fields object', () => { + const d = baseDict({ + bundleDir: 'device-local-uniqueId-123', + files: { mdx: 'webster.mdx' }, + }); + const packed = dictionaryAdapter.pack(d); + expect(packed['bundleDir']).toBeUndefined(); + expect(packed['files']).toBeUndefined(); + }); + + test('unavailable / deletedAt are NOT synced as fields (tombstones handle delete)', () => { + const d = baseDict({ unavailable: true, deletedAt: 999 }); + const packed = dictionaryAdapter.pack(d); + expect(packed['unavailable']).toBeUndefined(); + expect(packed['deletedAt']).toBeUndefined(); + }); + + test('unsupportedReason round-trips', () => { + const d = baseDict({ unsupported: true, unsupportedReason: 'encrypted MDX' }); + const packed = dictionaryAdapter.pack(d); + const unpacked = dictionaryAdapter.unpack(packed); + expect(unpacked.unsupported).toBe(true); + expect(unpacked.unsupportedReason).toBe('encrypted MDX'); + }); +}); + +describe('primaryDictionaryFile', () => { + test('mdict → .mdx', () => { + expect(primaryDictionaryFile(baseDict({ kind: 'mdict', files: { mdx: 'w.mdx' } }))).toBe( + 'w.mdx', + ); + }); + + test('stardict → .ifo', () => { + expect( + primaryDictionaryFile( + baseDict({ kind: 'stardict', files: { ifo: 'w.ifo', dict: 'w.dict.dz' } }), + ), + ).toBe('w.ifo'); + }); + + test('dict → .dict (or .dict.dz)', () => { + expect( + primaryDictionaryFile( + baseDict({ kind: 'dict', files: { dict: 'w.dict.dz', index: 'w.index' } }), + ), + ).toBe('w.dict.dz'); + }); + + test('slob → .slob', () => { + expect(primaryDictionaryFile(baseDict({ kind: 'slob', files: { slob: 'w.slob' } }))).toBe( + 'w.slob', + ); + }); + + test('returns null when no primary file is recorded', () => { + expect(primaryDictionaryFile(baseDict({ kind: 'mdict', files: {} }))).toBe(null); + }); +}); + +describe('enumerateDictionaryFiles', () => { + test('mdict bundle: mdx + mdd[] + css[]', () => { + const d = baseDict({ + kind: 'mdict', + bundleDir: 'b1', + files: { + mdx: 'webster.mdx', + mdd: ['webster.mdd', 'webster.1.mdd'], + css: ['webster.css'], + }, + }); + const files = enumerateDictionaryFiles(d); + expect(files.map((f) => f.logical)).toEqual([ + 'webster.mdx', + 'webster.mdd', + 'webster.1.mdd', + 'webster.css', + ]); + expect(files.every((f) => f.lfp.startsWith('b1/'))).toBe(true); + }); + + test('stardict bundle: ifo + idx + dict + syn (skips offset sidecars)', () => { + const d = baseDict({ + kind: 'stardict', + bundleDir: 's1', + files: { + ifo: 'd.ifo', + idx: 'd.idx', + dict: 'd.dict.dz', + syn: 'd.syn', + idxOffsets: 'd.idx.offsets', + synOffsets: 'd.syn.offsets', + }, + }); + const files = enumerateDictionaryFiles(d); + expect(files.map((f) => f.logical).sort()).toEqual(['d.dict.dz', 'd.idx', 'd.ifo', 'd.syn']); + }); + + test('dict bundle: dict + index', () => { + const d = baseDict({ + kind: 'dict', + bundleDir: 'dd', + files: { dict: 'w.dict.dz', index: 'w.index' }, + }); + const files = enumerateDictionaryFiles(d); + expect(files.map((f) => f.logical).sort()).toEqual(['w.dict.dz', 'w.index']); + }); + + test('slob bundle: single .slob file', () => { + const d = baseDict({ kind: 'slob', bundleDir: 'sl', files: { slob: 'w.slob' } }); + const files = enumerateDictionaryFiles(d); + expect(files.map((f) => f.logical)).toEqual(['w.slob']); + }); + + test('absent files are skipped', () => { + const d = baseDict({ kind: 'mdict', bundleDir: 'b', files: { mdx: 'm.mdx' } }); + const files = enumerateDictionaryFiles(d); + expect(files.map((f) => f.logical)).toEqual(['m.mdx']); + }); +}); + +describe('computeDictionaryReplicaId', () => { + test('deterministic over (partialMd5, byteSize, sorted filenames)', () => { + const a = computeDictionaryReplicaId('abc123', 1024, ['x.mdx', 'x.mdd']); + const b = computeDictionaryReplicaId('abc123', 1024, ['x.mdd', 'x.mdx']); + expect(a).toBe(b); + }); + + test('different partialMd5 → different id', () => { + const a = computeDictionaryReplicaId('abc', 1024, ['x.mdx']); + const b = computeDictionaryReplicaId('def', 1024, ['x.mdx']); + expect(a).not.toBe(b); + }); + + test('different byteSize → different id', () => { + const a = computeDictionaryReplicaId('abc', 1024, ['x.mdx']); + const b = computeDictionaryReplicaId('abc', 2048, ['x.mdx']); + expect(a).not.toBe(b); + }); + + test('different filename set → different id', () => { + const a = computeDictionaryReplicaId('abc', 1024, ['x.mdx']); + const b = computeDictionaryReplicaId('abc', 1024, ['x.mdx', 'x.mdd']); + expect(a).not.toBe(b); + }); + + test('returns 32-hex md5', () => { + const id = computeDictionaryReplicaId('abc', 1024, ['x.mdx']); + expect(id).toMatch(/^[0-9a-f]{32}$/); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/replicaBootstrap.test.ts b/apps/readest-app/src/__tests__/services/sync/replicaBootstrap.test.ts new file mode 100644 index 00000000..63cd9bd8 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/replicaBootstrap.test.ts @@ -0,0 +1,35 @@ +import { afterEach, describe, expect, test } from 'vitest'; +import { + __resetBootstrapForTests, + bootstrapReplicaAdapters, +} from '@/services/sync/replicaBootstrap'; +import { + clearReplicaAdapters, + getReplicaAdapter, + listReplicaAdapters, +} from '@/services/sync/replicaRegistry'; +import { dictionaryAdapter } from '@/services/sync/adapters/dictionary'; + +afterEach(() => { + clearReplicaAdapters(); + __resetBootstrapForTests(); +}); + +describe('bootstrapReplicaAdapters', () => { + test('registers the dictionary adapter', () => { + bootstrapReplicaAdapters(); + expect(getReplicaAdapter('dictionary')).toBe(dictionaryAdapter); + }); + + test('is idempotent: calling twice is a no-op (does not throw)', () => { + bootstrapReplicaAdapters(); + bootstrapReplicaAdapters(); + expect(listReplicaAdapters()).toHaveLength(1); + }); + + test('only registers the kinds in the PR-1 allowlist', () => { + bootstrapReplicaAdapters(); + const kinds = listReplicaAdapters().map((a) => a.kind); + expect(kinds).toEqual(['dictionary']); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/replicaRegistry.test.ts b/apps/readest-app/src/__tests__/services/sync/replicaRegistry.test.ts new file mode 100644 index 00000000..b9fb9b86 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/replicaRegistry.test.ts @@ -0,0 +1,62 @@ +import { afterEach, describe, expect, test } from 'vitest'; +import { + clearReplicaAdapters, + getReplicaAdapter, + listReplicaAdapters, + registerReplicaAdapter, +} from '@/services/sync/replicaRegistry'; +import type { ReplicaAdapter } from '@/services/sync/replicaRegistry'; + +interface DictRecord { + id: string; + name: string; + enabled: boolean; +} + +const dictionaryAdapter: ReplicaAdapter = { + kind: 'dictionary', + schemaVersion: 1, + pack: (r) => ({ id: r.id, name: r.name, enabled: r.enabled }), + unpack: (f) => ({ id: String(f['id']), name: String(f['name']), enabled: Boolean(f['enabled']) }), + computeId: async (r: DictRecord) => r.id, +}; + +afterEach(() => clearReplicaAdapters()); + +describe('replicaRegistry', () => { + test('register + get round-trip', () => { + registerReplicaAdapter(dictionaryAdapter); + const got = getReplicaAdapter('dictionary'); + expect(got).toBe(dictionaryAdapter); + }); + + test('unknown kind returns undefined', () => { + expect(getReplicaAdapter('not-a-kind')).toBeUndefined(); + }); + + test('double registration throws (defensive against doubly-imported modules)', () => { + registerReplicaAdapter(dictionaryAdapter); + expect(() => registerReplicaAdapter(dictionaryAdapter)).toThrow(/already registered/i); + }); + + test('listReplicaAdapters returns all registered kinds', () => { + registerReplicaAdapter(dictionaryAdapter); + expect(listReplicaAdapters().map((a) => a.kind)).toEqual(['dictionary']); + }); + + test('pack ∘ unpack = identity for sample adapter', () => { + registerReplicaAdapter(dictionaryAdapter); + const adapter = getReplicaAdapter('dictionary')!; + const original: DictRecord = { id: 'd1', name: 'Webster', enabled: true }; + const packed = adapter.pack(original); + const unpacked = adapter.unpack(packed); + expect(unpacked).toEqual(original); + }); + + test('clearReplicaAdapters empties the registry (test-only)', () => { + registerReplicaAdapter(dictionaryAdapter); + clearReplicaAdapters(); + expect(getReplicaAdapter('dictionary')).toBeUndefined(); + expect(listReplicaAdapters()).toEqual([]); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/replicaSync.test.ts b/apps/readest-app/src/__tests__/services/sync/replicaSync.test.ts new file mode 100644 index 00000000..f247f452 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/replicaSync.test.ts @@ -0,0 +1,125 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { + __resetReplicaSyncForTests, + getReplicaSync, + initReplicaSync, + isReplicaSyncReady, +} from '@/services/sync/replicaSync'; +import { InMemoryHlcStore } from '@/libs/hlcStore'; +import type { ReplicaRow, Hlc } from '@/types/replica'; +import type { CursorStore } from '@/services/sync/replicaSyncManager'; + +const makeMemoryCursorStore = (): CursorStore => { + const map = new Map(); + return { + get: (k) => map.get(k) ?? null, + set: (k, v) => { + map.set(k, v); + }, + }; +}; + +const makeFakeClient = () => ({ + push: vi.fn(async (rows: ReplicaRow[]) => rows), + pull: vi.fn(async (_kind: string, _since: Hlc | null) => [] as ReplicaRow[]), +}); + +afterEach(() => { + __resetReplicaSyncForTests(); +}); + +describe('replicaSync singleton', () => { + test('getReplicaSync returns null before init', () => { + expect(getReplicaSync()).toBe(null); + expect(isReplicaSyncReady()).toBe(false); + }); + + test('initReplicaSync produces a manager + hlc context', () => { + const ctx = initReplicaSync({ + deviceId: 'dev-a', + cursorStore: makeMemoryCursorStore(), + hlcStore: new InMemoryHlcStore(), + client: makeFakeClient() as never, + }); + expect(ctx.manager).toBeDefined(); + expect(ctx.hlc).toBeDefined(); + expect(ctx.deviceId).toBe('dev-a'); + expect(isReplicaSyncReady()).toBe(true); + }); + + test('subsequent initReplicaSync calls are idempotent (return same instance)', () => { + const a = initReplicaSync({ + deviceId: 'dev-a', + cursorStore: makeMemoryCursorStore(), + hlcStore: new InMemoryHlcStore(), + client: makeFakeClient() as never, + }); + const b = initReplicaSync({ + deviceId: 'dev-different', + cursorStore: makeMemoryCursorStore(), + hlcStore: new InMemoryHlcStore(), + client: makeFakeClient() as never, + }); + expect(b).toBe(a); + expect(b.deviceId).toBe('dev-a'); + }); + + test('hlc.next() persists snapshot to the injected store', () => { + const hlcStore = new InMemoryHlcStore(); + const ctx = initReplicaSync({ + deviceId: 'dev-a', + cursorStore: makeMemoryCursorStore(), + hlcStore, + client: makeFakeClient() as never, + }); + expect(hlcStore.load()).toBe(null); + ctx.hlc.next(); + const snap = hlcStore.load(); + expect(snap).not.toBe(null); + expect(snap!.counter).toBeGreaterThanOrEqual(0); + }); + + test('hlc.observe() persists snapshot to the injected store', () => { + const hlcStore = new InMemoryHlcStore(); + const ctx = initReplicaSync({ + deviceId: 'dev-a', + cursorStore: makeMemoryCursorStore(), + hlcStore, + client: makeFakeClient() as never, + }); + ctx.hlc.observe('fffffffffffff-00000000-dev-other' as Hlc); + expect(hlcStore.load()).not.toBe(null); + }); + + test('init restores from existing snapshot (counter survives reload)', () => { + const hlcStore = new InMemoryHlcStore(); + // Choose a far-future ms so the wall clock won't advance past it during + // the test — that way next() bumps the counter rather than the ms. + const farFutureMs = Date.now() + 10 * 60 * 60 * 1000; + hlcStore.save({ physicalMs: farFutureMs, counter: 42 }); + const ctx = initReplicaSync({ + deviceId: 'dev-a', + cursorStore: makeMemoryCursorStore(), + hlcStore, + client: makeFakeClient() as never, + }); + const serialized = ctx.hlc.serialize(); + expect(serialized.physicalMs).toBe(farFutureMs); + expect(serialized.counter).toBe(42); + ctx.hlc.next(); + expect(ctx.hlc.serialize().counter).toBe(43); + }); + + test('__resetReplicaSyncForTests clears the singleton', () => { + initReplicaSync({ + deviceId: 'dev-a', + cursorStore: makeMemoryCursorStore(), + hlcStore: new InMemoryHlcStore(), + client: makeFakeClient() as never, + }); + expect(isReplicaSyncReady()).toBe(true); + __resetReplicaSyncForTests(); + expect(isReplicaSyncReady()).toBe(false); + expect(getReplicaSync()).toBe(null); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/replicaSyncManager.test.ts b/apps/readest-app/src/__tests__/services/sync/replicaSyncManager.test.ts new file mode 100644 index 00000000..10d854a7 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/replicaSyncManager.test.ts @@ -0,0 +1,183 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { HlcGenerator, hlcPack } from '@/libs/crdt'; +import { SyncError } from '@/libs/errors'; +import { ReplicaSyncManager } from '@/services/sync/replicaSyncManager'; +import type { Hlc, ReplicaRow } from '@/types/replica'; + +const NOW = 1_700_000_000_000; +const DEV = 'dev-a'; +const HLC_NOW = hlcPack(NOW, 0, DEV) as Hlc; + +const makeRow = (id: string, hlcStr: Hlc = HLC_NOW): ReplicaRow => ({ + user_id: 'u1', + kind: 'dictionary', + replica_id: id, + fields_jsonb: { name: { v: id, t: hlcStr, s: DEV } }, + manifest_jsonb: null, + deleted_at_ts: null, + reincarnation: null, + updated_at_ts: hlcStr, + schema_version: 1, +}); + +const makeFakeClient = () => ({ + push: vi.fn(async (rows: ReplicaRow[]) => rows), + pull: vi.fn(async (_kind: string, _since: Hlc | null) => [] as ReplicaRow[]), +}); + +const makeManager = (clientOverrides: Partial> = {}) => { + const client = { ...makeFakeClient(), ...clientOverrides }; + const hlc = new HlcGenerator(DEV, () => NOW); + const cursors = new Map(); + const manager = new ReplicaSyncManager({ + hlc, + client, + debounceMs: 5000, + cursorStore: { + get: (k) => cursors.get(k) ?? null, + set: (k, v) => { + cursors.set(k, v); + }, + }, + }); + return { manager, client, hlc, cursors }; +}; + +beforeEach(() => { + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +describe('ReplicaSyncManager.markDirty + flush', () => { + test('markDirty alone does not push', async () => { + const { manager, client } = makeManager(); + manager.markDirty(makeRow('r1')); + await Promise.resolve(); + expect(client.push).not.toHaveBeenCalled(); + }); + + test('markDirty then 5s debounce fires push', async () => { + const { manager, client } = makeManager(); + manager.markDirty(makeRow('r1')); + await vi.advanceTimersByTimeAsync(4999); + expect(client.push).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(2); + expect(client.push).toHaveBeenCalledOnce(); + expect(client.push.mock.calls[0]![0]).toHaveLength(1); + }); + + test('successive markDirty resets the debounce window', async () => { + const { manager, client } = makeManager(); + manager.markDirty(makeRow('r1')); + await vi.advanceTimersByTimeAsync(4000); + manager.markDirty(makeRow('r2')); + await vi.advanceTimersByTimeAsync(4000); + expect(client.push).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1100); + expect(client.push).toHaveBeenCalledOnce(); + expect(client.push.mock.calls[0]![0]).toHaveLength(2); + }); + + test('flush() pushes immediately', async () => { + const { manager, client } = makeManager(); + manager.markDirty(makeRow('r1')); + manager.markDirty(makeRow('r2')); + await manager.flush(); + expect(client.push).toHaveBeenCalledOnce(); + expect(client.push.mock.calls[0]![0]).toHaveLength(2); + }); + + test('flush() with no dirty rows is a no-op', async () => { + const { manager, client } = makeManager(); + await manager.flush(); + expect(client.push).not.toHaveBeenCalled(); + }); + + test('same replica re-marked dirty: only the latest row pushes', async () => { + const { manager, client } = makeManager(); + manager.markDirty(makeRow('r1', hlcPack(NOW, 0, DEV) as Hlc)); + manager.markDirty(makeRow('r1', hlcPack(NOW, 1, DEV) as Hlc)); + await manager.flush(); + const pushed = client.push.mock.calls[0]![0]; + expect(pushed).toHaveLength(1); + expect(pushed[0]!.fields_jsonb['name']!.t).toBe(hlcPack(NOW, 1, DEV)); + }); + + test('flush() clears the dirty set on success', async () => { + const { manager, client } = makeManager(); + manager.markDirty(makeRow('r1')); + await manager.flush(); + expect(client.push).toHaveBeenCalledOnce(); + await manager.flush(); + expect(client.push).toHaveBeenCalledOnce(); + }); + + test('push rejection: dirty set is preserved for retry', async () => { + const client = { + ...makeFakeClient(), + push: vi.fn(async (_rows: ReplicaRow[]): Promise => { + throw new SyncError('SERVER', 'simulated outage'); + }), + }; + const { manager } = makeManager(client); + manager.markDirty(makeRow('r1')); + await expect(manager.flush()).rejects.toThrow(/simulated outage/); + client.push.mockResolvedValueOnce([makeRow('r1')]); + await manager.flush(); + expect(client.push).toHaveBeenCalledTimes(2); + }); +}); + +describe('ReplicaSyncManager.pull', () => { + test('passes cursor + advances on success', async () => { + const r1 = makeRow('r1', hlcPack(NOW + 100, 0, DEV) as Hlc); + const r2 = makeRow('r2', hlcPack(NOW + 200, 0, DEV) as Hlc); + const client = { + ...makeFakeClient(), + pull: vi.fn(async () => [r1, r2]), + }; + const { manager, cursors } = makeManager(client); + const result = await manager.pull('dictionary'); + expect(result).toEqual([r1, r2]); + expect(client.pull).toHaveBeenCalledWith('dictionary', null); + expect(cursors.get('dictionary')).toBe(r2.updated_at_ts); + }); + + test('subsequent pull uses advanced cursor', async () => { + const r1 = makeRow('r1', hlcPack(NOW + 100, 0, DEV) as Hlc); + const client = { + ...makeFakeClient(), + pull: vi.fn().mockResolvedValueOnce([r1]).mockResolvedValueOnce([]), + }; + const { manager } = makeManager(client); + await manager.pull('dictionary'); + await manager.pull('dictionary'); + expect(client.pull).toHaveBeenNthCalledWith(2, 'dictionary', r1.updated_at_ts); + }); + + test('pull observes remote HLCs into local generator', async () => { + const remoteHlc = hlcPack(NOW + 60_000, 7, 'dev-other') as Hlc; + const client = { + ...makeFakeClient(), + pull: vi.fn(async () => [makeRow('r1', remoteHlc)]), + }; + const { manager, hlc } = makeManager(client); + await manager.pull('dictionary'); + const next = hlc.next(); + expect(next > remoteHlc).toBe(true); + }); + + test('empty pull does not advance cursor', async () => { + const client = { + ...makeFakeClient(), + pull: vi.fn(async () => []), + }; + const { manager, cursors } = makeManager(client); + await manager.pull('dictionary'); + expect(cursors.get('dictionary')).toBeUndefined(); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/transfer-manager.test.ts b/apps/readest-app/src/__tests__/services/transfer-manager.test.ts index c7fb920d..df09af48 100644 --- a/apps/readest-app/src/__tests__/services/transfer-manager.test.ts +++ b/apps/readest-app/src/__tests__/services/transfer-manager.test.ts @@ -32,6 +32,7 @@ function makeBook(overrides: Partial = {}): Book { function makeTransferItem(overrides: Partial = {}): TransferItem { return { id: 't1', + kind: 'book', bookHash: 'hash1', bookTitle: 'Test Book', type: 'upload', @@ -671,4 +672,120 @@ describe('TransferManager', () => { ).resolves.not.toThrow(); }); }); + + // ── queueReplicaUpload / queueReplicaDownload / queueReplicaDelete ── + describe('replica transfer queueing', () => { + const dictFiles = [ + { logical: 'webster.mdx', lfp: 'd1/webster.mdx', byteSize: 1000 }, + { logical: 'webster.mdd', lfp: 'd1/webster.mdd', byteSize: 4000 }, + ]; + + test('queueReplicaUpload returns null when not initialized', () => { + const id = transferManager.queueReplicaUpload( + 'dictionary', + 'd1', + 'Webster', + dictFiles, + 'Dictionaries', + ); + expect(id).toBeNull(); + }); + + test('queueReplicaUpload creates a kind="replica" transfer with files + base', async () => { + const appService = makeAppService(); + appService['uploadReplicaFile'] = vi.fn().mockResolvedValue(undefined); + await transferManager.initialize(appService as never, () => [], vi.fn(), translationFn); + + const id = transferManager.queueReplicaUpload( + 'dictionary', + 'd1', + 'Webster', + dictFiles, + 'Dictionaries', + ); + expect(id).toBeTruthy(); + const t = useTransferStore.getState().transfers[id!]!; + expect(t.kind).toBe('replica'); + expect(t.replicaKind).toBe('dictionary'); + expect(t.replicaId).toBe('d1'); + expect(t.replicaFiles).toEqual(dictFiles); + expect(t.replicaBase).toBe('Dictionaries'); + expect(t.totalBytes).toBe(5000); + }); + + test('queueReplicaUpload returns existing id if same (kind, id) is already queued', async () => { + const appService = makeAppService(); + appService['uploadReplicaFile'] = vi.fn().mockResolvedValue(undefined); + await transferManager.initialize(appService as never, () => [], vi.fn(), translationFn); + + const id1 = transferManager.queueReplicaUpload( + 'dictionary', + 'd1', + 'Webster', + dictFiles, + 'Dictionaries', + ); + const id2 = transferManager.queueReplicaUpload( + 'dictionary', + 'd1', + 'Webster', + dictFiles, + 'Dictionaries', + ); + expect(id1).toBe(id2); + }); + + test('different replicaKinds with same id are distinct queue entries', async () => { + const appService = makeAppService(); + appService['uploadReplicaFile'] = vi.fn().mockResolvedValue(undefined); + await transferManager.initialize(appService as never, () => [], vi.fn(), translationFn); + + const dictId = transferManager.queueReplicaUpload( + 'dictionary', + 'shared-hash', + 'D', + dictFiles, + 'Dictionaries', + ); + const fontId = transferManager.queueReplicaUpload( + 'font', + 'shared-hash', + 'F', + [{ logical: 'roboto.ttf', lfp: 'f/roboto.ttf', byteSize: 200000 }], + 'Fonts', + ); + expect(dictId).not.toBe(fontId); + }); + + test('queueReplicaDownload creates a download-typed transfer', async () => { + const appService = makeAppService(); + appService['downloadReplicaFile'] = vi.fn().mockResolvedValue(undefined); + await transferManager.initialize(appService as never, () => [], vi.fn(), translationFn); + + const id = transferManager.queueReplicaDownload( + 'dictionary', + 'd1', + 'Webster', + dictFiles, + 'Dictionaries', + ); + const t = useTransferStore.getState().transfers[id!]!; + expect(t.type).toBe('download'); + expect(t.replicaFiles).toEqual(dictFiles); + }); + + test('queueReplicaDelete creates a delete-typed transfer with filename list', async () => { + const appService = makeAppService(); + appService['deleteReplicaBundle'] = vi.fn().mockResolvedValue(undefined); + await transferManager.initialize(appService as never, () => [], vi.fn(), translationFn); + + const id = transferManager.queueReplicaDelete('dictionary', 'd1', 'Webster', [ + 'webster.mdx', + 'webster.mdd', + ]); + const t = useTransferStore.getState().transfers[id!]!; + expect(t.type).toBe('delete'); + expect(t.replicaFiles?.map((f) => f.logical)).toEqual(['webster.mdx', 'webster.mdd']); + }); + }); }); diff --git a/apps/readest-app/src/__tests__/store/transfer-store.test.ts b/apps/readest-app/src/__tests__/store/transfer-store.test.ts index ac5797aa..79bab174 100644 --- a/apps/readest-app/src/__tests__/store/transfer-store.test.ts +++ b/apps/readest-app/src/__tests__/store/transfer-store.test.ts @@ -349,6 +349,7 @@ describe('transferStore', () => { const transfers: Record = { t1: { id: 't1', + kind: 'book', bookHash: 'h1', bookTitle: 'B1', type: 'upload', @@ -366,6 +367,7 @@ describe('transferStore', () => { }, t2: { id: 't2', + kind: 'book', bookHash: 'h2', bookTitle: 'B2', type: 'download', @@ -429,4 +431,133 @@ describe('transferStore', () => { expect(useTransferStore.getState().activeCount).toBe(5); }); }); + + // ── kind discriminator ─────────────────────────────────────────── + describe('replica transfers', () => { + test('addTransfer defaults kind to "book"', () => { + const id = useTransferStore.getState().addTransfer('h1', 'B1', 'upload'); + expect(useTransferStore.getState().transfers[id]!.kind).toBe('book'); + }); + + test('addReplicaTransfer creates a kind="replica" item with replicaKind/replicaId', () => { + const id = useTransferStore + .getState() + .addReplicaTransfer('dictionary', 'dict-content-hash', 'Webster', 'upload'); + const t = useTransferStore.getState().transfers[id]!; + expect(t.kind).toBe('replica'); + expect(t.replicaKind).toBe('dictionary'); + expect(t.replicaId).toBe('dict-content-hash'); + expect(t.bookTitle).toBe('Webster'); + expect(t.bookHash).toBe(''); + expect(t.type).toBe('upload'); + expect(t.status).toBe('pending'); + }); + + test('addReplicaTransfer accepts custom priority and isBackground', () => { + const id = useTransferStore + .getState() + .addReplicaTransfer('dictionary', 'd1', 'D1', 'download', { + priority: 1, + isBackground: true, + }); + const t = useTransferStore.getState().transfers[id]!; + expect(t.priority).toBe(1); + expect(t.isBackground).toBe(true); + }); + + test('addReplicaTransfer stores replicaFiles + replicaBase + computes totalBytes', () => { + const id = useTransferStore + .getState() + .addReplicaTransfer('dictionary', 'd1', 'Webster', 'upload', { + files: [ + { logical: 'webster.mdx', lfp: 'd1/webster.mdx', byteSize: 1000 }, + { logical: 'webster.mdd', lfp: 'd1/webster.mdd', byteSize: 2500 }, + ], + base: 'Dictionaries', + }); + const t = useTransferStore.getState().transfers[id]!; + expect(t.replicaFiles).toHaveLength(2); + expect(t.replicaBase).toBe('Dictionaries'); + expect(t.totalBytes).toBe(3500); + }); + + test('getReplicaTransfer finds replica items by (replicaKind, replicaId, type)', () => { + const id = useTransferStore.getState().addReplicaTransfer('dictionary', 'd1', 'D1', 'upload'); + const found = useTransferStore.getState().getReplicaTransfer('dictionary', 'd1', 'upload'); + expect(found?.id).toBe(id); + }); + + test('getReplicaTransfer returns undefined for completed/failed transfers', () => { + const id = useTransferStore.getState().addReplicaTransfer('dictionary', 'd1', 'D1', 'upload'); + useTransferStore.getState().setTransferStatus(id, 'completed'); + expect( + useTransferStore.getState().getReplicaTransfer('dictionary', 'd1', 'upload'), + ).toBeUndefined(); + }); + + test('getReplicaTransfer ignores book transfers', () => { + useTransferStore.getState().addTransfer('book-hash', 'Book', 'upload'); + expect( + useTransferStore.getState().getReplicaTransfer('dictionary', 'book-hash', 'upload'), + ).toBeUndefined(); + }); + + test('getTransferByBookHash ignores replica transfers (defensive against bookHash="" collisions)', () => { + useTransferStore.getState().addReplicaTransfer('dictionary', 'd1', 'D1', 'upload'); + expect(useTransferStore.getState().getTransferByBookHash('', 'upload')).toBeUndefined(); + }); + + test('different replicaKinds with same replicaId are distinct', () => { + const idA = useTransferStore + .getState() + .addReplicaTransfer('dictionary', 'shared-hash', 'Dict', 'upload'); + const idB = useTransferStore + .getState() + .addReplicaTransfer('font', 'shared-hash', 'Font', 'upload'); + expect(idA).not.toBe(idB); + expect( + useTransferStore.getState().getReplicaTransfer('dictionary', 'shared-hash', 'upload')!.id, + ).toBe(idA); + expect( + useTransferStore.getState().getReplicaTransfer('font', 'shared-hash', 'upload')!.id, + ).toBe(idB); + }); + + test('clearAll removes book and replica transfers alike', () => { + useTransferStore.getState().addTransfer('h1', 'B1', 'upload'); + useTransferStore.getState().addReplicaTransfer('dictionary', 'd1', 'D1', 'upload'); + useTransferStore.getState().clearAll(); + expect(Object.keys(useTransferStore.getState().transfers)).toHaveLength(0); + }); + + test('getQueueStats counts both book and replica transfers', () => { + useTransferStore.getState().addTransfer('h1', 'B1', 'upload'); + useTransferStore.getState().addReplicaTransfer('dictionary', 'd1', 'D1', 'upload'); + expect(useTransferStore.getState().getQueueStats().total).toBe(2); + expect(useTransferStore.getState().getQueueStats().pending).toBe(2); + }); + + test('restoreTransfers fills kind="book" for legacy persisted rows missing the field', () => { + const legacy = { + legacy1: { + id: 'legacy1', + bookHash: 'h', + bookTitle: 'Old', + type: 'upload', + status: 'pending', + progress: 0, + totalBytes: 0, + transferredBytes: 0, + transferSpeed: 0, + retryCount: 0, + maxRetries: 3, + createdAt: 1, + priority: 10, + isBackground: false, + } as unknown as TransferItem, + }; + useTransferStore.getState().restoreTransfers(legacy, false); + expect(useTransferStore.getState().transfers['legacy1']!.kind).toBe('book'); + }); + }); }); diff --git a/apps/readest-app/src/app/api/share/[token]/cover/route.ts b/apps/readest-app/src/app/api/share/[token]/cover/route.ts index 075071fe..cb6b3907 100644 --- a/apps/readest-app/src/app/api/share/[token]/cover/route.ts +++ b/apps/readest-app/src/app/api/share/[token]/cover/route.ts @@ -1,6 +1,6 @@ import { NextResponse } from 'next/server'; import { getDownloadSignedUrl } from '@/utils/object'; -import { rejectionToHttp, resolveActiveShare } from '@/libs/share-server'; +import { rejectionToHttp, resolveActiveShare } from '@/libs/shareServer'; import { SHARE_PRESIGN_TTL_SECONDS } from '@/services/constants'; interface RouteParams { diff --git a/apps/readest-app/src/app/api/share/[token]/download/confirm/route.ts b/apps/readest-app/src/app/api/share/[token]/download/confirm/route.ts index 82635ac9..f1fb5ff2 100644 --- a/apps/readest-app/src/app/api/share/[token]/download/confirm/route.ts +++ b/apps/readest-app/src/app/api/share/[token]/download/confirm/route.ts @@ -1,6 +1,6 @@ import { NextResponse } from 'next/server'; import { createSupabaseAdminClient } from '@/utils/supabase'; -import { hashShareToken, isValidShareToken } from '@/libs/share-server'; +import { hashShareToken, isValidShareToken } from '@/libs/shareServer'; interface RouteParams { params: Promise<{ token: string }>; diff --git a/apps/readest-app/src/app/api/share/[token]/download/route.ts b/apps/readest-app/src/app/api/share/[token]/download/route.ts index 7855f120..99540138 100644 --- a/apps/readest-app/src/app/api/share/[token]/download/route.ts +++ b/apps/readest-app/src/app/api/share/[token]/download/route.ts @@ -1,6 +1,6 @@ import { NextResponse } from 'next/server'; import { getDownloadSignedUrl } from '@/utils/object'; -import { rejectionToHttp, resolveActiveShare } from '@/libs/share-server'; +import { rejectionToHttp, resolveActiveShare } from '@/libs/shareServer'; import { SHARE_PRESIGN_TTL_SECONDS } from '@/services/constants'; interface RouteParams { diff --git a/apps/readest-app/src/app/api/share/[token]/import/route.ts b/apps/readest-app/src/app/api/share/[token]/import/route.ts index ae2d4c40..bb462226 100644 --- a/apps/readest-app/src/app/api/share/[token]/import/route.ts +++ b/apps/readest-app/src/app/api/share/[token]/import/route.ts @@ -6,7 +6,7 @@ import { getStoragePlanData, validateUserAndToken, } from '@/utils/access'; -import { rejectionToHttp, resolveActiveShare } from '@/libs/share-server'; +import { rejectionToHttp, resolveActiveShare } from '@/libs/shareServer'; interface RouteParams { params: Promise<{ token: string }>; diff --git a/apps/readest-app/src/app/api/share/[token]/og.png/render.tsx b/apps/readest-app/src/app/api/share/[token]/og.png/render.tsx index e31818db..c9b02198 100644 --- a/apps/readest-app/src/app/api/share/[token]/og.png/render.tsx +++ b/apps/readest-app/src/app/api/share/[token]/og.png/render.tsx @@ -1,7 +1,7 @@ import { ImageResponse } from 'next/og'; import { NextResponse } from 'next/server'; import { getDownloadSignedUrl } from '@/utils/object'; -import { rejectionToHttp, resolveActiveShare } from '@/libs/share-server'; +import { rejectionToHttp, resolveActiveShare } from '@/libs/shareServer'; import { SHARE_PRESIGN_TTL_SECONDS } from '@/services/constants'; // JSX renderer for the OG image. Lives in a non-route `.tsx` so the route diff --git a/apps/readest-app/src/app/api/share/[token]/revoke/route.ts b/apps/readest-app/src/app/api/share/[token]/revoke/route.ts index dc80b3f4..b971afd8 100644 --- a/apps/readest-app/src/app/api/share/[token]/revoke/route.ts +++ b/apps/readest-app/src/app/api/share/[token]/revoke/route.ts @@ -1,7 +1,7 @@ import { NextResponse } from 'next/server'; import { createSupabaseAdminClient } from '@/utils/supabase'; import { validateUserAndToken } from '@/utils/access'; -import { hashShareToken, isValidShareToken } from '@/libs/share-server'; +import { hashShareToken, isValidShareToken } from '@/libs/shareServer'; interface RouteParams { params: Promise<{ token: string }>; diff --git a/apps/readest-app/src/app/api/share/[token]/route.ts b/apps/readest-app/src/app/api/share/[token]/route.ts index 20032019..c13a5191 100644 --- a/apps/readest-app/src/app/api/share/[token]/route.ts +++ b/apps/readest-app/src/app/api/share/[token]/route.ts @@ -1,5 +1,5 @@ import { NextResponse } from 'next/server'; -import { rejectionToHttp, resolveActiveShare } from '@/libs/share-server'; +import { rejectionToHttp, resolveActiveShare } from '@/libs/shareServer'; interface RouteParams { params: Promise<{ token: string }>; diff --git a/apps/readest-app/src/app/api/share/create/route.ts b/apps/readest-app/src/app/api/share/create/route.ts index 98d1077b..0fcec2c9 100644 --- a/apps/readest-app/src/app/api/share/create/route.ts +++ b/apps/readest-app/src/app/api/share/create/route.ts @@ -1,7 +1,7 @@ import { NextResponse } from 'next/server'; import { createSupabaseAdminClient } from '@/utils/supabase'; import { validateUserAndToken } from '@/utils/access'; -import { generateShareToken } from '@/libs/share-server'; +import { generateShareToken } from '@/libs/shareServer'; import { objectExists } from '@/utils/object'; import { SHARE_BASE_URL, diff --git a/apps/readest-app/src/app/s/page.tsx b/apps/readest-app/src/app/s/page.tsx index e4b7bf66..dc9213ab 100644 --- a/apps/readest-app/src/app/s/page.tsx +++ b/apps/readest-app/src/app/s/page.tsx @@ -1,7 +1,7 @@ import type { Metadata } from 'next'; import { Suspense } from 'react'; import { READEST_WEB_BASE_URL, SHARE_BASE_URL } from '@/services/constants'; -import { resolveActiveShare } from '@/libs/share-server'; +import { resolveActiveShare } from '@/libs/shareServer'; import ShareLanding from './ShareLanding'; // Server-rendered metadata for chat unfurls. Lives on the page (not the diff --git a/apps/readest-app/src/context/EnvContext.tsx b/apps/readest-app/src/context/EnvContext.tsx index 98d8491c..31ab97ac 100644 --- a/apps/readest-app/src/context/EnvContext.tsx +++ b/apps/readest-app/src/context/EnvContext.tsx @@ -4,6 +4,7 @@ import React, { createContext, useContext, useState, useMemo, ReactNode } from ' import { EnvConfigType } from '../services/environment'; import { AppService } from '@/types/system'; import env from '../services/environment'; +import { bootstrapReplicaAdapters } from '@/services/sync/replicaBootstrap'; interface EnvContextType { envConfig: EnvConfigType; @@ -17,6 +18,7 @@ export const EnvProvider = ({ children }: { children: ReactNode }) => { const [appService, setAppService] = useState(null); React.useEffect(() => { + bootstrapReplicaAdapters(); envConfig.getAppService().then((service) => setAppService(service)); window.addEventListener('error', (e) => { if (e.message === 'ResizeObserver loop limit exceeded') { diff --git a/apps/readest-app/src/libs/crdt.README.md b/apps/readest-app/src/libs/crdt.README.md new file mode 100644 index 00000000..1c80a373 --- /dev/null +++ b/apps/readest-app/src/libs/crdt.README.md @@ -0,0 +1,103 @@ +# `crdt.ts` — Hybrid Logical Clock + per-field LWW + +Internal sync primitive for the polymorphic `replicas` table. See +`apps/readest-app/.claude/plans/vivid-orbiting-thimble.md` for the full design. + +## What this module does + +- Generates **HLC** timestamps that are monotonic per device, absorb remote + ordering when observed, and survive wall-clock regression. +- Merges replica rows under **field-level LWW** with **remove-wins + tombstones**. +- Is pure — no IndexedDB, no fetch, no DOM. Persistence is layered on top + by `replicaSyncManager` and a Tauri/web HLC store. + +## HLC packing format + +``` +0000018e7d6ab5c0-00000007-device-uuid +└── physicalMs ─┘ └counter┘ └─deviceId─┘ + 13 hex chars 8 hex free-form +``` + +Lexicographic comparison of the packed string matches temporal order: + +``` +hlcCompare(packA, packB) === -1 ⟺ (msA, ctrA) < (msB, ctrB) +``` + +This is invariant — see the 1000-sample property test in +`__tests__/libs/crdt.test.ts`. + +## Per-field LWW envelope + +Each field in `fields_jsonb` is wrapped: + +``` +{ v: , t: , s: } + value when who wrote it +``` + +`mergeFields(local, remote)` keeps the envelope with the larger HLC. Ties +on HLC are broken by `deviceId` lex order (deterministic, both sides +converge). + +## Remove-wins tombstones + +A `deleted_at_ts` HLC marks the row deleted. **Field writes do NOT revive +a tombstoned row** — that is the unsafe pattern. To revive, use +`withReincarnation(row, token)` which creates a new logical entity. + +``` + deleted_at_ts ≠ null + │ + ┌─ field writes accumulate normally + │ but the row stays deleted + │ + reincarnation = 'epoch-N' ← explicit token from importer + │ + ▼ + row is alive again under new logical identity +``` + +## Merge semantics summary + +``` +mergeReplica(local, remote): + fields_jsonb ← per-field LWW + deleted_at_ts ← max(local, remote) (tombstones never disappear) + reincarnation ← LWW by row updated_at_ts + manifest_jsonb ← whichever side is newer + schema_version ← max + updated_at_ts ← max over fields and tombstone +``` + +CRDT properties verified by tests: + +- `mergeFields(a, b) === mergeFields(b, a)` +- `mergeFields(mergeFields(a, b), c) === mergeFields(a, mergeFields(b, c))` +- `mergeFields(a, a) === a` +- `mergeReplica` is commutative and idempotent + +## When you change this module + +This is one of the few places in Readest where correctness is +non-negotiable. Bugs cause silent data loss across devices. + +- Never weaken any of the four CRDT properties. +- Never let a field write revive a tombstone. +- Never break the HLC packing format (other devices comparing on the + string format will diverge). +- Add a property test for any new merge rule. +- Re-run `pnpm test src/__tests__/libs/crdt.test.ts` after every change. + +## Where things go from here + +- `crypto/` (Lane B) — encrypted-field envelopes plug into `setField` / + `mergeFields` transparently. The CRDT machinery sees opaque + ciphertext. +- `replicaSyncManager` (Lane E) — owns the HLC generator instance, + observes remote HLCs on pull, persists snapshots to IndexedDB. +- `crdt_merge_replica()` Postgres function — must implement the same + merge semantics on the server side, atomically. Tests in PR 1 will + enforce client/server parity. diff --git a/apps/readest-app/src/libs/crdt.ts b/apps/readest-app/src/libs/crdt.ts new file mode 100644 index 00000000..f494f1e1 --- /dev/null +++ b/apps/readest-app/src/libs/crdt.ts @@ -0,0 +1,192 @@ +import type { FieldEnvelope, FieldsObject, Hlc, ReplicaRow } from '@/types/replica'; + +const MAX_PHYSICAL_MS = 0xfffffffffffff; +const MAX_COUNTER = 0xffffffff; + +export const hlcPack = (physicalMs: number, counter: number, deviceId: string): Hlc => { + if (physicalMs < 0 || physicalMs > MAX_PHYSICAL_MS) { + throw new RangeError(`physicalMs out of range: ${physicalMs}`); + } + if (counter < 0 || counter > MAX_COUNTER) { + throw new RangeError(`counter out of range: ${counter}`); + } + const ms = physicalMs.toString(16).padStart(13, '0'); + const c = counter.toString(16).padStart(8, '0'); + return `${ms}-${c}-${deviceId}` as Hlc; +}; + +export const hlcParse = (h: Hlc): { physicalMs: number; counter: number; deviceId: string } => { + const dash1 = h.indexOf('-'); + const dash2 = h.indexOf('-', dash1 + 1); + if (dash1 < 0 || dash2 < 0) throw new Error(`malformed HLC: ${h}`); + return { + physicalMs: parseInt(h.slice(0, dash1), 16), + counter: parseInt(h.slice(dash1 + 1, dash2), 16), + deviceId: h.slice(dash2 + 1), + }; +}; + +export const hlcCompare = (a: Hlc, b: Hlc): -1 | 0 | 1 => (a < b ? -1 : a > b ? 1 : 0); + +export const hlcMax = (a: Hlc | null, b: Hlc | null): Hlc | null => { + if (!a) return b; + if (!b) return a; + return hlcCompare(a, b) >= 0 ? a : b; +}; + +export interface HlcSnapshot { + physicalMs: number; + counter: number; +} + +export class HlcGenerator { + private physicalMs = 0; + private counter = 0; + + constructor( + private readonly deviceId: string, + private readonly now: () => number = Date.now, + ) {} + + static restore( + snapshot: HlcSnapshot, + deviceId: string, + now: () => number = Date.now, + ): HlcGenerator { + const gen = new HlcGenerator(deviceId, now); + gen.physicalMs = snapshot.physicalMs; + gen.counter = snapshot.counter; + return gen; + } + + next(): Hlc { + const wallMs = this.now(); + if (wallMs > this.physicalMs) { + this.physicalMs = wallMs; + this.counter = 0; + } else { + this.counter += 1; + if (this.counter > MAX_COUNTER) { + this.physicalMs += 1; + this.counter = 0; + } + } + return hlcPack(this.physicalMs, this.counter, this.deviceId); + } + + observe(remote: Hlc): void { + const { physicalMs: rMs, counter: rC } = hlcParse(remote); + const wallMs = this.now(); + const newMs = Math.max(this.physicalMs, rMs, wallMs); + if (newMs === this.physicalMs && newMs === rMs) { + this.counter = Math.max(this.counter, rC) + 1; + } else if (newMs === rMs) { + this.physicalMs = newMs; + this.counter = rC + 1; + } else if (newMs === this.physicalMs) { + this.counter += 1; + } else { + this.physicalMs = newMs; + this.counter = 0; + } + } + + serialize(): HlcSnapshot { + return { physicalMs: this.physicalMs, counter: this.counter }; + } +} + +export const setField = ( + fields: FieldsObject, + name: string, + value: V, + hlc: Hlc, + deviceId: string, +): FieldsObject => ({ + ...fields, + [name]: { v: value, t: hlc, s: deviceId }, +}); + +const pickWinner = (a: FieldEnvelope, b: FieldEnvelope): FieldEnvelope => { + const cmp = hlcCompare(a.t, b.t); + if (cmp > 0) return a; + if (cmp < 0) return b; + return a.s >= b.s ? a : b; +}; + +export const mergeFields = (local: FieldsObject, remote: FieldsObject): FieldsObject => { + const out: FieldsObject = { ...local }; + for (const key of Object.keys(remote)) { + const lo = local[key]; + const re = remote[key]!; + out[key] = lo ? pickWinner(lo, re) : re; + } + return out; +}; + +const computeUpdatedAt = (fields: FieldsObject, deletedAt: Hlc | null): Hlc => { + let max: Hlc | null = deletedAt; + for (const key of Object.keys(fields)) { + const env = fields[key]!; + max = hlcMax(max, env.t); + } + return (max ?? hlcPack(0, 0, '')) as Hlc; +}; + +export const removeReplica = (row: ReplicaRow, hlc: Hlc): ReplicaRow => { + const deletedAt = hlcMax(row.deleted_at_ts, hlc) ?? hlc; + return { + ...row, + deleted_at_ts: deletedAt, + updated_at_ts: hlcMax(row.updated_at_ts, hlc) ?? hlc, + }; +}; + +export const withReincarnation = (row: ReplicaRow, token: string): ReplicaRow => ({ + ...row, + reincarnation: token, + deleted_at_ts: null, + fields_jsonb: {}, +}); + +export const mergeReplica = (local: ReplicaRow, remote: ReplicaRow): ReplicaRow => { + if ( + local.user_id !== remote.user_id || + local.kind !== remote.kind || + local.replica_id !== remote.replica_id + ) { + throw new Error('mergeReplica: identity mismatch'); + } + const fields = mergeFields(local.fields_jsonb, remote.fields_jsonb); + const deleted_at_ts = hlcMax(local.deleted_at_ts, remote.deleted_at_ts); + + let reincarnation: string | null; + const localReinc = local.reincarnation; + const remoteReinc = remote.reincarnation; + if (localReinc === remoteReinc) { + reincarnation = localReinc; + } else { + const cmp = hlcCompare(local.updated_at_ts, remote.updated_at_ts); + reincarnation = cmp >= 0 ? localReinc : remoteReinc; + } + + const manifest_jsonb = + hlcCompare(remote.updated_at_ts, local.updated_at_ts) > 0 + ? remote.manifest_jsonb + : local.manifest_jsonb; + + const schema_version = Math.max(local.schema_version, remote.schema_version); + const updated_at_ts = computeUpdatedAt(fields, deleted_at_ts); + + return { + user_id: local.user_id, + kind: local.kind, + replica_id: local.replica_id, + fields_jsonb: fields, + manifest_jsonb, + deleted_at_ts, + reincarnation, + updated_at_ts, + schema_version, + }; +}; diff --git a/apps/readest-app/src/libs/crypto/derive.ts b/apps/readest-app/src/libs/crypto/derive.ts new file mode 100644 index 00000000..91321a01 --- /dev/null +++ b/apps/readest-app/src/libs/crypto/derive.ts @@ -0,0 +1,41 @@ +import { SyncError } from '@/libs/errors'; + +const PBKDF2_DEFAULT_ITERATIONS = 600_000; +const KEY_LENGTH_BITS = 256; + +const requireSubtle = (): SubtleCrypto => { + if (typeof crypto === 'undefined' || !crypto.subtle) { + throw new SyncError('CRYPTO_UNAVAILABLE', 'Web Crypto subtle is not available'); + } + return crypto.subtle; +}; + +export const derivePbkdf2Key = async ( + passphrase: string, + salt: Uint8Array, + iterations: number = PBKDF2_DEFAULT_ITERATIONS, +): Promise => { + const subtle = requireSubtle(); + const passphraseKey = await subtle.importKey( + 'raw', + new TextEncoder().encode(passphrase), + { name: 'PBKDF2' }, + false, + ['deriveBits', 'deriveKey'], + ); + return subtle.deriveKey( + { name: 'PBKDF2', salt: salt as BufferSource, iterations, hash: 'SHA-256' }, + passphraseKey, + { name: 'AES-GCM', length: KEY_LENGTH_BITS }, + true, + ['encrypt', 'decrypt'], + ); +}; + +export const exportRawKey = async (key: CryptoKey): Promise => { + const subtle = requireSubtle(); + const raw = await subtle.exportKey('raw', key); + return new Uint8Array(raw); +}; + +export const PBKDF2_ITERATIONS = PBKDF2_DEFAULT_ITERATIONS; diff --git a/apps/readest-app/src/libs/crypto/encrypt.ts b/apps/readest-app/src/libs/crypto/encrypt.ts new file mode 100644 index 00000000..41681c87 --- /dev/null +++ b/apps/readest-app/src/libs/crypto/encrypt.ts @@ -0,0 +1,46 @@ +import { SyncError } from '@/libs/errors'; + +const IV_BYTES = 12; + +export interface EncryptedPayload { + iv: Uint8Array; + ciphertext: Uint8Array; +} + +const requireSubtle = (): SubtleCrypto => { + if (typeof crypto === 'undefined' || !crypto.subtle) { + throw new SyncError('CRYPTO_UNAVAILABLE', 'Web Crypto subtle is not available'); + } + return crypto.subtle; +}; + +export const encryptPlaintext = async ( + plaintext: Uint8Array, + key: CryptoKey, +): Promise => { + const subtle = requireSubtle(); + const iv = crypto.getRandomValues(new Uint8Array(IV_BYTES)); + const ciphertext = await subtle.encrypt( + { name: 'AES-GCM', iv: iv as BufferSource }, + key, + plaintext as BufferSource, + ); + return { iv, ciphertext: new Uint8Array(ciphertext) }; +}; + +export const decryptCiphertext = async ( + payload: EncryptedPayload, + key: CryptoKey, +): Promise => { + const subtle = requireSubtle(); + try { + const plaintext = await subtle.decrypt( + { name: 'AES-GCM', iv: payload.iv as BufferSource }, + key, + payload.ciphertext as BufferSource, + ); + return new Uint8Array(plaintext); + } catch (cause) { + throw new SyncError('DECRYPT', 'AES-GCM decryption failed', { cause }); + } +}; diff --git a/apps/readest-app/src/libs/crypto/envelope.ts b/apps/readest-app/src/libs/crypto/envelope.ts new file mode 100644 index 00000000..66174162 --- /dev/null +++ b/apps/readest-app/src/libs/crypto/envelope.ts @@ -0,0 +1,83 @@ +import { SyncError } from '@/libs/errors'; +import type { CipherEnvelope } from '@/types/replica'; +import { decryptCiphertext, encryptPlaintext } from './encrypt'; + +export const CURRENT_ALG = 'aes-gcm/pbkdf2-600k-sha256'; +const SUPPORTED_ALGS = new Set([CURRENT_ALG]); + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +const requireSubtle = (): SubtleCrypto => { + if (typeof crypto === 'undefined' || !crypto.subtle) { + throw new SyncError('CRYPTO_UNAVAILABLE', 'Web Crypto subtle is not available'); + } + return crypto.subtle; +}; + +const bytesToBase64 = (bytes: Uint8Array): string => { + let s = ''; + const chunk = 0x8000; + for (let i = 0; i < bytes.length; i += chunk) { + s += String.fromCharCode(...bytes.subarray(i, i + chunk)); + } + return btoa(s); +}; + +const base64ToBytes = (b64: string): Uint8Array => { + const s = atob(b64); + const out = new Uint8Array(s.length); + for (let i = 0; i < s.length; i++) out[i] = s.charCodeAt(i); + return out; +}; + +const sha256 = async (bytes: Uint8Array): Promise => { + const hash = await requireSubtle().digest('SHA-256', bytes as BufferSource); + return new Uint8Array(hash); +}; + +const constantTimeEq = (a: Uint8Array, b: Uint8Array): boolean => { + if (a.length !== b.length) return false; + let diff = 0; + for (let i = 0; i < a.length; i++) diff |= a[i]! ^ b[i]!; + return diff === 0; +}; + +export const encryptToEnvelope = async ( + plaintext: string | Uint8Array, + key: CryptoKey, + saltId: string, +): Promise => { + const bytes = typeof plaintext === 'string' ? enc.encode(plaintext) : plaintext; + const payload = await encryptPlaintext(bytes, key); + const sidecar = await sha256(bytes); + return { + c: bytesToBase64(payload.ciphertext), + i: bytesToBase64(payload.iv), + s: saltId, + alg: CURRENT_ALG, + h: bytesToBase64(sidecar), + }; +}; + +export const decryptFromEnvelope = async ( + envelope: CipherEnvelope, + key: CryptoKey, +): Promise => { + if (!SUPPORTED_ALGS.has(envelope.alg)) { + throw new SyncError('UNSUPPORTED_ALG', `Unsupported envelope alg: ${envelope.alg}`, { + cause: envelope.alg, + }); + } + const ciphertext = base64ToBytes(envelope.c); + const iv = base64ToBytes(envelope.i); + const expectedSidecar = base64ToBytes(envelope.h); + const plain = await decryptCiphertext({ iv, ciphertext }, key); + const actualSidecar = await sha256(plain); + if (!constantTimeEq(expectedSidecar, actualSidecar)) { + throw new SyncError('INTEGRITY', 'SHA-256 sidecar verification failed'); + } + return dec.decode(plain); +}; + +export const isSupportedAlg = (alg: string): boolean => SUPPORTED_ALGS.has(alg); diff --git a/apps/readest-app/src/libs/crypto/passphrase.ts b/apps/readest-app/src/libs/crypto/passphrase.ts new file mode 100644 index 00000000..e4434a8d --- /dev/null +++ b/apps/readest-app/src/libs/crypto/passphrase.ts @@ -0,0 +1,49 @@ +import { SyncError } from '@/libs/errors'; + +export interface PassphraseStore { + set(passphrase: string): Promise; + get(): Promise; + clear(): Promise; + isAvailable(): boolean; +} + +export class EphemeralPassphraseStore implements PassphraseStore { + private value: string | null = null; + + async set(passphrase: string): Promise { + this.value = passphrase; + } + + async get(): Promise { + return this.value; + } + + async clear(): Promise { + this.value = null; + } + + isAvailable(): boolean { + return true; + } +} + +export class TauriPassphraseStore implements PassphraseStore { + async set(_passphrase: string): Promise { + throw new SyncError( + 'CRYPTO_UNAVAILABLE', + 'Tauri keychain backend not yet wired (TODO before PR 4 ships).', + ); + } + + async get(): Promise { + return null; + } + + async clear(): Promise {} + + isAvailable(): boolean { + return false; + } +} + +export const createPassphraseStore = (): PassphraseStore => new EphemeralPassphraseStore(); diff --git a/apps/readest-app/src/libs/errors.ts b/apps/readest-app/src/libs/errors.ts new file mode 100644 index 00000000..b46bd07b --- /dev/null +++ b/apps/readest-app/src/libs/errors.ts @@ -0,0 +1,48 @@ +export type SyncErrorCode = + | 'TIMEOUT' + | 'AUTH' + | 'QUOTA_EXCEEDED' + | 'CLOCK_SKEW' + | 'VALIDATION' + | 'SERVER' + | 'DECRYPT' + | 'INTEGRITY' + | 'UNSUPPORTED_ALG' + | 'SALT_NOT_FOUND' + | 'CRYPTO_UNAVAILABLE' + | 'NO_PASSPHRASE' + | 'LOCAL_FILE_MISSING' + | 'TRANSFER' + | 'STORAGE' + | 'MANIFEST_COMMIT' + | 'UNKNOWN_KIND' + | 'SCHEMA_TOO_NEW' + | 'LEGACY_MIGRATION_SKIP' + | 'HLC_PERSIST'; + +export interface SyncErrorContext { + replicaId?: string; + kind?: string; + field?: string; + status?: number; + cause?: unknown; +} + +export class SyncError extends Error { + readonly code: SyncErrorCode; + readonly context: SyncErrorContext; + + constructor(code: SyncErrorCode, message: string, context: SyncErrorContext = {}) { + super(message); + this.name = 'SyncError'; + this.code = code; + this.context = context; + } +} + +export const isSyncError = (e: unknown): e is SyncError => + e instanceof SyncError || (e instanceof Error && e.name === 'SyncError'); + +export const assertNever = (x: never): never => { + throw new SyncError('VALIDATION', `Unexpected value: ${JSON.stringify(x)}`); +}; diff --git a/apps/readest-app/src/libs/hlcStore.ts b/apps/readest-app/src/libs/hlcStore.ts new file mode 100644 index 00000000..7f45262d --- /dev/null +++ b/apps/readest-app/src/libs/hlcStore.ts @@ -0,0 +1,49 @@ +import type { HlcSnapshot } from '@/libs/crdt'; + +export const HLC_LOCAL_STORAGE_KEY = 'readest_replica_hlc'; + +export interface HlcSnapshotStore { + load(): HlcSnapshot | null; + save(snapshot: HlcSnapshot): void; +} + +export class InMemoryHlcStore implements HlcSnapshotStore { + private snapshot: HlcSnapshot | null = null; + + load(): HlcSnapshot | null { + return this.snapshot; + } + + save(snapshot: HlcSnapshot): void { + this.snapshot = snapshot; + } +} + +export class LocalStorageHlcStore implements HlcSnapshotStore { + constructor(private readonly key: string = HLC_LOCAL_STORAGE_KEY) {} + + load(): HlcSnapshot | null { + try { + const raw = localStorage.getItem(this.key); + if (!raw) return null; + const parsed = JSON.parse(raw) as HlcSnapshot; + if (typeof parsed?.physicalMs !== 'number' || typeof parsed?.counter !== 'number') { + return null; + } + return parsed; + } catch { + return null; + } + } + + save(snapshot: HlcSnapshot): void { + try { + localStorage.setItem(this.key, JSON.stringify(snapshot)); + } catch { + // localStorage unavailable (private mode, quota exceeded, SSR); + // HLC reverts to in-memory only for this session. Documented as a + // tolerable degradation; clients re-derive from server max(updated_at_ts) + // on next pull per replicaSyncManager. + } + } +} diff --git a/apps/readest-app/src/libs/replicaSchemas.ts b/apps/readest-app/src/libs/replicaSchemas.ts new file mode 100644 index 00000000..dd9b20ce --- /dev/null +++ b/apps/readest-app/src/libs/replicaSchemas.ts @@ -0,0 +1,172 @@ +import { z } from 'zod'; +import type { ReplicaRow } from '@/types/replica'; +import type { SyncErrorCode } from '@/libs/errors'; + +export const MAX_JSON_BYTES = 64 * 1024; +export const MAX_FIELD_COUNT = 64; +export const MAX_FILENAME_LEN = 255; + +const fieldEnvelopeSchema = z.object({ + v: z.unknown(), + t: z.string(), + s: z.string(), +}); + +const cipherEnvelopeSchema = z.object({ + c: z.string(), + i: z.string(), + s: z.string(), + alg: z.string(), + h: z.string(), +}); + +const fieldEnvelopeWithCipher = z.union([fieldEnvelopeSchema, cipherEnvelopeSchema]); + +const fieldsObjectSchema = z.record(z.string(), fieldEnvelopeWithCipher); + +const dictionaryFieldsSchema = z + .object({ + name: fieldEnvelopeSchema.optional(), + enabled: fieldEnvelopeSchema.optional(), + lang: fieldEnvelopeSchema.optional(), + }) + .catchall(fieldEnvelopeWithCipher); + +interface KindSpec { + minSchemaVersion: number; + maxSchemaVersion: number; + maxRowsPerUser: number; + fields: z.ZodTypeAny; + binary: boolean; +} + +export const KIND_ALLOWLIST: Record = { + dictionary: { + minSchemaVersion: 1, + maxSchemaVersion: 1, + maxRowsPerUser: 200, + fields: dictionaryFieldsSchema, + binary: true, + }, +}; + +export const isAllowedKind = (kind: string): boolean => Object.hasOwn(KIND_ALLOWLIST, kind); + +export interface FilenameOk { + ok: true; +} +export interface FilenameError { + ok: false; + reason: string; +} + +export const validateFilename = (name: string): FilenameOk | FilenameError => { + if (name.length === 0) return { ok: false, reason: 'empty' }; + if (name.length > MAX_FILENAME_LEN) return { ok: false, reason: 'too long' }; + if (name === '.' || name === '..') return { ok: false, reason: 'dot path' }; + if (name.includes('/') || name.includes('\\')) return { ok: false, reason: 'path separator' }; + if (name.includes('..')) return { ok: false, reason: 'path traversal' }; + for (let i = 0; i < name.length; i++) { + const code = name.charCodeAt(i); + if (code < 0x20 || code === 0x7f) { + return { ok: false, reason: 'control character' }; + } + } + return { ok: true }; +}; + +export type ValidationResult = + | { ok: true } + | { ok: false; code: SyncErrorCode; message: string; cause?: unknown }; + +const measureJsonBytes = (value: unknown): number => + new TextEncoder().encode(JSON.stringify(value)).length; + +const manifestFileSchema = z.object({ + filename: z.string(), + byteSize: z.number().int().nonnegative(), + partialMd5: z.string(), + mtime: z.number().optional(), +}); + +const manifestSchema = z.object({ + files: z.array(manifestFileSchema), + schemaVersion: z.number().int(), +}); + +export const validateRow = (row: ReplicaRow): ValidationResult => { + if (!isAllowedKind(row.kind)) { + return { ok: false, code: 'UNKNOWN_KIND', message: `Unknown kind: ${row.kind}` }; + } + const spec = KIND_ALLOWLIST[row.kind]!; + + if (row.schema_version < spec.minSchemaVersion || row.schema_version > spec.maxSchemaVersion) { + return { + ok: false, + code: 'SCHEMA_TOO_NEW', + message: `schemaVersion ${row.schema_version} out of bounds [${spec.minSchemaVersion}, ${spec.maxSchemaVersion}] for kind ${row.kind}`, + }; + } + + const fieldKeys = Object.keys(row.fields_jsonb); + if (fieldKeys.length > MAX_FIELD_COUNT) { + return { + ok: false, + code: 'VALIDATION', + message: `field count ${fieldKeys.length} exceeds MAX_FIELD_COUNT=${MAX_FIELD_COUNT}`, + }; + } + + const fieldsBytes = measureJsonBytes(row.fields_jsonb); + if (fieldsBytes > MAX_JSON_BYTES) { + return { + ok: false, + code: 'VALIDATION', + message: `fields_jsonb size ${fieldsBytes}B exceeds MAX_JSON_BYTES=${MAX_JSON_BYTES}`, + }; + } + + const fieldsParse = fieldsObjectSchema.safeParse(row.fields_jsonb); + if (!fieldsParse.success) { + return { + ok: false, + code: 'VALIDATION', + message: 'malformed field envelope', + cause: fieldsParse.error, + }; + } + + const kindParse = spec.fields.safeParse(row.fields_jsonb); + if (!kindParse.success) { + return { + ok: false, + code: 'VALIDATION', + message: `kind=${row.kind} fields validation failed`, + cause: kindParse.error, + }; + } + + if (row.manifest_jsonb !== null) { + const manifestParse = manifestSchema.safeParse(row.manifest_jsonb); + if (!manifestParse.success) { + return { + ok: false, + code: 'VALIDATION', + message: 'malformed manifest_jsonb', + cause: manifestParse.error, + }; + } + for (const file of row.manifest_jsonb.files) { + const fnCheck = validateFilename(file.filename); + if (!fnCheck.ok) { + return { + ok: false, + code: 'VALIDATION', + message: `manifest filename invalid: ${file.filename} (${fnCheck.reason})`, + }; + } + } + } + + return { ok: true }; +}; diff --git a/apps/readest-app/src/libs/replicaSyncClient.ts b/apps/readest-app/src/libs/replicaSyncClient.ts new file mode 100644 index 00000000..4e261caf --- /dev/null +++ b/apps/readest-app/src/libs/replicaSyncClient.ts @@ -0,0 +1,94 @@ +import { getAccessToken } from '@/utils/access'; +import { getAPIBaseUrl } from '@/services/environment'; +import { SyncError } from '@/libs/errors'; +import type { Hlc, ReplicaRow } from '@/types/replica'; +import type { SyncErrorCode } from '@/libs/errors'; + +const ENDPOINT = () => `${getAPIBaseUrl()}/sync/replicas`; + +interface ErrorBody { + error?: string; + code?: SyncErrorCode; + offendingIndex?: number; +} + +const statusToDefaultCode = (status: number): SyncErrorCode => { + if (status === 401 || status === 403) return 'AUTH'; + if (status === 402 || status === 507) return 'QUOTA_EXCEEDED'; + if (status === 409) return 'CLOCK_SKEW'; + if (status === 413) return 'VALIDATION'; + if (status === 422) return 'VALIDATION'; + if (status >= 500) return 'SERVER'; + return 'VALIDATION'; +}; + +const parseErrorBody = async (response: Response): Promise => { + try { + return (await response.json()) as ErrorBody; + } catch { + return {}; + } +}; + +const requireToken = async (): Promise => { + const token = await getAccessToken(); + if (!token) throw new SyncError('AUTH', 'Not authenticated'); + return token; +}; + +export class ReplicaSyncClient { + async push(rows: ReplicaRow[]): Promise { + if (rows.length === 0) return []; + const token = await requireToken(); + let response: Response; + try { + response = await fetch(ENDPOINT(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ rows }), + }); + } catch (cause) { + throw new SyncError('SERVER', 'Network failure during push', { cause }); + } + if (!response.ok) { + const body = await parseErrorBody(response); + const code = body.code ?? statusToDefaultCode(response.status); + throw new SyncError(code, body.error ?? `Push failed with status ${response.status}`, { + status: response.status, + }); + } + const data = (await response.json()) as { rows: ReplicaRow[] }; + return data.rows ?? []; + } + + async pull(kind: string, since: Hlc | null): Promise { + const token = await requireToken(); + const params = new URLSearchParams({ kind }); + if (since) params.set('since', since); + const url = `${ENDPOINT()}?${params.toString()}`; + let response: Response; + try { + response = await fetch(url, { + method: 'GET', + headers: { Authorization: `Bearer ${token}` }, + }); + } catch (cause) { + throw new SyncError('SERVER', 'Network failure during pull', { cause }); + } + if (response.status === 404) return []; + if (!response.ok) { + const body = await parseErrorBody(response); + const code = body.code ?? statusToDefaultCode(response.status); + throw new SyncError(code, body.error ?? `Pull failed with status ${response.status}`, { + status: response.status, + }); + } + const data = (await response.json()) as { rows: ReplicaRow[] }; + return data.rows ?? []; + } +} + +export const replicaSyncClient = new ReplicaSyncClient(); diff --git a/apps/readest-app/src/libs/replicaSyncServer.ts b/apps/readest-app/src/libs/replicaSyncServer.ts new file mode 100644 index 00000000..2c9fd711 --- /dev/null +++ b/apps/readest-app/src/libs/replicaSyncServer.ts @@ -0,0 +1,121 @@ +import { hlcParse } from '@/libs/crdt'; +import { isAllowedKind, validateRow } from '@/libs/replicaSchemas'; +import type { Hlc, ReplicaRow } from '@/types/replica'; +import type { SyncErrorCode } from '@/libs/errors'; + +export const HLC_SKEW_TOLERANCE_MS = 60_000; +export const MAX_PUSH_BATCH = 100; + +export interface PushReplicasBody { + rows: ReplicaRow[]; +} + +export type PushValidation = + | { ok: true; rows: ReplicaRow[] } + | { ok: false; status: number; code: SyncErrorCode; message: string; offendingIndex?: number }; + +export const clampHlcSkew = (hlc: Hlc, nowMs: number): boolean => { + const { physicalMs } = hlcParse(hlc); + return Math.abs(physicalMs - nowMs) <= HLC_SKEW_TOLERANCE_MS; +}; + +export const validatePushBatch = (body: unknown, userId: string, nowMs: number): PushValidation => { + if (typeof body !== 'object' || body === null) { + return { ok: false, status: 400, code: 'VALIDATION', message: 'body must be an object' }; + } + const rows = (body as PushReplicasBody).rows; + if (!Array.isArray(rows)) { + return { ok: false, status: 400, code: 'VALIDATION', message: 'body.rows must be an array' }; + } + if (rows.length === 0) { + return { ok: true, rows: [] }; + } + if (rows.length > MAX_PUSH_BATCH) { + return { + ok: false, + status: 413, + code: 'VALIDATION', + message: `batch size ${rows.length} exceeds MAX_PUSH_BATCH=${MAX_PUSH_BATCH}`, + }; + } + for (let i = 0; i < rows.length; i++) { + const row = rows[i]!; + if (row.user_id !== userId) { + return { + ok: false, + status: 403, + code: 'AUTH', + message: `row[${i}].user_id does not match authenticated user`, + offendingIndex: i, + }; + } + if (!isAllowedKind(row.kind)) { + return { + ok: false, + status: 422, + code: 'UNKNOWN_KIND', + message: `row[${i}].kind=${row.kind} is not in the server allowlist`, + offendingIndex: i, + }; + } + const v = validateRow(row); + if (!v.ok) { + return { + ok: false, + status: 422, + code: v.code, + message: `row[${i}] ${v.message}`, + offendingIndex: i, + }; + } + if (!clampHlcSkew(row.updated_at_ts, nowMs)) { + return { + ok: false, + status: 409, + code: 'CLOCK_SKEW', + message: `row[${i}].updated_at_ts physical time is outside ±${HLC_SKEW_TOLERANCE_MS}ms of server`, + offendingIndex: i, + }; + } + if (row.deleted_at_ts && !clampHlcSkew(row.deleted_at_ts, nowMs)) { + return { + ok: false, + status: 409, + code: 'CLOCK_SKEW', + message: `row[${i}].deleted_at_ts physical time is outside ±${HLC_SKEW_TOLERANCE_MS}ms of server`, + offendingIndex: i, + }; + } + } + return { ok: true, rows }; +}; + +export interface PullParams { + kind: string; + since: Hlc | null; +} + +export type PullValidation = + | { ok: true; params: PullParams } + | { ok: false; status: number; code: SyncErrorCode; message: string }; + +export const validatePullParams = (kind: string | null, since: string | null): PullValidation => { + if (!kind) { + return { ok: false, status: 400, code: 'VALIDATION', message: 'kind query parameter required' }; + } + if (!isAllowedKind(kind)) { + return { + ok: false, + status: 422, + code: 'UNKNOWN_KIND', + message: `kind=${kind} is not in the server allowlist`, + }; + } + return { + ok: true, + params: { + kind, + since: since ? (since as Hlc) : null, + }, + }; +}; diff --git a/apps/readest-app/src/libs/share-server.ts b/apps/readest-app/src/libs/shareServer.ts similarity index 100% rename from apps/readest-app/src/libs/share-server.ts rename to apps/readest-app/src/libs/shareServer.ts diff --git a/apps/readest-app/src/libs/storage.ts b/apps/readest-app/src/libs/storage.ts index 68a77335..0fea8508 100644 --- a/apps/readest-app/src/libs/storage.ts +++ b/apps/readest-app/src/libs/storage.ts @@ -77,6 +77,43 @@ export const uploadFile = async ( } }; +// Replica file upload. Reuses the books-style signed-URL path so 1+ GB +// dictionaries bypass the CF Workers body limit (per plan-eng-review §1). +// `cfp` is the cloud file path (key under the user's prefix); it must +// already contain the kind + replica-id prefix from CLOUD_REPLICAS_SUBDIR. +// Filenames are server-validated (see src/libs/replicaSchemas.ts:validateFilename). +export const uploadReplicaFile = async ( + file: File, + fileFullPath: string, + cfp: string, + onProgress?: ProgressHandler, +) => { + try { + const response = await fetchWithAuth(API_ENDPOINTS.upload, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + fileName: cfp, + fileSize: file.size, + temp: false, + }), + }); + + const { uploadUrl }: { uploadUrl: string } = await response.json(); + if (isWebAppPlatform()) { + await webUpload(file, uploadUrl, onProgress); + } else { + await tauriUpload(uploadUrl, fileFullPath, 'PUT', onProgress); + } + } catch (error) { + console.error('Replica file upload failed:', error); + if (error instanceof Error) throw error; + throw new Error('Replica file upload failed'); + } +}; + export const batchGetDownloadUrls = async (files: { lfp: string; cfp: string }[]) => { try { const userId = await getUserID(); diff --git a/apps/readest-app/src/pages/api/sync/replicas.ts b/apps/readest-app/src/pages/api/sync/replicas.ts new file mode 100644 index 00000000..5a8469c4 --- /dev/null +++ b/apps/readest-app/src/pages/api/sync/replicas.ts @@ -0,0 +1,146 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { NextRequest, NextResponse } from 'next/server'; +import { createSupabaseClient } from '@/utils/supabase'; +import { validateUserAndToken } from '@/utils/access'; +import { runMiddleware, corsAllMethods } from '@/utils/cors'; +import { validatePullParams, validatePushBatch } from '@/libs/replicaSyncServer'; +import type { ReplicaRow } from '@/types/replica'; + +const errorResponse = (status: number, code: string, message: string, offendingIndex?: number) => + NextResponse.json( + { + error: message, + code, + ...(typeof offendingIndex === 'number' ? { offendingIndex } : {}), + }, + { status }, + ); + +export async function POST(req: NextRequest) { + const { user, token } = await validateUserAndToken(req.headers.get('authorization')); + if (!user || !token) { + return errorResponse(401, 'AUTH', 'Not authenticated'); + } + const supabase = createSupabaseClient(token); + + let body: unknown; + try { + body = await req.json(); + } catch { + return errorResponse(400, 'VALIDATION', 'Invalid JSON body'); + } + + const validation = validatePushBatch(body, user.id, Date.now()); + if (!validation.ok) { + return errorResponse( + validation.status, + validation.code, + validation.message, + validation.offendingIndex, + ); + } + + const merged: ReplicaRow[] = []; + for (const row of validation.rows) { + const { data, error } = await supabase + .rpc('crdt_merge_replica', { + p_user_id: row.user_id, + p_kind: row.kind, + p_replica_id: row.replica_id, + p_fields_jsonb: row.fields_jsonb, + p_manifest_jsonb: row.manifest_jsonb, + p_deleted_at_ts: row.deleted_at_ts, + p_reincarnation: row.reincarnation, + p_updated_at_ts: row.updated_at_ts, + p_schema_version: row.schema_version, + }) + .single(); + + if (error) { + console.error('crdt_merge_replica failed', { row, error }); + return errorResponse(500, 'SERVER', error.message); + } + if (data) merged.push(data); + } + + return NextResponse.json({ rows: merged }, { status: 200 }); +} + +export async function GET(req: NextRequest) { + const { user, token } = await validateUserAndToken(req.headers.get('authorization')); + if (!user || !token) { + return errorResponse(401, 'AUTH', 'Not authenticated'); + } + const supabase = createSupabaseClient(token); + + const { searchParams } = new URL(req.url); + const validation = validatePullParams(searchParams.get('kind'), searchParams.get('since')); + if (!validation.ok) { + return errorResponse(validation.status, validation.code, validation.message); + } + const { kind, since } = validation.params; + + let query = supabase + .from('replicas') + .select('*') + .eq('user_id', user.id) + .eq('kind', kind) + .order('updated_at_ts', { ascending: true }) + .limit(1000); + + if (since) query = query.gt('updated_at_ts', since); + + const { data, error } = await query; + if (error) { + console.error('pull replicas failed', { kind, since, error }); + return errorResponse(500, 'SERVER', error.message); + } + + return NextResponse.json({ rows: data ?? [] }, { status: 200 }); +} + +const handler = async (req: NextApiRequest, res: NextApiResponse) => { + if (!req.url) { + return res.status(400).json({ error: 'Invalid request URL' }); + } + const protocol = process.env['PROTOCOL'] || 'http'; + const host = process.env['HOST'] || 'localhost:3000'; + const url = new URL(req.url, `${protocol}://${host}`); + + await runMiddleware(req, res, corsAllMethods); + + try { + let response: Response; + if (req.method === 'GET') { + const nextReq = new NextRequest(url.toString(), { + headers: new Headers(req.headers as Record), + method: 'GET', + }); + response = await GET(nextReq); + } else if (req.method === 'POST') { + const nextReq = new NextRequest(url.toString(), { + headers: new Headers(req.headers as Record), + method: 'POST', + body: JSON.stringify(req.body), + }); + response = await POST(nextReq); + } else { + res.setHeader('Allow', ['GET', 'POST']); + return res.status(405).json({ error: 'Method Not Allowed' }); + } + + res.status(response.status); + response.headers.forEach((value, key) => { + res.setHeader(key, value); + }); + + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + res.send(buffer); + } catch (error) { + console.error('Error processing /api/sync/replicas request:', error); + res.status(500).json({ error: 'Internal Server Error' }); + } +}; + +export default handler; diff --git a/apps/readest-app/src/services/appService.ts b/apps/readest-app/src/services/appService.ts index 8d9e866a..100679c0 100644 --- a/apps/readest-app/src/services/appService.ts +++ b/apps/readest-app/src/services/appService.ts @@ -278,6 +278,44 @@ export abstract class BaseAppService implements AppService { ); } + async uploadReplicaFile( + kind: string, + replicaId: string, + filename: string, + lfp: string, + base: BaseDir, + onProgress: ProgressHandler, + ) { + return CloudSvc.uploadReplicaFileToCloud(this.fs, this.resolveFilePath.bind(this), { + kind, + replicaId, + filename, + lfp, + base, + onProgress, + }); + } + + async downloadReplicaFile( + kind: string, + replicaId: string, + filename: string, + dst: string, + onProgress?: ProgressHandler, + ) { + return CloudSvc.downloadReplicaFileFromCloud(this, { + kind, + replicaId, + filename, + dst, + onProgress, + }); + } + + async deleteReplicaBundle(kind: string, replicaId: string, filenames: string[]) { + return CloudSvc.deleteReplicaBundleFromCloud(kind, replicaId, filenames); + } + async uploadBook(book: Book, onProgress?: ProgressHandler): Promise { return CloudSvc.uploadBook(this.fs, this.resolveFilePath.bind(this), book, onProgress); } diff --git a/apps/readest-app/src/services/cloudService.ts b/apps/readest-app/src/services/cloudService.ts index 4771a7a7..30b06ec8 100644 --- a/apps/readest-app/src/services/cloudService.ts +++ b/apps/readest-app/src/services/cloudService.ts @@ -9,13 +9,14 @@ import { import { downloadFile, uploadFile, + uploadReplicaFile, deleteFile as deleteCloudFile, createProgressHandler, batchGetDownloadUrls, } from '@/libs/storage'; import { ClosableFile } from '@/utils/file'; import { ProgressHandler } from '@/utils/transfer'; -import { CLOUD_BOOKS_SUBDIR } from './constants'; +import { CLOUD_BOOKS_SUBDIR, CLOUD_REPLICAS_SUBDIR } from './constants'; export async function deleteBook( fs: FileSystem, @@ -75,6 +76,71 @@ export async function uploadFileToCloud( return downloadUrl; } +// Upload a single replica binary to the cloud under +// CLOUD_REPLICAS_SUBDIR///. Filename is the +// caller-supplied logical name (server-validated; see replicaSchemas.ts). +export async function uploadReplicaFileToCloud( + fs: FileSystem, + resolveFilePath: (path: string, base: BaseDir) => Promise, + opts: { + kind: string; + replicaId: string; + filename: string; + lfp: string; + base: BaseDir; + onProgress: ProgressHandler; + }, +): Promise { + const cfp = `${CLOUD_REPLICAS_SUBDIR}/${opts.kind}/${opts.replicaId}/${opts.filename}`; + console.log('Uploading replica file:', opts.lfp, 'to', cfp); + const file = await fs.openFile(opts.lfp, opts.base, opts.filename); + const localFullpath = await resolveFilePath(opts.lfp, opts.base); + await uploadReplicaFile(file, localFullpath, cfp, opts.onProgress); + const f = file as ClosableFile; + if (f && f.close) { + await f.close(); + } +} + +// Cloud key for a replica binary. Centralized so adapters and the +// download path share the same path-construction rule. +export const replicaCloudKey = (kind: string, replicaId: string, filename: string): string => + `${CLOUD_REPLICAS_SUBDIR}/${kind}/${replicaId}/${filename}`; + +export async function downloadReplicaFileFromCloud( + appService: AppService, + opts: { + kind: string; + replicaId: string; + filename: string; + dst: string; + onProgress?: ProgressHandler; + }, +): Promise { + const cfp = replicaCloudKey(opts.kind, opts.replicaId, opts.filename); + await downloadFile({ + appService, + cfp, + dst: opts.dst, + onProgress: opts.onProgress, + }); +} + +export async function deleteReplicaBundleFromCloud( + kind: string, + replicaId: string, + filenames: string[], +): Promise { + for (const filename of filenames) { + const cfp = replicaCloudKey(kind, replicaId, filename); + try { + await deleteCloudFile(cfp); + } catch (error) { + console.log(`Failed to delete replica file ${cfp}:`, error); + } + } +} + export async function uploadBook( fs: FileSystem, resolveFilePath: (path: string, base: BaseDir) => Promise, diff --git a/apps/readest-app/src/services/constants.ts b/apps/readest-app/src/services/constants.ts index 42d9f503..dc568abd 100644 --- a/apps/readest-app/src/services/constants.ts +++ b/apps/readest-app/src/services/constants.ts @@ -33,6 +33,7 @@ import { DEFAULT_AI_SETTINGS } from './ai/constants'; export const DATA_SUBDIR = 'Readest'; export const LOCAL_BOOKS_SUBDIR = `${DATA_SUBDIR}/Books`; export const CLOUD_BOOKS_SUBDIR = `${DATA_SUBDIR}/Books`; +export const CLOUD_REPLICAS_SUBDIR = `${DATA_SUBDIR}/Replicas`; export const LOCAL_FONTS_SUBDIR = `${DATA_SUBDIR}/Fonts`; export const LOCAL_IMAGES_SUBDIR = `${DATA_SUBDIR}/Images`; export const LOCAL_DICTIONARIES_SUBDIR = `${DATA_SUBDIR}/Dictionaries`; @@ -127,6 +128,13 @@ export const DEFAULT_SYSTEM_SETTINGS: Partial = { lastSyncedAtBooks: 0, lastSyncedAtConfigs: 0, lastSyncedAtNotes: 0, + lastSyncedAtReplicas: {}, + syncCategories: { + book: true, + progress: true, + note: true, + dictionary: true, + }, }; export const DEFAULT_MOBILE_SYSTEM_SETTINGS: Partial = { diff --git a/apps/readest-app/src/services/dictionaries/contentId.ts b/apps/readest-app/src/services/dictionaries/contentId.ts new file mode 100644 index 00000000..4c14a5f8 --- /dev/null +++ b/apps/readest-app/src/services/dictionaries/contentId.ts @@ -0,0 +1,24 @@ +import { partialMD5 } from '@/utils/md5'; +import { computeDictionaryReplicaId } from '@/services/sync/adapters/dictionary'; + +/** + * Compute the cross-device content-hash id for a dictionary bundle at + * import time. Uses partialMD5 (head + middle + tail sample) of the + * primary file, mixed with byteSize and the sorted filename list. The + * mixing makes the id resilient to the partialMD5 collision risk + * (Codex flagged 9/10 confidence; eng review accepted the mitigation). + * + * Stardict primary = .ifo (small text; partialMD5 is effectively full-hash). + * MDict primary = .mdx (body). + * DICT primary = .dict.dz (compressed body). + * Slob primary = .slob (single-file bundle). + * + * Same content → same id across devices, enabling per-record sync identity. + */ +export const computeDictionaryContentId = async ( + primary: File, + filenames: string[], +): Promise => { + const partial = await partialMD5(primary); + return computeDictionaryReplicaId(partial, primary.size, filenames); +}; diff --git a/apps/readest-app/src/services/dictionaries/dictionaryService.ts b/apps/readest-app/src/services/dictionaries/dictionaryService.ts index 5611e618..503b5700 100644 --- a/apps/readest-app/src/services/dictionaries/dictionaryService.ts +++ b/apps/readest-app/src/services/dictionaries/dictionaryService.ts @@ -15,6 +15,7 @@ import { uniqueId } from '@/utils/misc'; import { getFilename } from '@/utils/path'; import type { ImportedDictionary } from './types'; import { scanEntryOffsets, serializeOffsetsSidecar } from './stardictReader'; +import { computeDictionaryContentId } from './contentId'; /** GZIP magic bytes — used to detect DictZip-compressed `.dict` files. */ const GZIP_MAGIC = [0x1f, 0x8b, 0x08]; @@ -277,8 +278,14 @@ async function importStarDictBundle( } } + // Stardict primary = .ifo (small text; partialMD5 is effectively full-hash). + const stardictFilenames = [group.ifo.name, group.idx.name, group.dict.name]; + if (group.syn?.name) stardictFilenames.push(group.syn.name); + const contentId = await computeDictionaryContentId(ifoFile, stardictFilenames); + return { id: bundleDir, + contentId, kind: 'stardict', name, bundleDir, @@ -348,8 +355,17 @@ async function importMdictBundle(fs: FileSystem, group: MDictGroup): Promise m.name), + ...group.css.map((c) => c.name), + ]; + const contentId = await computeDictionaryContentId(mdxFile, mdictFilenames); + return { id: bundleDir, + contentId, kind: 'mdict', name, bundleDir, @@ -419,8 +435,13 @@ async function importDictBundle(fs: FileSystem, group: DictGroup): Promise { + switch (d.kind) { + case 'mdict': + return d.files.mdx ?? null; + case 'stardict': + return d.files.ifo ?? null; + case 'dict': + return d.files.dict ?? null; + case 'slob': + return d.files.slob ?? null; + default: + return null; + } +}; + +export const enumerateDictionaryFiles = ( + d: ImportedDictionary, +): { logical: string; lfp: string; byteSize: number }[] => { + const out: { logical: string; lfp: string; byteSize: number }[] = []; + const push = (filename?: string) => { + if (!filename) return; + out.push({ + logical: filename, + lfp: `${d.bundleDir}/${filename}`, + byteSize: 0, + }); + }; + switch (d.kind) { + case 'mdict': + push(d.files.mdx); + d.files.mdd?.forEach(push); + d.files.css?.forEach(push); + break; + case 'stardict': + push(d.files.ifo); + push(d.files.idx); + push(d.files.dict); + push(d.files.syn); + break; + case 'dict': + push(d.files.dict); + push(d.files.index); + break; + case 'slob': + push(d.files.slob); + break; + } + return out; +}; + +export const computeDictionaryReplicaId = ( + partialMd5: string, + byteSize: number, + filenames: string[], +): string => { + const sortedFilenames = [...filenames].sort(); + return md5(`${partialMd5}|${byteSize}|${sortedFilenames.join(',')}`); +}; + +export const dictionaryAdapter: ReplicaAdapter = { + kind: DICTIONARY_KIND, + schemaVersion: DICTIONARY_SCHEMA_VERSION, + + pack(d: ImportedDictionary): Record { + const fields: Record = { + name: d.name, + kind: d.kind, + addedAt: d.addedAt, + }; + if (d.lang !== undefined) fields['lang'] = d.lang; + if (d.unsupported) fields['unsupported'] = true; + if (d.unsupportedReason) fields['unsupportedReason'] = d.unsupportedReason; + return fields; + }, + + unpack(fields: Record): ImportedDictionary { + return { + id: '', + kind: fields['kind'] as ImportedDictionary['kind'], + name: String(fields['name'] ?? ''), + bundleDir: '', + files: {}, + lang: fields['lang'] !== undefined ? String(fields['lang']) : undefined, + addedAt: Number(fields['addedAt'] ?? 0), + unsupported: fields['unsupported'] === true ? true : undefined, + unsupportedReason: + fields['unsupportedReason'] !== undefined ? String(fields['unsupportedReason']) : undefined, + }; + }, + + async computeId(d: ImportedDictionary): Promise { + return d.id; + }, + + binary: { + localBaseDir: 'Dictionaries', + enumerateFiles: enumerateDictionaryFiles, + }, +}; diff --git a/apps/readest-app/src/services/sync/replicaBootstrap.ts b/apps/readest-app/src/services/sync/replicaBootstrap.ts new file mode 100644 index 00000000..b3c4bc31 --- /dev/null +++ b/apps/readest-app/src/services/sync/replicaBootstrap.ts @@ -0,0 +1,22 @@ +import { dictionaryAdapter } from './adapters/dictionary'; +import { getReplicaAdapter, registerReplicaAdapter } from './replicaRegistry'; +import type { ReplicaAdapter } from './replicaRegistry'; + +const KNOWN_ADAPTERS: ReplicaAdapter[] = [ + dictionaryAdapter as unknown as ReplicaAdapter, +]; + +let didBootstrap = false; + +export const bootstrapReplicaAdapters = (): void => { + if (didBootstrap) return; + for (const adapter of KNOWN_ADAPTERS) { + if (getReplicaAdapter(adapter.kind)) continue; + registerReplicaAdapter(adapter); + } + didBootstrap = true; +}; + +export const __resetBootstrapForTests = (): void => { + didBootstrap = false; +}; diff --git a/apps/readest-app/src/services/sync/replicaRegistry.ts b/apps/readest-app/src/services/sync/replicaRegistry.ts new file mode 100644 index 00000000..7af32f61 --- /dev/null +++ b/apps/readest-app/src/services/sync/replicaRegistry.ts @@ -0,0 +1,39 @@ +import type { BaseDir, FileSystem } from '@/types/system'; + +export interface BinaryCapability { + localBaseDir: BaseDir; + enumerateFiles(replica: T): { logical: string; lfp: string; byteSize: number }[]; +} + +export interface LifecycleHooks { + postDownload?(replica: T, fs: FileSystem): Promise; + validateOnLoad?(replica: T, fs: FileSystem): Promise<{ unavailable?: boolean }>; +} + +export interface ReplicaAdapter { + kind: string; + schemaVersion: number; + pack(replica: T): Record; + unpack(fields: Record): T; + computeId(input: T): Promise; + binary?: BinaryCapability; + lifecycle?: LifecycleHooks; +} + +const registry = new Map>(); + +export const registerReplicaAdapter = (adapter: ReplicaAdapter): void => { + if (registry.has(adapter.kind)) { + throw new Error(`Replica adapter for kind="${adapter.kind}" is already registered`); + } + registry.set(adapter.kind, adapter as ReplicaAdapter); +}; + +export const getReplicaAdapter = (kind: string): ReplicaAdapter | undefined => + registry.get(kind) as ReplicaAdapter | undefined; + +export const listReplicaAdapters = (): ReplicaAdapter[] => Array.from(registry.values()); + +export const clearReplicaAdapters = (): void => { + registry.clear(); +}; diff --git a/apps/readest-app/src/services/sync/replicaSync.ts b/apps/readest-app/src/services/sync/replicaSync.ts new file mode 100644 index 00000000..8e8c17ed --- /dev/null +++ b/apps/readest-app/src/services/sync/replicaSync.ts @@ -0,0 +1,67 @@ +import { HlcGenerator } from '@/libs/crdt'; +import { LocalStorageHlcStore, type HlcSnapshotStore } from '@/libs/hlcStore'; +import { ReplicaSyncClient } from '@/libs/replicaSyncClient'; +import { ReplicaSyncManager, type CursorStore } from './replicaSyncManager'; + +export interface ReplicaSyncInitOpts { + deviceId: string; + cursorStore: CursorStore; + hlcStore?: HlcSnapshotStore; + client?: Pick; +} + +export interface ReplicaSyncContext { + manager: ReplicaSyncManager; + hlc: HlcGenerator; + deviceId: string; +} + +let instance: ReplicaSyncContext | null = null; + +const wrapHlcWithPersistence = (hlc: HlcGenerator, hlcStore: HlcSnapshotStore): HlcGenerator => { + const originalNext = hlc.next.bind(hlc); + const originalObserve = hlc.observe.bind(hlc); + hlc.next = () => { + const v = originalNext(); + hlcStore.save(hlc.serialize()); + return v; + }; + hlc.observe = (remote) => { + originalObserve(remote); + hlcStore.save(hlc.serialize()); + }; + return hlc; +}; + +export const initReplicaSync = (opts: ReplicaSyncInitOpts): ReplicaSyncContext => { + if (instance) return instance; + + const hlcStore = opts.hlcStore ?? new LocalStorageHlcStore(); + const snapshot = hlcStore.load(); + const baseHlc = snapshot + ? HlcGenerator.restore(snapshot, opts.deviceId) + : new HlcGenerator(opts.deviceId); + const hlc = wrapHlcWithPersistence(baseHlc, hlcStore); + + const client = opts.client ?? new ReplicaSyncClient(); + + const manager = new ReplicaSyncManager({ + hlc, + client, + cursorStore: opts.cursorStore, + }); + + instance = { manager, hlc, deviceId: opts.deviceId }; + return instance; +}; + +export const getReplicaSync = (): ReplicaSyncContext | null => instance; + +export const isReplicaSyncReady = (): boolean => instance !== null; + +export const __resetReplicaSyncForTests = (): void => { + if (instance) { + instance.manager.stopAutoSync(); + } + instance = null; +}; diff --git a/apps/readest-app/src/services/sync/replicaSyncManager.ts b/apps/readest-app/src/services/sync/replicaSyncManager.ts new file mode 100644 index 00000000..7c61f411 --- /dev/null +++ b/apps/readest-app/src/services/sync/replicaSyncManager.ts @@ -0,0 +1,122 @@ +import { HlcGenerator, hlcCompare } from '@/libs/crdt'; +import type { Hlc, ReplicaRow } from '@/types/replica'; +import type { ReplicaSyncClient } from '@/libs/replicaSyncClient'; + +export interface CursorStore { + get(kind: string): Hlc | null; + set(kind: string, hlc: Hlc): void; +} + +export interface ReplicaSyncManagerOpts { + hlc: HlcGenerator; + client: Pick; + cursorStore: CursorStore; + debounceMs?: number; +} + +interface DirtyKey { + kind: string; + replicaId: string; +} + +const dirtyKeyOf = (row: ReplicaRow): string => `${row.kind}::${row.replica_id}`; +const splitKey = (k: string): DirtyKey => { + const idx = k.indexOf('::'); + return { kind: k.slice(0, idx), replicaId: k.slice(idx + 2) }; +}; + +export class ReplicaSyncManager { + private readonly dirty = new Map(); + private readonly debounceMs: number; + private debounceTimer: ReturnType | null = null; + private autoSyncInstalled = false; + private readonly visibilityHandler = () => { + if (typeof document !== 'undefined' && document.visibilityState === 'hidden') { + void this.flush().catch((e) => console.warn('replica sync flush on hide failed', e)); + } + }; + private readonly onlineHandler = () => { + void this.flush().catch((e) => console.warn('replica sync flush on online failed', e)); + }; + + constructor(private readonly opts: ReplicaSyncManagerOpts) { + this.debounceMs = opts.debounceMs ?? 5000; + } + + markDirty(row: ReplicaRow): void { + this.dirty.set(dirtyKeyOf(row), row); + this.scheduleDebouncedFlush(); + } + + private scheduleDebouncedFlush(): void { + if (this.debounceTimer !== null) clearTimeout(this.debounceTimer); + this.debounceTimer = setTimeout(() => { + this.debounceTimer = null; + void this.flush().catch((e) => console.warn('replica sync debounced flush failed', e)); + }, this.debounceMs); + } + + async flush(): Promise { + if (this.debounceTimer !== null) { + clearTimeout(this.debounceTimer); + this.debounceTimer = null; + } + if (this.dirty.size === 0) return; + const snapshot = Array.from(this.dirty.values()); + const snapshotKeys = Array.from(this.dirty.keys()); + try { + await this.opts.client.push(snapshot); + for (const k of snapshotKeys) { + const stillSame = this.dirty.get(k); + if (stillSame === snapshot[snapshotKeys.indexOf(k)]) { + this.dirty.delete(k); + } + } + } catch (err) { + throw err; + } + } + + async pull(kind: string): Promise { + const since = this.opts.cursorStore.get(kind); + const rows = await this.opts.client.pull(kind, since); + if (rows.length === 0) return rows; + let maxHlc: Hlc = rows[0]!.updated_at_ts; + for (const row of rows) { + if (hlcCompare(row.updated_at_ts, maxHlc) > 0) maxHlc = row.updated_at_ts; + this.opts.hlc.observe(row.updated_at_ts); + } + this.opts.cursorStore.set(kind, maxHlc); + return rows; + } + + startAutoSync(): void { + if (this.autoSyncInstalled) return; + if (typeof document !== 'undefined') { + document.addEventListener('visibilitychange', this.visibilityHandler); + } + if (typeof window !== 'undefined') { + window.addEventListener('online', this.onlineHandler); + } + this.autoSyncInstalled = true; + } + + stopAutoSync(): void { + if (!this.autoSyncInstalled) return; + if (typeof document !== 'undefined') { + document.removeEventListener('visibilitychange', this.visibilityHandler); + } + if (typeof window !== 'undefined') { + window.removeEventListener('online', this.onlineHandler); + } + this.autoSyncInstalled = false; + } + + pendingCount(): number { + return this.dirty.size; + } + + pendingKeys(): DirtyKey[] { + return Array.from(this.dirty.keys()).map(splitKey); + } +} diff --git a/apps/readest-app/src/services/transferManager.ts b/apps/readest-app/src/services/transferManager.ts index 3c1e4ff6..c4654f6d 100644 --- a/apps/readest-app/src/services/transferManager.ts +++ b/apps/readest-app/src/services/transferManager.ts @@ -1,8 +1,8 @@ import { Book } from '@/types/book'; -import { AppService } from '@/types/system'; -import { useTransferStore, TransferItem } from '@/store/transferStore'; +import { AppService, BaseDir } from '@/types/system'; +import { useTransferStore, TransferItem, ReplicaTransferFile } from '@/store/transferStore'; import { TranslationFunc } from '@/hooks/useTranslation'; -import { ProgressPayload } from '@/utils/transfer'; +import { ProgressHandler, ProgressPayload } from '@/utils/transfer'; import { eventDispatcher } from '@/utils/event'; const TRANSFER_QUEUE_KEY = 'readest_transfer_queue'; @@ -119,6 +119,85 @@ class TransferManager { .filter((id): id is string => id !== null); } + queueReplicaUpload( + replicaKind: string, + replicaId: string, + displayTitle: string, + files: ReplicaTransferFile[], + base: BaseDir, + opts: { priority?: number; isBackground?: boolean } = {}, + ): string | null { + if (!this.isReady()) { + console.warn('TransferManager not initialized'); + return null; + } + const store = useTransferStore.getState(); + const existing = store.getReplicaTransfer(replicaKind, replicaId, 'upload'); + if (existing) return existing.id; + + const id = store.addReplicaTransfer(replicaKind, replicaId, displayTitle, 'upload', { + priority: opts.priority, + isBackground: opts.isBackground, + files, + base, + }); + this.persistQueue(); + this.processQueue(); + return id; + } + + queueReplicaDownload( + replicaKind: string, + replicaId: string, + displayTitle: string, + files: ReplicaTransferFile[], + base: BaseDir, + opts: { priority?: number; isBackground?: boolean } = {}, + ): string | null { + if (!this.isReady()) { + console.warn('TransferManager not initialized'); + return null; + } + const store = useTransferStore.getState(); + const existing = store.getReplicaTransfer(replicaKind, replicaId, 'download'); + if (existing) return existing.id; + + const id = store.addReplicaTransfer(replicaKind, replicaId, displayTitle, 'download', { + priority: opts.priority, + isBackground: opts.isBackground, + files, + base, + }); + this.persistQueue(); + this.processQueue(); + return id; + } + + queueReplicaDelete( + replicaKind: string, + replicaId: string, + displayTitle: string, + filenames: string[], + opts: { priority?: number; isBackground?: boolean } = {}, + ): string | null { + if (!this.isReady()) { + console.warn('TransferManager not initialized'); + return null; + } + const store = useTransferStore.getState(); + const existing = store.getReplicaTransfer(replicaKind, replicaId, 'delete'); + if (existing) return existing.id; + + const id = store.addReplicaTransfer(replicaKind, replicaId, displayTitle, 'delete', { + priority: opts.priority, + isBackground: opts.isBackground, + files: filenames.map((logical) => ({ logical, lfp: '', byteSize: 0 })), + }); + this.persistQueue(); + this.processQueue(); + return id; + } + cancelTransfer(transferId: string): void { const controller = this.abortControllers.get(transferId); if (controller) { @@ -230,24 +309,10 @@ class TransferManager { }; try { - const library = this.getLibrary(); - const book = library.find((b) => b.hash === transfer.bookHash); - - if (!book) { - throw new Error(_('Book not found in library')); - } - - if (transfer.type === 'upload') { - await this.appService.uploadBook(book, progressHandler); - book.uploadedAt = Date.now(); - await this.updateBook(book); - } else if (transfer.type === 'download') { - await this.appService.downloadBook(book, false, false, progressHandler); - book.downloadedAt = Date.now(); - await this.updateBook(book); - } else if (transfer.type === 'delete') { - await this.appService.deleteBook(book, 'cloud'); - await this.updateBook(book); + if (transfer.kind === 'replica') { + await this.executeReplicaTransfer(transfer, progressHandler, abortController); + } else { + await this.executeBookTransfer(transfer, progressHandler, abortController); } useTransferStore.getState().setTransferStatus(transfer.id, 'completed'); @@ -328,6 +393,118 @@ class TransferManager { } } + private async executeBookTransfer( + transfer: TransferItem, + progressHandler: (p: ProgressPayload) => void, + _abortController: AbortController, + ): Promise { + const _ = this._!; + const library = this.getLibrary!(); + const book = library.find((b) => b.hash === transfer.bookHash); + + if (!book) { + throw new Error(_('Book not found in library')); + } + + if (transfer.type === 'upload') { + await this.appService!.uploadBook(book, progressHandler); + book.uploadedAt = Date.now(); + await this.updateBook!(book); + } else if (transfer.type === 'download') { + await this.appService!.downloadBook(book, false, false, progressHandler); + book.downloadedAt = Date.now(); + await this.updateBook!(book); + } else if (transfer.type === 'delete') { + await this.appService!.deleteBook(book, 'cloud'); + await this.updateBook!(book); + } + } + + private async executeReplicaTransfer( + transfer: TransferItem, + progressHandler: (p: ProgressPayload) => void, + _abortController: AbortController, + ): Promise { + const kind = transfer.replicaKind!; + const replicaId = transfer.replicaId!; + const files = transfer.replicaFiles ?? []; + + if (transfer.type === 'delete') { + await this.appService!.deleteReplicaBundle( + kind, + replicaId, + files.map((f) => f.logical), + ); + eventDispatcher.dispatch('replica-transfer-complete', { + kind, + replicaId, + type: 'delete', + filenames: files.map((f) => f.logical), + }); + return; + } + + if (files.length === 0) { + throw new Error(`replica ${transfer.type} requires replicaFiles on the transfer`); + } + + const totalBytes = files.reduce((sum, f) => sum + f.byteSize, 0) || 1; + let bytesAlreadyDone = 0; + const fileProgressHandler = + (filenameSize: number): ProgressHandler => + (p: ProgressPayload) => { + const fileFraction = p.total > 0 ? p.progress / p.total : 0; + const overallTransferred = bytesAlreadyDone + filenameSize * fileFraction; + progressHandler({ + progress: overallTransferred, + total: totalBytes, + transferSpeed: p.transferSpeed, + }); + }; + + if (transfer.type === 'upload') { + const base = transfer.replicaBase!; + for (const file of files) { + await this.appService!.uploadReplicaFile( + kind, + replicaId, + file.logical, + file.lfp, + base, + fileProgressHandler(file.byteSize), + ); + bytesAlreadyDone += file.byteSize; + } + eventDispatcher.dispatch('replica-transfer-complete', { + kind, + replicaId, + type: 'upload', + files, + }); + return; + } + + if (transfer.type === 'download') { + for (const file of files) { + await this.appService!.downloadReplicaFile( + kind, + replicaId, + file.logical, + file.lfp, + fileProgressHandler(file.byteSize), + ); + bytesAlreadyDone += file.byteSize; + } + eventDispatcher.dispatch('replica-transfer-complete', { + kind, + replicaId, + type: 'download', + files, + }); + return; + } + } + private async loadPersistedQueue(): Promise { try { if (typeof localStorage === 'undefined') return; diff --git a/apps/readest-app/src/store/transferStore.ts b/apps/readest-app/src/store/transferStore.ts index e546f609..690ea4e7 100644 --- a/apps/readest-app/src/store/transferStore.ts +++ b/apps/readest-app/src/store/transferStore.ts @@ -1,12 +1,25 @@ import { create } from 'zustand'; +import type { BaseDir } from '@/types/system'; export type TransferType = 'upload' | 'download' | 'delete'; export type TransferStatus = 'pending' | 'in_progress' | 'completed' | 'failed' | 'cancelled'; +export type TransferKind = 'book' | 'replica'; + +export interface ReplicaTransferFile { + logical: string; + lfp: string; + byteSize: number; +} export interface TransferItem { id: string; + kind: TransferKind; bookHash: string; bookTitle: string; + replicaKind?: string; + replicaId?: string; + replicaFiles?: ReplicaTransferFile[]; + replicaBase?: BaseDir; type: TransferType; status: TransferStatus; progress: number; // 0-100 percentage @@ -41,6 +54,18 @@ interface TransferState { priority?: number, isBackground?: boolean, ) => string; + addReplicaTransfer: ( + replicaKind: string, + replicaId: string, + displayTitle: string, + type: TransferType, + opts?: { + priority?: number; + isBackground?: boolean; + files?: ReplicaTransferFile[]; + base?: BaseDir; + }, + ) => string; removeTransfer: (transferId: string) => void; updateTransferProgress: ( transferId: string, @@ -66,6 +91,11 @@ interface TransferState { getFailedTransfers: () => TransferItem[]; getCompletedTransfers: () => TransferItem[]; getTransferByBookHash: (bookHash: string, type: TransferType) => TransferItem | undefined; + getReplicaTransfer: ( + replicaKind: string, + replicaId: string, + type: TransferType, + ) => TransferItem | undefined; getQueueStats: () => { pending: number; active: number; @@ -98,6 +128,7 @@ export const useTransferStore = create((set, get) => ({ const id = generateTransferId(); const transfer: TransferItem = { id, + kind: 'book', bookHash, bookTitle, type, @@ -120,6 +151,37 @@ export const useTransferStore = create((set, get) => ({ return id; }, + addReplicaTransfer: (replicaKind, replicaId, displayTitle, type, opts = {}) => { + const id = generateTransferId(); + const transfer: TransferItem = { + id, + kind: 'replica', + bookHash: '', + bookTitle: displayTitle, + replicaKind, + replicaId, + replicaFiles: opts.files, + replicaBase: opts.base, + type, + status: 'pending', + progress: 0, + totalBytes: opts.files?.reduce((sum, f) => sum + f.byteSize, 0) ?? 0, + transferredBytes: 0, + transferSpeed: 0, + retryCount: 0, + maxRetries: 3, + createdAt: Date.now(), + priority: opts.priority ?? 10, + isBackground: opts.isBackground ?? false, + }; + + set((state) => ({ + transfers: { ...state.transfers, [id]: transfer }, + })); + + return id; + }, + removeTransfer: (transferId) => { set((state) => { const { [transferId]: _, ...remaining } = state.transfers; @@ -261,7 +323,21 @@ export const useTransferStore = create((set, get) => ({ getTransferByBookHash: (bookHash, type) => { return Object.values(get().transfers).find( (t) => - t.bookHash === bookHash && t.type === type && ['pending', 'in_progress'].includes(t.status), + t.kind === 'book' && + t.bookHash === bookHash && + t.type === type && + ['pending', 'in_progress'].includes(t.status), + ); + }, + + getReplicaTransfer: (replicaKind, replicaId, type) => { + return Object.values(get().transfers).find( + (t) => + t.kind === 'replica' && + t.replicaKind === replicaKind && + t.replicaId === replicaId && + t.type === type && + ['pending', 'in_progress'].includes(t.status), ); }, @@ -279,19 +355,20 @@ export const useTransferStore = create((set, get) => ({ setActiveCount: (count) => set({ activeCount: count }), restoreTransfers: (transfers, isQueuePaused) => { - // Reset in_progress transfers to pending when restoring + // Legacy rows persisted before the kind discriminator default to 'book'. const restoredTransfers: Record = {}; Object.entries(transfers).forEach(([id, transfer]) => { - if (transfer.status === 'in_progress') { + const withKind: TransferItem = { ...transfer, kind: transfer.kind ?? 'book' }; + if (withKind.status === 'in_progress') { restoredTransfers[id] = { - ...transfer, + ...withKind, status: 'pending', progress: 0, transferredBytes: 0, transferSpeed: 0, }; } else { - restoredTransfers[id] = transfer; + restoredTransfers[id] = withKind; } }); set({ transfers: restoredTransfers, isQueuePaused }); diff --git a/apps/readest-app/src/types/replica.ts b/apps/readest-app/src/types/replica.ts new file mode 100644 index 00000000..456226f4 --- /dev/null +++ b/apps/readest-app/src/types/replica.ts @@ -0,0 +1,54 @@ +/** + * Branded HLC string. Lexicographic comparison matches temporal order. + * Format: `${physicalMs:13-hex}-${counter:8-hex}-${deviceId}` + */ +export type Hlc = string & { readonly __brand: 'Hlc' }; + +export interface FieldEnvelope { + v: V; + t: Hlc; + s: string; +} + +export type FieldsObject = Record; + +export interface ManifestFile { + filename: string; + byteSize: number; + partialMd5: string; + mtime?: number; +} + +export interface Manifest { + files: ManifestFile[]; + schemaVersion: number; +} + +export interface ReplicaRow { + user_id: string; + kind: string; + replica_id: string; + fields_jsonb: FieldsObject; + manifest_jsonb: Manifest | null; + deleted_at_ts: Hlc | null; + reincarnation: string | null; + updated_at_ts: Hlc; + schema_version: number; +} + +export interface CipherEnvelope { + c: string; + i: string; + s: string; + alg: string; + h: string; +} + +export const isCipherEnvelope = (v: unknown): v is CipherEnvelope => + typeof v === 'object' && + v !== null && + typeof (v as CipherEnvelope).c === 'string' && + typeof (v as CipherEnvelope).i === 'string' && + typeof (v as CipherEnvelope).s === 'string' && + typeof (v as CipherEnvelope).alg === 'string' && + typeof (v as CipherEnvelope).h === 'string'; diff --git a/apps/readest-app/src/types/settings.ts b/apps/readest-app/src/types/settings.ts index 3b878d49..77d1f8b8 100644 --- a/apps/readest-app/src/types/settings.ts +++ b/apps/readest-app/src/types/settings.ts @@ -79,6 +79,21 @@ export interface HardcoverSettings { lastSyncedAt: number; } +/** + * User-facing sync categories. 'progress' gates the existing book-config + * (reading progress) sync, 'note' gates annotations, 'book' gates book + * binaries + metadata, 'dictionary' gates the imported-dictionary replica + * sync. Adding a new replica kind extends this union. + */ +export type SyncCategory = 'book' | 'progress' | 'note' | 'dictionary'; + +export const SYNC_CATEGORIES: readonly SyncCategory[] = [ + 'book', + 'progress', + 'note', + 'dictionary', +] as const; + export interface SystemSettings { version: number; localBooksDir: string; @@ -124,6 +139,19 @@ export interface SystemSettings { lastSyncedAtBooks: number; lastSyncedAtConfigs: number; lastSyncedAtNotes: number; + /** + * Per-kind cursor for replica sync. Stores the HLC string of the last + * pulled row per kind. Absent kinds pull from the beginning. + */ + lastSyncedAtReplicas?: Record; + /** + * Per-category sync toggles. Missing keys default to ON. The + * 'progress' category gates the existing book-config (reading + * progress) sync; 'note' gates annotation sync; 'book' gates book + * binary + metadata sync; 'dictionary' gates the imported-dictionary + * replica sync. Future replica kinds add new SyncCategory members. + */ + syncCategories?: Partial>; migrationVersion: number; diff --git a/apps/readest-app/src/types/system.ts b/apps/readest-app/src/types/system.ts index 3da2fa9b..37f5f37a 100644 --- a/apps/readest-app/src/types/system.ts +++ b/apps/readest-app/src/types/system.ts @@ -154,6 +154,22 @@ export interface AppService { hash: string, temp?: boolean, ): Promise; + uploadReplicaFile( + kind: string, + replicaId: string, + filename: string, + lfp: string, + base: BaseDir, + onProgress: ProgressHandler, + ): Promise; + downloadReplicaFile( + kind: string, + replicaId: string, + filename: string, + dst: string, + onProgress?: ProgressHandler, + ): Promise; + deleteReplicaBundle(kind: string, replicaId: string, filenames: string[]): Promise; downloadBookCovers(books: Book[], redownload?: boolean): Promise; exportBook(book: Book): Promise; isBookAvailable(book: Book): Promise; diff --git a/docker/volumes/db/migrations/003_add_replicas.sql b/docker/volumes/db/migrations/003_add_replicas.sql new file mode 100644 index 00000000..c6161425 --- /dev/null +++ b/docker/volumes/db/migrations/003_add_replicas.sql @@ -0,0 +1,84 @@ +-- Migration 003: Add replicas table for cross-device sync of user-imported assets +-- (dictionaries, fonts, textures, OPDS catalogs, dict settings — gated by +-- a per-kind allowlist enforced both in DB CHECK and in server validation). +-- See ~/.claude/plans/vivid-orbiting-thimble.md and +-- src/libs/crdt.README.md for the design. + +-- ───────────────────────────────────────────────────────────────────────── +-- replica_keys: per-account PBKDF2 salt for the encrypted-fields envelope. +-- One row per (user_id, alg). A passphrase rotation appends a new row; +-- forgot-passphrase deletes all rows for the user (and the migrate-time +-- check on encrypted envelopes makes them unreadable client-side). +-- ───────────────────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS public.replica_keys ( + user_id uuid NOT NULL, + salt_id text NOT NULL, + alg text NOT NULL, + salt bytea NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT now(), + CONSTRAINT replica_keys_pkey PRIMARY KEY (user_id, salt_id), + CONSTRAINT replica_keys_user_id_fkey FOREIGN KEY (user_id) REFERENCES auth.users (id) ON DELETE CASCADE +); + +ALTER TABLE public.replica_keys ENABLE ROW LEVEL SECURITY; + +CREATE POLICY replica_keys_select ON public.replica_keys + FOR SELECT TO authenticated USING ((SELECT auth.uid()) = user_id); +CREATE POLICY replica_keys_insert ON public.replica_keys + FOR INSERT TO authenticated WITH CHECK ((SELECT auth.uid()) = user_id); +CREATE POLICY replica_keys_delete ON public.replica_keys + FOR DELETE TO authenticated USING ((SELECT auth.uid()) = user_id); + +-- ───────────────────────────────────────────────────────────────────────── +-- replicas: polymorphic per-user CRDT-backed metadata. +-- kind — server-allowlisted: 'dictionary' in PR 1; future kinds +-- require a server release that updates the CHECK below. +-- fields_jsonb — per-field LWW envelope: {: {v, t: , s}} +-- PR-validated 64 KiB / 64-field caps server-side. +-- manifest_jsonb — committed last after binary upload completes. +-- null = "binaries pending"; row not yet downloadable. +-- deleted_at_ts — remove-wins tombstone HLC. A field write does NOT +-- revive a tombstoned row. +-- reincarnation — explicit re-import token; swaps row to alive under a +-- new logical identity. +-- updated_at_ts — max(field HLCs, deleted_at_ts). Used as the pull +-- cursor. +-- schema_version — per-kind schema bump; server enforces bounds. +-- ───────────────────────────────────────────────────────────────────────── +CREATE TABLE IF NOT EXISTS public.replicas ( + user_id uuid NOT NULL, + kind text NOT NULL, + replica_id text NOT NULL, + fields_jsonb jsonb NOT NULL DEFAULT '{}'::jsonb, + manifest_jsonb jsonb NULL, + deleted_at_ts text NULL, + reincarnation text NULL, + updated_at_ts text NOT NULL, + schema_version integer NOT NULL DEFAULT 1, + created_at timestamp with time zone NOT NULL DEFAULT now(), + modified_at timestamp with time zone NOT NULL DEFAULT now(), + CONSTRAINT replicas_pkey PRIMARY KEY (user_id, kind, replica_id), + CONSTRAINT replicas_user_id_fkey FOREIGN KEY (user_id) REFERENCES auth.users (id) ON DELETE CASCADE, + -- Server allowlist for kind. Adding a new kind = a coordinated client + + -- server PR; this CHECK is part of the server release. + CONSTRAINT replicas_kind_allowlist CHECK (kind IN ('dictionary')), + -- Hard caps. validateRow() in src/libs/replica-schemas.ts enforces the + -- same bounds at the API layer with a clearer error code; these CHECKs + -- are belt-and-suspenders for direct DB writes. + CONSTRAINT replicas_fields_size CHECK (pg_column_size(fields_jsonb) <= 65536), + CONSTRAINT replicas_schema_version CHECK (schema_version >= 1 AND schema_version <= 1000) +); + +CREATE INDEX IF NOT EXISTS idx_replicas_pull_cursor + ON public.replicas (user_id, kind, updated_at_ts); + +ALTER TABLE public.replicas ENABLE ROW LEVEL SECURITY; + +CREATE POLICY replicas_select ON public.replicas + FOR SELECT TO authenticated USING ((SELECT auth.uid()) = user_id); +CREATE POLICY replicas_insert ON public.replicas + FOR INSERT TO authenticated WITH CHECK ((SELECT auth.uid()) = user_id); +CREATE POLICY replicas_update ON public.replicas + FOR UPDATE TO authenticated USING ((SELECT auth.uid()) = user_id); +CREATE POLICY replicas_delete ON public.replicas + FOR DELETE TO authenticated USING ((SELECT auth.uid()) = user_id); diff --git a/docker/volumes/db/migrations/004_crdt_merge_replica_fn.sql b/docker/volumes/db/migrations/004_crdt_merge_replica_fn.sql new file mode 100644 index 00000000..5fb9a001 --- /dev/null +++ b/docker/volumes/db/migrations/004_crdt_merge_replica_fn.sql @@ -0,0 +1,189 @@ +-- Migration 004: crdt_merge_replica() — atomic CRDT merge for the replicas +-- table. Mirrors src/libs/crdt.ts mergeReplica() so client and server +-- converge on identical merge results. +-- +-- Properties (verified by tests in src/__tests__/libs/crdt.test.ts and +-- the server-merge race test): +-- * commutative, associative, idempotent on fields_jsonb +-- * remove-wins: a field write never revives a tombstone (deleted_at_ts +-- stays the larger of the two sides) +-- * preserves unknown fields from either side (forwards-compat across +-- schemaVersion bumps) +-- * deviceId tiebreak when two field envelopes share the same HLC +-- +-- Called via INSERT … ON CONFLICT … DO UPDATE in a single SQL statement +-- so two concurrent pushes can't interleave fetch-then-upsert. RUNS AS +-- SECURITY INVOKER (default) — the surrounding INSERT/UPDATE is RLS- +-- gated, so we don't need DEFINER. The server endpoint additionally +-- asserts auth.uid() = NEW.user_id before invoking the upsert. + +-- ───────────────────────────────────────────────────────────────────────── +-- HLC max helper. NULLs lose. Plain text comparison since the HLC packing +-- format makes lexicographic order match temporal order. +-- ───────────────────────────────────────────────────────────────────────── +CREATE OR REPLACE FUNCTION public.hlc_max(a text, b text) +RETURNS text +LANGUAGE sql IMMUTABLE PARALLEL SAFE +AS $$ + SELECT CASE + WHEN a IS NULL THEN b + WHEN b IS NULL THEN a + WHEN a >= b THEN a + ELSE b + END; +$$; + +-- ───────────────────────────────────────────────────────────────────────── +-- Field-level LWW merge for fields_jsonb. Per-key: keep the envelope with +-- the larger envelope.t (HLC string). Tie on HLC: deviceId (envelope.s) +-- lex-order tiebreak. Preserves keys present on either side. +-- ───────────────────────────────────────────────────────────────────────── +CREATE OR REPLACE FUNCTION public.crdt_merge_fields(local_fields jsonb, remote_fields jsonb) +RETURNS jsonb +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE +AS $$ +DECLARE + result jsonb := COALESCE(local_fields, '{}'::jsonb); + k text; + l_env jsonb; + r_env jsonb; + l_t text; + r_t text; + l_s text; + r_s text; +BEGIN + IF remote_fields IS NULL THEN + RETURN result; + END IF; + FOR k IN SELECT jsonb_object_keys(remote_fields) LOOP + r_env := remote_fields -> k; + l_env := result -> k; + IF l_env IS NULL THEN + result := jsonb_set(result, ARRAY[k], r_env, true); + ELSE + l_t := l_env ->> 't'; + r_t := r_env ->> 't'; + IF r_t > l_t THEN + result := jsonb_set(result, ARRAY[k], r_env, true); + ELSIF r_t = l_t THEN + l_s := COALESCE(l_env ->> 's', ''); + r_s := COALESCE(r_env ->> 's', ''); + IF r_s > l_s THEN + result := jsonb_set(result, ARRAY[k], r_env, true); + END IF; + END IF; + END IF; + END LOOP; + RETURN result; +END; +$$; + +-- ───────────────────────────────────────────────────────────────────────── +-- updated_at_ts = max over field HLCs and tombstone HLC. +-- ───────────────────────────────────────────────────────────────────────── +CREATE OR REPLACE FUNCTION public.crdt_compute_updated_at(fields jsonb, deleted_at text) +RETURNS text +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE +AS $$ +DECLARE + result text := COALESCE(deleted_at, '0000000000000-00000000-'); + k text; + env jsonb; + t text; +BEGIN + IF fields IS NULL THEN + RETURN result; + END IF; + FOR k IN SELECT jsonb_object_keys(fields) LOOP + env := fields -> k; + t := env ->> 't'; + IF t IS NOT NULL AND t > result THEN + result := t; + END IF; + END LOOP; + RETURN result; +END; +$$; + +-- ───────────────────────────────────────────────────────────────────────── +-- Full row merge. Used in: +-- INSERT INTO replicas (...) VALUES (...) +-- ON CONFLICT (user_id, kind, replica_id) DO UPDATE SET +-- fields_jsonb = crdt_merge_fields(replicas.fields_jsonb, EXCLUDED.fields_jsonb), +-- deleted_at_ts = hlc_max(replicas.deleted_at_ts, EXCLUDED.deleted_at_ts), +-- reincarnation = CASE WHEN replicas.reincarnation = EXCLUDED.reincarnation +-- THEN replicas.reincarnation +-- WHEN EXCLUDED.updated_at_ts > replicas.updated_at_ts +-- THEN EXCLUDED.reincarnation +-- ELSE replicas.reincarnation END, +-- manifest_jsonb = CASE WHEN EXCLUDED.updated_at_ts > replicas.updated_at_ts +-- THEN EXCLUDED.manifest_jsonb +-- ELSE replicas.manifest_jsonb END, +-- schema_version = GREATEST(replicas.schema_version, EXCLUDED.schema_version), +-- updated_at_ts = crdt_compute_updated_at( +-- crdt_merge_fields(replicas.fields_jsonb, EXCLUDED.fields_jsonb), +-- hlc_max(replicas.deleted_at_ts, EXCLUDED.deleted_at_ts) +-- ), +-- modified_at = now() +-- +-- Or via the wrapper below for shorter call sites. +-- ───────────────────────────────────────────────────────────────────────── +CREATE OR REPLACE FUNCTION public.crdt_merge_replica( + p_user_id uuid, + p_kind text, + p_replica_id text, + p_fields_jsonb jsonb, + p_manifest_jsonb jsonb, + p_deleted_at_ts text, + p_reincarnation text, + p_updated_at_ts text, + p_schema_version integer +) RETURNS public.replicas +LANGUAGE plpgsql +AS $$ +DECLARE + result public.replicas; +BEGIN + INSERT INTO public.replicas AS r ( + user_id, kind, replica_id, + fields_jsonb, manifest_jsonb, deleted_at_ts, + reincarnation, updated_at_ts, schema_version + ) VALUES ( + p_user_id, p_kind, p_replica_id, + COALESCE(p_fields_jsonb, '{}'::jsonb), + p_manifest_jsonb, p_deleted_at_ts, + p_reincarnation, p_updated_at_ts, p_schema_version + ) + ON CONFLICT (user_id, kind, replica_id) DO UPDATE SET + fields_jsonb = public.crdt_merge_fields(r.fields_jsonb, EXCLUDED.fields_jsonb), + deleted_at_ts = public.hlc_max(r.deleted_at_ts, EXCLUDED.deleted_at_ts), + reincarnation = CASE + WHEN r.reincarnation IS NOT DISTINCT FROM EXCLUDED.reincarnation + THEN r.reincarnation + WHEN EXCLUDED.updated_at_ts > r.updated_at_ts + THEN EXCLUDED.reincarnation + ELSE r.reincarnation + END, + manifest_jsonb = CASE + WHEN EXCLUDED.updated_at_ts > r.updated_at_ts + THEN EXCLUDED.manifest_jsonb + ELSE r.manifest_jsonb + END, + schema_version = GREATEST(r.schema_version, EXCLUDED.schema_version), + updated_at_ts = public.crdt_compute_updated_at( + public.crdt_merge_fields(r.fields_jsonb, EXCLUDED.fields_jsonb), + public.hlc_max(r.deleted_at_ts, EXCLUDED.deleted_at_ts) + ), + modified_at = now() + RETURNING * INTO result; + RETURN result; +END; +$$; + +-- Surface the function to the API role only after an explicit user-id +-- match check in src/pages/api/sync/replicas.ts. RLS on the replicas +-- table is the second line of defense (the function runs as caller). +GRANT EXECUTE ON FUNCTION public.hlc_max(text, text) TO authenticated; +GRANT EXECUTE ON FUNCTION public.crdt_merge_fields(jsonb, jsonb) TO authenticated; +GRANT EXECUTE ON FUNCTION public.crdt_compute_updated_at(jsonb, text) TO authenticated; +GRANT EXECUTE ON FUNCTION public.crdt_merge_replica(uuid, text, text, jsonb, jsonb, text, text, text, integer) TO authenticated;