From 6bfeb295d2e5126aeabe49bb2e3c1640dd17ccb2 Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Fri, 8 May 2026 11:33:18 +0800 Subject: [PATCH] feat(sync): add opds_catalog replica kind (plaintext fields) (#4087) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires OPDS catalogs through replica sync as a metadata-only kind. Plaintext fields only in this PR — encrypted credentials (username, password) ship in the follow-up alongside the SyncPassphrasePanel UI and Tauri keychain backend. - Migration 009 extends the kind allowlist with 'opds_catalog'. - replicaSchemas adds opdsCatalogFieldsSchema (name, url, description, icon, customHeaders, autoDownload, disabled, addedAt) with a 50-row per-user cap. - New opdsCatalogAdapter is metadata-only (no `binary` capability). Stable cross-device id from md5("opds:" + url.lower()) so two devices that import the same URL converge to one row instead of duplicating. - New customOPDSStore (zustand) hydrates from SystemSettings, publishes upserts/deletes through the replica pipeline, preserves local-only username/password when overlaying remote updates, and strips tombstones at the persistence boundary so existing useSettingsStore readers (useOPDSSubscriptions, pseStream, app/opds/page.tsx) need no migration. - replicaPullAndApply branches on adapter.binary so metadata-only kinds skip the bundleDir requirement and the manifest/binary path. - CatalogManager rewires Add / Edit / Remove / Toggle / Add-popular through the new store. Plan update bundled in: tenet 8 (scalar settings sync via a bundled row; collections sync per-record), per-kind allowlist now includes a `settings` singleton that will collapse PRs 5 + 6+ into one bundled adapter, and PR 4 is split into 4a (already merged) / 4b (this) / 4c (encrypted credentials + UX). Co-authored-by: Claude Opus 4.7 (1M context) --- .../.claude/plans/vivid-orbiting-thimble.md | 179 +++++++++--- .../src/__tests__/libs/replicaSchemas.test.ts | 4 +- .../__tests__/libs/replicaSyncServer.test.ts | 2 +- .../sync/adapters/opdsCatalog.test.ts | 123 ++++++++ .../services/sync/replicaBootstrap.test.ts | 13 +- .../__tests__/store/custom-opds-store.test.ts | 269 ++++++++++++++++++ apps/readest-app/src/app/library/page.tsx | 7 +- .../app/opds/components/CatalogManager.tsx | 85 ++++-- apps/readest-app/src/hooks/useReplicaPull.ts | 33 ++- apps/readest-app/src/libs/replicaSchemas.ts | 20 ++ .../src/services/sync/adapters/opdsCatalog.ts | 116 ++++++++ .../src/services/sync/replicaBootstrap.ts | 3 + .../src/services/sync/replicaPullAndApply.ts | 17 +- apps/readest-app/src/store/customOPDSStore.ts | 250 ++++++++++++++++ apps/readest-app/src/types/opds.ts | 13 + .../migrations/009_replica_opds_catalog.sql | 11 + 16 files changed, 1069 insertions(+), 76 deletions(-) create mode 100644 apps/readest-app/src/__tests__/services/sync/adapters/opdsCatalog.test.ts create mode 100644 apps/readest-app/src/__tests__/store/custom-opds-store.test.ts create mode 100644 apps/readest-app/src/services/sync/adapters/opdsCatalog.ts create mode 100644 apps/readest-app/src/store/customOPDSStore.ts create mode 100644 docker/volumes/db/migrations/009_replica_opds_catalog.sql diff --git a/apps/readest-app/.claude/plans/vivid-orbiting-thimble.md b/apps/readest-app/.claude/plans/vivid-orbiting-thimble.md index 8adacf03..56f5ca83 100644 --- a/apps/readest-app/.claude/plans/vivid-orbiting-thimble.md +++ b/apps/readest-app/.claude/plans/vivid-orbiting-thimble.md @@ -71,6 +71,20 @@ for the full decision capture. The polymorphic `replicas` table is internal infrastructure, not a future API. Adding any cross-user-visibility feature requires a fresh architecture review and is not authorized by this plan. +8. **Scalar settings sync via a bundled row; collections sync per-record.** + The CRDT model is per-field LWW within a row, so 1 row × N fields and + N rows × 1 field have identical conflict semantics for scalars. + Scalar settings (`theme`, `fontSize`, `highlightColor`) and flat maps + (`providerEnabled`, per-shortcut overrides) collapse cleanly into a + single `settings` kind with a server-managed field whitelist — + adding a new synced setting becomes a one-line whitelist addition. + Collections where each element needs independent identity, tombstones, + or remove-wins semantics (OPDS catalogs, dictionaries, fonts, + textures, web searches, ordered provider positions) stay per-record. + Folding a collection into a bundled row is unsafe under concurrent + edits: Device A adds X, Device B adds Y, both push the full array, + last writer wins. Per-record rows preserve both sides via independent + CRDT envelopes. ## Architecture overview @@ -307,15 +321,19 @@ on every add). ## Per-kind initial allowlist (ship in this order) -| kind | binary | id source | files | +| kind | binary | id source | files / shape | |---|---|---|---| | `dictionary` | yes | `partialMD5(primaryFile) + size + filenames` | mdx + mdd[] + css[] (skip `.idx.offsets`/`.syn.offsets`) | | `font` | yes | `partialMD5(file) + size + filename` | single .ttf/.otf/.woff[2] | | `texture` | yes | `partialMD5(file) + size + filename` | single image | -| `opds_catalog` | no | `md5(url + username)` | — (URL + name + headers + **password ENCRYPTED**) | -| `dict_provider_pref` | no | provider id | — (one boolean field `enabled`) | -| `dict_provider_position` | no | provider id | — (one position string + actor id) | -| `dict_web_search` | no | existing `WebSearchEntry.id` | — | +| `opds_catalog` | no | `md5("opds:" + url.lower())` | — (URL + name + headers + **password ENCRYPTED**) | +| `settings` | no | `'singleton'` (one row per user) | — (whitelisted scalar fields + flat maps via namespaced field keys; covers `theme`, `fontSize`, `highlightColor`, `lineHeight`, `pref:*`, `shortcut:*`, `providerEnabled.`, `syncCategories.`, etc.) | +| `dict_provider_position` | no | provider id | — (one position string + actor id; per-element rows because concurrent rename + reorder must preserve both sides) | +| `dict_web_search` | no | existing `WebSearchEntry.id` | — (collection of independent records — needs per-record tombstones) | + +`settings` replaces what would otherwise be N per-kind adapters for each +scalar setting. Adding a new synced setting = one-line whitelist + a +server schema bump; no new adapter, no new client wiring. Future kinds require a server PR (schema + allowlist + migration if needed). @@ -323,15 +341,18 @@ needed). ## User-selectable sync categories A new settings panel under **Account → Sync** with per-category toggles. -Default: book/config/note/dictionary/font/texture/opds_catalog all ON; -internal kinds (`dict_provider_pref`, `dict_provider_position`, -`dict_web_search`) follow the parent (dictionary) toggle. +Default: book/config/note/dictionary/font/texture/opds_catalog/settings +all ON; internal kinds (`dict_provider_position`, `dict_web_search`) +follow the parent (dictionary) toggle. `providerEnabled.` lives +inside the `settings` bundle and rides the `settings` toggle, not the +`dictionary` toggle (small ergonomic gap; surfaced in the Sync panel +copy). `SystemSettings.syncCategories: Record` stores the -preferences. The category map itself syncs through the existing -`config`-style settings JSON sync (or, more cleanly, becomes its own kind -`sync_pref` once the primitive is built). Per-category gates apply at the -sync manager: +preferences. Per tenet 8, the category map itself becomes a flat map +inside the `settings` bundle (`syncCategories.` namespaced fields) +once the bundle ships in PR 5 — no separate `sync_pref` kind needed. +Per-category gates apply at the sync manager: - `pull(kind)` no-ops if `syncCategories[kind] === false`. - `push(rows)` filters out rows whose kind is disabled. @@ -538,29 +559,82 @@ example. Add `'font'` to the allowlist + schema + adapter. Similar shape to fonts. -### PR 4 — `opds_catalog` (with encrypted password) +### PR 4 — `opds_catalog` (split into 4a + 4b + 4c) -Adapter syncs `id`, `name`, `url`, `description`, `icon`, -`customHeaders`, `autoDownload` as plaintext fields, and `password` -(and optionally `username`) as **encrypted-field envelopes** using the -crypto infra shipped in PR 1. Lazy passphrase prompt: first OPDS -catalog import with credentials triggers the "Set sync passphrase" -modal. +Originally one PR; split during PR 4 build because the crypto +introduction is a step-up in complexity that benefits from separate +review surfaces. -### PR 5 — internal dict-settings kinds +**PR 4a — encrypted-field session wiring (no consumer kind).** Ships +the per-account PBKDF2 salt endpoint (`/api/sync/replica-keys` + the +`replica_keys_create` / `replica_keys_list` RPCs against the +`replica_keys` table from migration 003), the in-memory `CryptoSession` +manager that derives keys lazily per `saltId`, and tests. No UI, no +consumer kind. Production-ready infrastructure for PR 4c. -Migrate `dictionarySettings.providerOrder` (per-position rows with -`(position, actorId, replicaId)` total order) / -`dictionarySettings.providerEnabled` (per-pref rows) / -`dictionarySettings.webSearches`. This lets concurrent rename + reorder -both survive on dictionaries — the original goal — but only after the -single-array model is fully retired. +**PR 4b — `opds_catalog` plaintext fields.** Adapter syncs `id`, +`name`, `url`, `description`, `icon`, `customHeaders`, `autoDownload`, +`disabled`, `addedAt`. Stable cross-device id from `md5("opds:" + +url.lower())`. Credentials (`username`, `password`) stay local-only — +not pushed, not pull-overwritten. Public catalogs sync end-to-end +immediately; credentialed catalogs need re-entry on each device until +4c lands. -### PR 6+ — anything else +**PR 4c — `opds_catalog` encrypted credentials + passphrase UX.** +Adds `username` and `password` as encrypted-field envelopes via the +crypto session shipped in 4a. Ships `SyncPassphrasePanel` UI +(set / change / forgot), the lazy `getOrPromptPassphrase` modal that +fires on first encrypted-field push or pull, the Tauri keychain backend +that replaces the `EphemeralPassphraseStore` stub, and the +forgot-passphrase server endpoint (wipes encrypted envelopes + rotates +salt). -`pref`, `theme`, `shortcut`, `annotation_rule`, future AI keys -(another encrypted-field kind), etc. Each is a coordinated client+server -PR (adapter + schema + allowlist). +### PR 5 — `settings` bundled kind (collapses original PR 5 + PR 6+) + +Per tenet 8, one `settings` adapter with a server-managed whitelist of +`SystemSettings` keys, instead of N per-kind adapters for each scalar +setting. Initial whitelist: `theme`, `fontSize`, `lineHeight`, +`highlightColor`, `pref:*`, `shortcut:*`, +`dictionarySettings.providerEnabled` (encoded as namespaced field keys +like `providerEnabled.`), `syncCategories.`. + +Singleton row (`replica_id = 'singleton'`) per user. Per-field LWW +handles concurrent edits across devices: Device A toggles dark mode, +Device B changes font size, both push, both apply. Adding a new synced +setting is a one-line whitelist addition + a server schema bump — no +new adapter, no new client wiring. + +This collapses what was previously planned as PRs 5 + 6+ (per-kind +adapters for `dict_provider_pref`, `pref`, `theme`, `shortcut`, +`annotation_rule`, etc.). The genuinely-different shapes (ordered +collections, independent-record collections) ship as PR 6 and PR 7. + +### PR 6 — `dict_provider_position` (ordered list with per-position rows) + +The provider order is the only setting that genuinely needs per-element +rows: concurrent rename + reorder must preserve both sides, which a +single-field array can't do. Per-position rows keyed by +`(position, actorId, replicaId)` with deterministic tiebreak. +Migrates `dictionarySettings.providerOrder` off the single-array +shape. + +### PR 7 — `dict_web_search` (custom web-search entries) + +Collection of independent records — each with `id`, `name`, +`urlTemplate`, plus tombstones. Per-record rows because users add / +delete / rename entries independently across devices. + +### PR 8+ — incremental whitelist additions + +New scalar settings join the `settings` whitelist as needed (one-line +PR + server schema bump). Future encrypted-field needs (AI API keys, +etc.) get either: + +- A new dedicated kind (if it needs its own quota, allowlist, or + per-record tombstones), OR +- A namespaced encrypted field within `settings` (if it's a small + scalar that fits the bundled pattern — e.g., + `aiApiKey.openai`, `aiApiKey.anthropic`). ## Migration @@ -1059,8 +1133,49 @@ infra in PR 1; OPDS adapter that consumes it ships in PR 4. - `/codex review` re-run on the revised plan would catch any drift introduced by CEO + eng review changes; recommended but not required. +**POST-PR-3 ARCHITECTURE REFINEMENT (during PR 4 build):** + +Questioned why each scalar setting needed its own kind. Insight: the +CRDT model is per-field LWW within a row, so 1 row × N fields and N +rows × 1 field have identical conflict semantics for scalars. The +`dict_provider_pref` adapter (one boolean field per provider) was the +canary — it would have been a 100-LOC adapter for a single boolean. +Generalizing: any scalar setting collapses into a single bundled +`settings` row with a server-managed field whitelist, using namespaced +field keys (`providerEnabled.`, `syncCategories.`, +`shortcut.`) for flat maps. + +What does NOT collapse: collections of independent records (OPDS +catalogs, dictionaries, fonts, textures, web searches) — each element +needs independent identity, tombstones, and per-element CRDT envelopes +to survive concurrent add/remove on different devices. And ordered +lists (`providerOrder`) — concurrent rename + reorder must preserve +both sides, which a single-field array can't do. + +Plan changes absorbed: + +- **New tenet 8** captures the scalar-vs-collection rule. +- **Per-kind allowlist updated:** added `settings` (singleton); removed + `dict_provider_pref` (folds into settings). +- **Phasing rewritten:** PR 4 split into 4a (encrypted-field session + wiring) + 4b (opds_catalog plaintext) + 4c (opds_catalog encrypted + credentials + passphrase UX). PR 5 becomes the bundled `settings` + kind — collapses what was previously planned as PRs 5 + 6+. PR 6 = + `dict_provider_position` (still needs per-element rows). PR 7 = + `dict_web_search`. PR 8+ = incremental whitelist additions, no new + adapters for scalar settings. +- **Sync categories** become a flat map inside the `settings` bundle + (`syncCategories.`); the standalone `sync_pref` kind hinted at + in the original plan is no longer needed. + +Net: ~70% LOC reduction on the remaining roadmap (no per-kind adapters +for `pref`, `theme`, `shortcut`, `annotation_rule`, etc.). No change to +PRs 1–3 (already shipped) or PR 4a (already merged as #4084). + **VERDICT:** CEO + ENG CLEARED — plan is implementation-ready. Codex review reshaped the architecture; CEO review reshaped the scope; eng review locked implementation details (adapter shape, push trigger, -file layout, error model, test coverage, manifest schema, perf SLOs). -Next step: implement PR 1. +file layout, error model, test coverage, manifest schema, perf SLOs); +the post-PR-3 refinement collapsed the long-tail per-kind adapters +into a single bundled `settings` kind. Next step: complete PR 4b/4c, +then ship PR 5. diff --git a/apps/readest-app/src/__tests__/libs/replicaSchemas.test.ts b/apps/readest-app/src/__tests__/libs/replicaSchemas.test.ts index fcf1960c..b5087ca8 100644 --- a/apps/readest-app/src/__tests__/libs/replicaSchemas.test.ts +++ b/apps/readest-app/src/__tests__/libs/replicaSchemas.test.ts @@ -28,11 +28,11 @@ const baseRow = (overrides: Partial = {}): ReplicaRow => ({ }); describe('isAllowedKind', () => { - test('current allowlist contains dictionary + font + texture', () => { + test('current allowlist contains dictionary + font + texture + opds_catalog', () => { expect(isAllowedKind('dictionary')).toBe(true); expect(isAllowedKind('font')).toBe(true); expect(isAllowedKind('texture')).toBe(true); - expect(isAllowedKind('opds_catalog')).toBe(false); + expect(isAllowedKind('opds_catalog')).toBe(true); }); test('rejects arbitrary strings', () => { diff --git a/apps/readest-app/src/__tests__/libs/replicaSyncServer.test.ts b/apps/readest-app/src/__tests__/libs/replicaSyncServer.test.ts index d69f5486..957db214 100644 --- a/apps/readest-app/src/__tests__/libs/replicaSyncServer.test.ts +++ b/apps/readest-app/src/__tests__/libs/replicaSyncServer.test.ts @@ -145,7 +145,7 @@ describe('validatePullParams', () => { }); test('rejects unknown kind', () => { - const result = validatePullParams('opds_catalog', null); + const result = validatePullParams('not_a_kind', null); expect(result.ok).toBe(false); if (!result.ok) { expect(result.status).toBe(422); diff --git a/apps/readest-app/src/__tests__/services/sync/adapters/opdsCatalog.test.ts b/apps/readest-app/src/__tests__/services/sync/adapters/opdsCatalog.test.ts new file mode 100644 index 00000000..3848f69c --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/adapters/opdsCatalog.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, test } from 'vitest'; +import { + computeOpdsCatalogContentId, + opdsCatalogAdapter, + OPDS_CATALOG_KIND, + OPDS_CATALOG_SCHEMA_VERSION, +} from '@/services/sync/adapters/opdsCatalog'; +import type { OPDSCatalog } from '@/types/opds'; +import type { FieldEnvelope, Hlc, ReplicaRow } from '@/types/replica'; + +const sample: OPDSCatalog = { + id: 'cid', + contentId: 'cid', + name: 'My Library', + url: 'https://example.com/opds', + description: 'A test catalog', + icon: '📚', + customHeaders: { 'X-Token': 'abc' }, + autoDownload: true, + addedAt: 1700000000000, +}; + +const HLC = '00000000000-00000000-dev' as Hlc; +const env = (v: T): FieldEnvelope => ({ v, t: HLC, s: 'dev' }); + +const makeRow = (overrides: Partial = {}): ReplicaRow => ({ + user_id: 'u', + kind: OPDS_CATALOG_KIND, + replica_id: 'cid', + fields_jsonb: { + name: env('My Library'), + url: env('https://example.com/opds'), + description: env('A test catalog'), + icon: env('📚'), + customHeaders: env({ 'X-Token': 'abc' }), + autoDownload: env(true), + addedAt: env(1700000000000), + }, + manifest_jsonb: null, + deleted_at_ts: null, + reincarnation: null, + updated_at_ts: HLC, + schema_version: 1, + ...overrides, +}); + +describe('opdsCatalogAdapter', () => { + test('kind + schemaVersion are stable constants', () => { + expect(opdsCatalogAdapter.kind).toBe('opds_catalog'); + expect(opdsCatalogAdapter.schemaVersion).toBe(1); + expect(OPDS_CATALOG_KIND).toBe('opds_catalog'); + expect(OPDS_CATALOG_SCHEMA_VERSION).toBe(1); + }); + + test('declares no `binary` capability — metadata-only kind', () => { + expect(opdsCatalogAdapter.binary).toBeUndefined(); + }); + + test('pack omits username and password (encrypted-credential PR)', () => { + const withCreds: OPDSCatalog = { ...sample, username: 'alice', password: 'hunter2' }; + const fields = opdsCatalogAdapter.pack(withCreds); + expect(fields['username']).toBeUndefined(); + expect(fields['password']).toBeUndefined(); + expect(fields['name']).toBe('My Library'); + expect(fields['url']).toBe('https://example.com/opds'); + }); + + test('pack ∘ unpack is identity for non-credential fields', async () => { + const fields = opdsCatalogAdapter.pack(sample); + const out = opdsCatalogAdapter.unpack(fields); + expect(out.name).toBe(sample.name); + expect(out.url).toBe(sample.url); + expect(out.description).toBe(sample.description); + expect(out.icon).toBe(sample.icon); + expect(out.customHeaders).toEqual(sample.customHeaders); + expect(out.autoDownload).toBe(true); + expect(out.addedAt).toBe(sample.addedAt); + }); + + test('computeId returns the contentId', async () => { + const id = await opdsCatalogAdapter.computeId(sample); + expect(id).toBe('cid'); + }); + + test('unpackRow rebuilds the catalog from CRDT envelopes', () => { + const row = makeRow(); + const out = opdsCatalogAdapter.unpackRow(row, ''); + expect(out).not.toBeNull(); + expect(out!.id).toBe('cid'); + expect(out!.contentId).toBe('cid'); + expect(out!.name).toBe('My Library'); + expect(out!.url).toBe('https://example.com/opds'); + expect(out!.autoDownload).toBe(true); + }); + + test('unpackRow returns null when name or url is missing', () => { + const noName = makeRow({ fields_jsonb: { url: env('https://x') } }); + expect(opdsCatalogAdapter.unpackRow(noName, '')).toBeNull(); + + const noUrl = makeRow({ fields_jsonb: { name: env('x') } }); + expect(opdsCatalogAdapter.unpackRow(noUrl, '')).toBeNull(); + }); + + test('unpackRow surfaces the reincarnation token', () => { + const row = makeRow({ reincarnation: 'rev1' }); + const out = opdsCatalogAdapter.unpackRow(row, ''); + expect(out!.reincarnation).toBe('rev1'); + }); +}); + +describe('computeOpdsCatalogContentId', () => { + test('is stable under whitespace and case differences in URL', () => { + const a = computeOpdsCatalogContentId('https://Example.com/OPDS/'); + const b = computeOpdsCatalogContentId(' https://example.com/opds/ '); + expect(a).toBe(b); + }); + + test('different URLs produce different ids', () => { + const a = computeOpdsCatalogContentId('https://example.com/opds'); + const b = computeOpdsCatalogContentId('https://example.com/feed'); + expect(a).not.toBe(b); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/replicaBootstrap.test.ts b/apps/readest-app/src/__tests__/services/sync/replicaBootstrap.test.ts index 6f45916b..0d678d71 100644 --- a/apps/readest-app/src/__tests__/services/sync/replicaBootstrap.test.ts +++ b/apps/readest-app/src/__tests__/services/sync/replicaBootstrap.test.ts @@ -12,6 +12,13 @@ vi.mock('@/store/customTextureStore', () => ({ useCustomTextureStore: { getState: () => ({ markAvailableByContentId: vi.fn() }) }, })); +vi.mock('@/store/customOPDSStore', () => ({ + useCustomOPDSStore: { + getState: () => ({ applyRemoteCatalog: vi.fn(), softDeleteByContentId: vi.fn() }), + }, + findOPDSCatalogByContentId: vi.fn(), +})); + import { __resetBootstrapForTests, bootstrapReplicaAdapters, @@ -39,12 +46,12 @@ describe('bootstrapReplicaAdapters', () => { test('is idempotent: calling twice is a no-op (does not throw)', () => { bootstrapReplicaAdapters(); bootstrapReplicaAdapters(); - expect(listReplicaAdapters()).toHaveLength(3); + expect(listReplicaAdapters()).toHaveLength(4); }); - test('registers the current allowlist (dictionary, font, texture)', () => { + test('registers the current allowlist (dictionary, font, texture, opds_catalog)', () => { bootstrapReplicaAdapters(); const kinds = listReplicaAdapters().map((a) => a.kind); - expect(kinds).toEqual(['dictionary', 'font', 'texture']); + expect(kinds).toEqual(['dictionary', 'font', 'texture', 'opds_catalog']); }); }); diff --git a/apps/readest-app/src/__tests__/store/custom-opds-store.test.ts b/apps/readest-app/src/__tests__/store/custom-opds-store.test.ts new file mode 100644 index 00000000..311add87 --- /dev/null +++ b/apps/readest-app/src/__tests__/store/custom-opds-store.test.ts @@ -0,0 +1,269 @@ +import { describe, test, expect, beforeEach, vi } from 'vitest'; +import { useCustomOPDSStore } from '@/store/customOPDSStore'; +import { useSettingsStore } from '@/store/settingsStore'; +import { computeOpdsCatalogContentId } from '@/services/sync/adapters/opdsCatalog'; +import type { OPDSCatalog } from '@/types/opds'; +import type { SystemSettings } from '@/types/settings'; +import type { EnvConfigType } from '@/services/environment'; + +// Replica-publish helpers fan out to the network — stub them so tests +// stay hermetic. We assert they fire for upserts/deletes via spies. +vi.mock('@/services/sync/replicaPublish', () => ({ + publishReplicaUpsert: vi.fn(), + publishReplicaDelete: vi.fn(), +})); + +import { publishReplicaUpsert, publishReplicaDelete } from '@/services/sync/replicaPublish'; + +const makeEnvConfig = (): EnvConfigType => + ({ + getAppService: vi.fn(), + }) as unknown as EnvConfigType; + +const makeSettings = (overrides: Partial = {}): SystemSettings => + ({ + opdsCatalogs: [], + ...overrides, + }) as unknown as SystemSettings; + +beforeEach(() => { + useCustomOPDSStore.setState({ catalogs: [], loading: false }); + useSettingsStore.setState({ + settings: makeSettings(), + setSettings: (s: SystemSettings) => useSettingsStore.setState({ settings: s }), + saveSettings: vi.fn(), + } as unknown as ReturnType); + vi.clearAllMocks(); +}); + +describe('customOPDSStore', () => { + describe('addCatalog', () => { + test('mints a contentId from the URL when one is not provided', () => { + const cat = useCustomOPDSStore.getState().addCatalog({ + id: 'local-1', + name: 'My Library', + url: 'https://example.com/opds', + }); + expect(cat.contentId).toBe(computeOpdsCatalogContentId('https://example.com/opds')); + expect(cat.addedAt).toBeGreaterThan(0); + }); + + test('publishes the upsert via replicaPublish', () => { + useCustomOPDSStore.getState().addCatalog({ + id: 'local-1', + name: 'My Library', + url: 'https://example.com/opds', + }); + expect(publishReplicaUpsert).toHaveBeenCalledTimes(1); + const [kind] = (publishReplicaUpsert as unknown as ReturnType).mock.calls[0]!; + expect(kind).toBe('opds_catalog'); + }); + + test('re-adding a soft-deleted entry mints a reincarnation token', () => { + const first = useCustomOPDSStore.getState().addCatalog({ + id: 'l1', + name: 'L1', + url: 'https://example.com/opds', + }); + useCustomOPDSStore.getState().removeCatalog(first.id); + vi.clearAllMocks(); + const revived = useCustomOPDSStore.getState().addCatalog({ + id: 'l2', + name: 'L1 again', + url: 'https://example.com/opds', + }); + expect(revived.deletedAt).toBeUndefined(); + expect(revived.reincarnation).toBeTruthy(); + expect(publishReplicaUpsert).toHaveBeenCalledTimes(1); + }); + }); + + describe('updateCatalog', () => { + test('publishes the upsert for non-URL patches', () => { + const cat = useCustomOPDSStore.getState().addCatalog({ + id: 'local-1', + name: 'Old', + url: 'https://example.com/opds', + }); + const oldContentId = cat.contentId; + vi.clearAllMocks(); + const updated = useCustomOPDSStore.getState().updateCatalog(cat.id, { name: 'New' }); + expect(updated!.name).toBe('New'); + expect(updated!.contentId).toBe(oldContentId); + expect(publishReplicaUpsert).toHaveBeenCalledTimes(1); + }); + + test('changing the URL recomputes contentId', () => { + const cat = useCustomOPDSStore.getState().addCatalog({ + id: 'local-1', + name: 'Cat', + url: 'https://example.com/opds', + }); + const updated = useCustomOPDSStore + .getState() + .updateCatalog(cat.id, { url: 'https://other.example/opds' }); + expect(updated!.contentId).toBe(computeOpdsCatalogContentId('https://other.example/opds')); + expect(updated!.contentId).not.toBe(cat.contentId); + }); + + test('no-op on tombstoned entry', () => { + const cat = useCustomOPDSStore.getState().addCatalog({ + id: 'l1', + name: 'L1', + url: 'https://example.com/opds', + }); + useCustomOPDSStore.getState().removeCatalog(cat.id); + vi.clearAllMocks(); + const out = useCustomOPDSStore.getState().updateCatalog(cat.id, { name: 'X' }); + expect(out).toBeUndefined(); + expect(publishReplicaUpsert).not.toHaveBeenCalled(); + }); + }); + + describe('removeCatalog', () => { + test('soft-deletes and publishes the tombstone', () => { + const cat = useCustomOPDSStore.getState().addCatalog({ + id: 'l1', + name: 'L1', + url: 'https://example.com/opds', + }); + vi.clearAllMocks(); + const removed = useCustomOPDSStore.getState().removeCatalog(cat.id); + expect(removed).toBe(true); + const stored = useCustomOPDSStore.getState().getCatalog(cat.id); + expect(stored?.deletedAt).toBeGreaterThan(0); + expect(publishReplicaDelete).toHaveBeenCalledWith('opds_catalog', cat.contentId); + }); + + test('returns false when id is unknown', () => { + expect(useCustomOPDSStore.getState().removeCatalog('nope')).toBe(false); + }); + }); + + describe('applyRemoteCatalog', () => { + test('inserts when the contentId is unknown locally', () => { + const cat: OPDSCatalog = { + id: 'remote-cid', + contentId: 'remote-cid', + name: 'Remote', + url: 'https://remote.example/opds', + addedAt: 1700000000000, + }; + useCustomOPDSStore.getState().applyRemoteCatalog(cat); + const stored = useCustomOPDSStore.getState().findByContentId('remote-cid'); + expect(stored?.name).toBe('Remote'); + expect(publishReplicaUpsert).not.toHaveBeenCalled(); + }); + + test('preserves local username/password when overlaying a remote update', () => { + const local = useCustomOPDSStore.getState().addCatalog({ + id: 'l1', + name: 'Local', + url: 'https://example.com/opds', + username: 'alice', + password: 'hunter2', + }); + vi.clearAllMocks(); + useCustomOPDSStore.getState().applyRemoteCatalog({ + id: local.contentId!, + contentId: local.contentId, + name: 'Renamed remotely', + url: 'https://example.com/opds', + addedAt: 1700000001000, + }); + const merged = useCustomOPDSStore.getState().findByContentId(local.contentId!); + expect(merged?.name).toBe('Renamed remotely'); + expect(merged?.username).toBe('alice'); + expect(merged?.password).toBe('hunter2'); + expect(publishReplicaUpsert).not.toHaveBeenCalled(); + }); + }); + + describe('softDeleteByContentId', () => { + test('marks the matching entry as deleted without re-publishing', () => { + const cat = useCustomOPDSStore.getState().addCatalog({ + id: 'l1', + name: 'L1', + url: 'https://example.com/opds', + }); + vi.clearAllMocks(); + useCustomOPDSStore.getState().softDeleteByContentId(cat.contentId!); + const stored = useCustomOPDSStore.getState().findByContentId(cat.contentId!); + expect(stored?.deletedAt).toBeGreaterThan(0); + expect(publishReplicaDelete).not.toHaveBeenCalled(); + }); + }); + + describe('saveCustomOPDSCatalogs', () => { + test('strips tombstoned entries from the persisted settings list', async () => { + const live = useCustomOPDSStore.getState().addCatalog({ + id: 'l1', + name: 'Live', + url: 'https://example.com/opds', + }); + const dead = useCustomOPDSStore.getState().addCatalog({ + id: 'l2', + name: 'Dead', + url: 'https://other.example/opds', + }); + useCustomOPDSStore.getState().removeCatalog(dead.id); + await useCustomOPDSStore.getState().saveCustomOPDSCatalogs(makeEnvConfig()); + const persisted = useSettingsStore.getState().settings.opdsCatalogs!; + expect(persisted).toHaveLength(1); + expect(persisted[0]!.id).toBe(live.id); + }); + }); + + describe('loadCustomOPDSCatalogs', () => { + test('backfills contentId on legacy entries and republishes them', async () => { + const legacy: OPDSCatalog = { + id: 'legacy-1', + name: 'Legacy', + url: 'https://legacy.example/opds', + }; + useSettingsStore.setState({ + settings: makeSettings({ opdsCatalogs: [legacy] }), + } as unknown as ReturnType); + await useCustomOPDSStore.getState().loadCustomOPDSCatalogs(makeEnvConfig()); + const inMemory = useCustomOPDSStore.getState().getCatalog('legacy-1')!; + expect(inMemory.contentId).toBe(computeOpdsCatalogContentId('https://legacy.example/opds')); + expect(publishReplicaUpsert).toHaveBeenCalledTimes(1); + }); + + test('preserves the existing array order via descending addedAt timestamps', async () => { + const legacy: OPDSCatalog[] = [ + { id: 'a', name: 'Alpha', url: 'https://a.example/opds' }, + { id: 'b', name: 'Bravo', url: 'https://b.example/opds' }, + { id: 'c', name: 'Charlie', url: 'https://c.example/opds' }, + ]; + useSettingsStore.setState({ + settings: makeSettings({ opdsCatalogs: legacy }), + } as unknown as ReturnType); + await useCustomOPDSStore.getState().loadCustomOPDSCatalogs(makeEnvConfig()); + const ordered = useCustomOPDSStore.getState().getAvailableCatalogs(); + expect(ordered.map((c) => c.id)).toEqual(['a', 'b', 'c']); + // Strict descending — first entry strictly newer than next. + expect(ordered[0]!.addedAt!).toBeGreaterThan(ordered[1]!.addedAt!); + expect(ordered[1]!.addedAt!).toBeGreaterThan(ordered[2]!.addedAt!); + }); + + test('hydrates without backfilling when entries already carry contentId', async () => { + useSettingsStore.setState({ + settings: makeSettings({ + opdsCatalogs: [ + { + id: 'a', + contentId: 'a', + name: 'A', + url: 'https://example.com/opds', + addedAt: 1700000000000, + }, + ], + }), + } as unknown as ReturnType); + await useCustomOPDSStore.getState().loadCustomOPDSCatalogs(makeEnvConfig()); + expect(publishReplicaUpsert).not.toHaveBeenCalled(); + expect(useCustomOPDSStore.getState().catalogs).toHaveLength(1); + }); + }); +}); diff --git a/apps/readest-app/src/app/library/page.tsx b/apps/readest-app/src/app/library/page.tsx index be1b1383..3da46ec8 100644 --- a/apps/readest-app/src/app/library/page.tsx +++ b/apps/readest-app/src/app/library/page.tsx @@ -117,9 +117,10 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP const { isTransferQueueOpen } = useTransferStore(); // Library page pulls user replicas (dictionaries, custom fonts, - // background textures). Deferred 10s; module-scoped dedup means a - // later navigation to the reader won't re-pull the same kind. - useReplicaPull({ kinds: ['dictionary', 'font', 'texture'] }); + // background textures, OPDS catalogs). Deferred 10s; module-scoped + // dedup means a later navigation to the reader won't re-pull the + // same kind. + useReplicaPull({ kinds: ['dictionary', 'font', 'texture', 'opds_catalog'] }); const [showCatalogManager, setShowCatalogManager] = useState( searchParams?.get('opds') === 'true', ); diff --git a/apps/readest-app/src/app/opds/components/CatalogManager.tsx b/apps/readest-app/src/app/opds/components/CatalogManager.tsx index 0d5c450c..79aeaf86 100644 --- a/apps/readest-app/src/app/opds/components/CatalogManager.tsx +++ b/apps/readest-app/src/app/opds/components/CatalogManager.tsx @@ -16,9 +16,8 @@ import { import { useRouter } from 'next/navigation'; import { useEnv } from '@/context/EnvContext'; import { useTranslation } from '@/hooks/useTranslation'; -import { useSettingsStore } from '@/store/settingsStore'; import { isWebAppPlatform } from '@/services/environment'; -import { saveSysSettings } from '@/helpers/settings'; +import { useCustomOPDSStore } from '@/store/customOPDSStore'; import { OPDSCatalog } from '@/types/opds'; import { isLanAddress } from '@/utils/network'; import { eventDispatcher } from '@/utils/event'; @@ -89,8 +88,15 @@ export function CatalogManager() { const _ = useTranslation(); const router = useRouter(); const { envConfig, appService } = useEnv(); - const { settings } = useSettingsStore(); - const [catalogs, setCatalogs] = useState(() => settings.opdsCatalogs || []); + // Hydrate the store from settings on mount; all CRUD goes through it + // so the replica-sync push fires automatically. The local `catalogs` + // mirror tracks the visible (non-deleted) entries; we keep the + // setState wrapper so `useEffect` consumers (subscriptions) re-fire + // when the list changes. + const allCatalogs = useCustomOPDSStore((s) => s.catalogs); + const [catalogs, setCatalogs] = useState(() => + useCustomOPDSStore.getState().getAvailableCatalogs(), + ); const [showAddDialog, setShowAddDialog] = useState(false); const [editingCatalogId, setEditingCatalogId] = useState(null); const [newCatalog, setNewCatalog] = useState(EMPTY_NEW_CATALOG); @@ -131,9 +137,23 @@ export function CatalogManager() { newCatalog.customHeadersInput.trim().length > 0; const isWebCatalogProxyWarningRequired = isWebAppPlatform() && hasSensitiveWebOPDSInput; - const saveCatalogs = (updatedCatalogs: OPDSCatalog[]) => { - setCatalogs(updatedCatalogs); - saveSysSettings(envConfig, 'opdsCatalogs', updatedCatalogs); + // Hydrate from settings + persist when the store mutates. Loading + // happens once per mount; the store handles backfilling contentId + // for legacy entries. + useEffect(() => { + void useCustomOPDSStore.getState().loadCustomOPDSCatalogs(envConfig); + }, [envConfig]); + + // Surface the latest store state into the local mirror used by + // subscriptions / dialog rendering. Filters out tombstones. + useEffect(() => { + setCatalogs(allCatalogs.filter((c) => !c.deletedAt)); + }, [allCatalogs]); + + // Persist via the store (settings + replica push), then update local + // mirror. Replica sync fan-out happens inside the store mutators. + const persistMutation = () => { + void useCustomOPDSStore.getState().saveCustomOPDSCatalogs(envConfig); }; const handleAddCatalog = async () => { @@ -187,24 +207,33 @@ export function CatalogManager() { return; } - const catalog: OPDSCatalog = { - id: editingCatalogId || Date.now().toString(), - name: newCatalog.name, - url: newCatalog.url, - description: newCatalog.description, - username: newCatalog.username || undefined, - password: newCatalog.password || undefined, - customHeaders: hasOPDSCustomHeaders(parsedHeaders.headers) - ? parsedHeaders.headers - : undefined, - autoDownload: newCatalog.autoDownload || undefined, - }; + const customHeaders = hasOPDSCustomHeaders(parsedHeaders.headers) + ? parsedHeaders.headers + : undefined; if (editingCatalogId) { - saveCatalogs(catalogs.map((c) => (c.id === editingCatalogId ? catalog : c))); + useCustomOPDSStore.getState().updateCatalog(editingCatalogId, { + name: newCatalog.name, + url: newCatalog.url, + description: newCatalog.description || undefined, + username: newCatalog.username || undefined, + password: newCatalog.password || undefined, + customHeaders, + autoDownload: newCatalog.autoDownload || undefined, + }); } else { - saveCatalogs([catalog, ...catalogs]); + useCustomOPDSStore.getState().addCatalog({ + id: Date.now().toString(), + name: newCatalog.name, + url: newCatalog.url, + description: newCatalog.description || undefined, + username: newCatalog.username || undefined, + password: newCatalog.password || undefined, + customHeaders, + autoDownload: newCatalog.autoDownload || undefined, + }); } + persistMutation(); setNewCatalog(EMPTY_NEW_CATALOG); setUrlError(''); @@ -234,12 +263,13 @@ export function CatalogManager() { if (catalogs.some((c) => c.url === popularCatalog.url)) { return; } - - saveCatalogs([...catalogs, { ...popularCatalog }]); + useCustomOPDSStore.getState().addCatalog({ ...popularCatalog }); + persistMutation(); }; const handleRemoveCatalog = (id: string) => { - saveCatalogs(catalogs.filter((c) => c.id !== id)); + useCustomOPDSStore.getState().removeCatalog(id); + persistMutation(); if (appService) { // Don't await — leftover state files are harmless and we don't want to // block UI removal if the filesystem call fails. @@ -248,8 +278,11 @@ export function CatalogManager() { }; const handleToggleAutoDownload = (id: string) => { - const wasEnabled = catalogs.find((c) => c.id === id)?.autoDownload; - saveCatalogs(catalogs.map((c) => (c.id === id ? { ...c, autoDownload: !c.autoDownload } : c))); + const target = catalogs.find((c) => c.id === id); + if (!target) return; + const wasEnabled = !!target.autoDownload; + useCustomOPDSStore.getState().updateCatalog(id, { autoDownload: !wasEnabled }); + persistMutation(); // When the user just enabled auto-download, sync now instead of waiting // for the next app launch / pull-to-refresh. if (!wasEnabled) { diff --git a/apps/readest-app/src/hooks/useReplicaPull.ts b/apps/readest-app/src/hooks/useReplicaPull.ts index ad78def4..8e5f9862 100644 --- a/apps/readest-app/src/hooks/useReplicaPull.ts +++ b/apps/readest-app/src/hooks/useReplicaPull.ts @@ -11,11 +11,13 @@ import { findTextureByContentId, migrateLegacyTextures, } from '@/store/customTextureStore'; +import { useCustomOPDSStore, findOPDSCatalogByContentId } from '@/store/customOPDSStore'; import { transferManager } from '@/services/transferManager'; import { getReplicaSync, subscribeReplicaSyncReady } from '@/services/sync/replicaSync'; import { dictionaryAdapter } from '@/services/sync/adapters/dictionary'; import { fontAdapter } from '@/services/sync/adapters/font'; import { textureAdapter } from '@/services/sync/adapters/texture'; +import { opdsCatalogAdapter } from '@/services/sync/adapters/opdsCatalog'; import { queueReplicaBinaryUpload } from '@/services/sync/replicaBinaryUpload'; import { replicaPullAndApply, @@ -31,8 +33,9 @@ import type { ReplicaSyncManager } from '@/services/sync/replicaSyncManager'; import type { ImportedDictionary } from '@/services/dictionaries/types'; import type { CustomFont } from '@/styles/fonts'; import type { CustomTexture } from '@/styles/textures'; +import type { OPDSCatalog } from '@/types/opds'; -export type ReplicaKind = 'dictionary' | 'font' | 'texture'; +export type ReplicaKind = 'dictionary' | 'font' | 'texture' | 'opds_catalog'; export interface UseReplicaPullOpts { /** Replica kinds this page wants pulled. */ @@ -60,7 +63,8 @@ const pulledKinds = new Set(); */ interface ReplicaPullConfig { kind: ReplicaKind; - baseDir: BaseDir; + /** Required for binary-bearing kinds; omitted for metadata-only kinds. */ + baseDir?: BaseDir; adapter: ReplicaAdapter; findByContentId: (id: string) => T | undefined; hydrateLocalStore?: (envConfig: EnvConfigType) => Promise; @@ -85,16 +89,22 @@ const buildReplicaPullDeps = ( : undefined, applyRemote: config.applyRemote, softDeleteByContentId: config.softDeleteByContentId, + // The bundle / binary callbacks below are only reached when the + // adapter declares a `binary` capability — replicaPullAndApply + // short-circuits metadata-only kinds before invoking them. The + // non-null assertion on baseDir is therefore safe in the binary + // path; metadata-only kinds (opds_catalog) leave config.baseDir + // unset and never hit these. createBundleDir: async () => { const id = uniqueId(); - await service.createDir(id, config.baseDir, true); + await service.createDir(id, config.baseDir!, true); return id; }, queueReplicaDownload: (contentId, displayTitle, files, _bundleDir, base) => transferManager.queueReplicaDownload(config.kind, contentId, displayTitle, files, base), filesExist: async (bundleDir, filenames) => { for (const filename of filenames) { - const exists = await service.exists(`${bundleDir}/${filename}`, config.baseDir); + const exists = await service.exists(`${bundleDir}/${filename}`, config.baseDir!); if (!exists) return false; } return true; @@ -153,6 +163,16 @@ const texturePullConfig: ReplicaPullConfig = { softDeleteByContentId: (id) => useCustomTextureStore.getState().softDeleteByContentId(id), }; +const opdsCatalogPullConfig: ReplicaPullConfig = { + kind: 'opds_catalog', + // metadata-only — no baseDir + adapter: opdsCatalogAdapter, + findByContentId: findOPDSCatalogByContentId, + hydrateLocalStore: (envConfig) => useCustomOPDSStore.getState().loadCustomOPDSCatalogs(envConfig), + applyRemote: (catalog) => useCustomOPDSStore.getState().applyRemoteCatalog(catalog), + softDeleteByContentId: (id) => useCustomOPDSStore.getState().softDeleteByContentId(id), +}; + const runPullForKind = async ( kind: ReplicaKind, service: AppService, @@ -179,6 +199,11 @@ const runPullForKind = async ( buildReplicaPullDeps(ctx.manager, service, envConfig, texturePullConfig), ); return; + case 'opds_catalog': + await replicaPullAndApply( + buildReplicaPullDeps(ctx.manager, service, envConfig, opdsCatalogPullConfig), + ); + return; } }; diff --git a/apps/readest-app/src/libs/replicaSchemas.ts b/apps/readest-app/src/libs/replicaSchemas.ts index 3b079157..c243c817 100644 --- a/apps/readest-app/src/libs/replicaSchemas.ts +++ b/apps/readest-app/src/libs/replicaSchemas.ts @@ -52,6 +52,19 @@ const textureFieldsSchema = z }) .catchall(fieldEnvelopeWithCipher); +const opdsCatalogFieldsSchema = z + .object({ + name: fieldEnvelopeSchema.optional(), + url: fieldEnvelopeSchema.optional(), + description: fieldEnvelopeSchema.optional(), + icon: fieldEnvelopeSchema.optional(), + customHeaders: fieldEnvelopeSchema.optional(), + autoDownload: fieldEnvelopeSchema.optional(), + disabled: fieldEnvelopeSchema.optional(), + addedAt: fieldEnvelopeSchema.optional(), + }) + .catchall(fieldEnvelopeWithCipher); + interface KindSpec { minSchemaVersion: number; maxSchemaVersion: number; @@ -82,6 +95,13 @@ export const KIND_ALLOWLIST: Record = { fields: textureFieldsSchema, binary: true, }, + opds_catalog: { + minSchemaVersion: 1, + maxSchemaVersion: 1, + maxRowsPerUser: 50, + fields: opdsCatalogFieldsSchema, + binary: false, + }, }; export const isAllowedKind = (kind: string): boolean => Object.hasOwn(KIND_ALLOWLIST, kind); diff --git a/apps/readest-app/src/services/sync/adapters/opdsCatalog.ts b/apps/readest-app/src/services/sync/adapters/opdsCatalog.ts new file mode 100644 index 00000000..0ee2d84d --- /dev/null +++ b/apps/readest-app/src/services/sync/adapters/opdsCatalog.ts @@ -0,0 +1,116 @@ +import { md5 } from '@/utils/md5'; +import type { OPDSCatalog } from '@/types/opds'; +import type { ReplicaAdapter } from '@/services/sync/replicaRegistry'; +import type { FieldsObject, ReplicaRow } from '@/types/replica'; +import { defaultComputeId, unwrap } from './helpers'; + +export const OPDS_CATALOG_KIND = 'opds_catalog'; +export const OPDS_CATALOG_SCHEMA_VERSION = 1; + +interface UnwrappedOpdsFields { + name?: string; + url?: string; + description?: string; + icon?: string; + customHeaders?: Record; + autoDownload?: boolean; + disabled?: boolean; + addedAt?: number; +} + +const unwrapOpdsFields = (fields: FieldsObject): UnwrappedOpdsFields => { + const name = unwrap(fields['name']); + const url = unwrap(fields['url']); + const description = unwrap(fields['description']); + const icon = unwrap(fields['icon']); + const customHeaders = unwrap(fields['customHeaders']); + const autoDownload = unwrap(fields['autoDownload']); + const disabled = unwrap(fields['disabled']); + const addedAt = unwrap(fields['addedAt']); + return { + name: typeof name === 'string' ? name : undefined, + url: typeof url === 'string' ? url : undefined, + description: typeof description === 'string' ? description : undefined, + icon: typeof icon === 'string' ? icon : undefined, + customHeaders: + customHeaders && typeof customHeaders === 'object' && !Array.isArray(customHeaders) + ? (customHeaders as Record) + : undefined, + autoDownload: autoDownload === true ? true : undefined, + disabled: disabled === true ? true : undefined, + addedAt: typeof addedAt === 'number' ? addedAt : undefined, + }; +}; + +/** + * Stable cross-device identity for an OPDS catalog. Two devices that import + * the same URL converge to a single replica row instead of duplicating. + * URL is normalized (trim + lower-case) so trailing-slash and case + * differences don't fragment identity. Username/password are intentionally + * excluded — encrypted-credential sync is in a follow-up PR; including + * username here would couple identity to a field that may not yet sync. + */ +export const computeOpdsCatalogContentId = (url: string): string => + md5(`opds:${url.trim().toLowerCase()}`); + +export const opdsCatalogAdapter: ReplicaAdapter = { + kind: OPDS_CATALOG_KIND, + schemaVersion: OPDS_CATALOG_SCHEMA_VERSION, + + pack(catalog: OPDSCatalog): Record { + const fields: Record = { + name: catalog.name, + url: catalog.url, + addedAt: catalog.addedAt ?? Date.now(), + }; + if (catalog.description !== undefined) fields['description'] = catalog.description; + if (catalog.icon !== undefined) fields['icon'] = catalog.icon; + if (catalog.customHeaders !== undefined) fields['customHeaders'] = catalog.customHeaders; + if (catalog.autoDownload !== undefined) fields['autoDownload'] = catalog.autoDownload; + if (catalog.disabled !== undefined) fields['disabled'] = catalog.disabled; + return fields; + }, + + unpack(fields: Record): OPDSCatalog { + return { + id: '', + name: String(fields['name'] ?? ''), + url: String(fields['url'] ?? ''), + description: fields['description'] !== undefined ? String(fields['description']) : undefined, + icon: fields['icon'] !== undefined ? String(fields['icon']) : undefined, + customHeaders: + fields['customHeaders'] && typeof fields['customHeaders'] === 'object' + ? (fields['customHeaders'] as Record) + : undefined, + autoDownload: fields['autoDownload'] === true ? true : undefined, + disabled: fields['disabled'] === true ? true : undefined, + addedAt: fields['addedAt'] !== undefined ? Number(fields['addedAt']) : undefined, + }; + }, + + computeId: defaultComputeId, + + unpackRow(row: ReplicaRow): OPDSCatalog | null { + const fields = unwrapOpdsFields(row.fields_jsonb); + if (!fields.name || !fields.url) return null; + const catalog: OPDSCatalog = { + // OPDS catalogs use contentId as their local id — they have no + // "bundle dir" pointer to disambiguate, and the URL-derived + // contentId is already a stable cross-device identifier. + id: row.replica_id, + contentId: row.replica_id, + name: fields.name, + url: fields.url, + }; + if (fields.description !== undefined) catalog.description = fields.description; + if (fields.icon !== undefined) catalog.icon = fields.icon; + if (fields.customHeaders !== undefined) catalog.customHeaders = fields.customHeaders; + if (fields.autoDownload !== undefined) catalog.autoDownload = fields.autoDownload; + if (fields.disabled !== undefined) catalog.disabled = fields.disabled; + if (fields.addedAt !== undefined) catalog.addedAt = fields.addedAt; + if (row.reincarnation) catalog.reincarnation = row.reincarnation; + return catalog; + }, + + // No `binary` capability — opds_catalog is metadata-only. +}; diff --git a/apps/readest-app/src/services/sync/replicaBootstrap.ts b/apps/readest-app/src/services/sync/replicaBootstrap.ts index bd511124..4ae51deb 100644 --- a/apps/readest-app/src/services/sync/replicaBootstrap.ts +++ b/apps/readest-app/src/services/sync/replicaBootstrap.ts @@ -4,6 +4,7 @@ import { useCustomTextureStore } from '@/store/customTextureStore'; import { dictionaryAdapter, DICTIONARY_KIND } from './adapters/dictionary'; import { fontAdapter, FONT_KIND } from './adapters/font'; import { textureAdapter, TEXTURE_KIND } from './adapters/texture'; +import { opdsCatalogAdapter } from './adapters/opdsCatalog'; import { getReplicaPersistEnv } from './replicaPersist'; import { getReplicaAdapter, registerReplicaAdapter } from './replicaRegistry'; import { registerReplicaDownloadHandler } from './replicaTransferIntegration'; @@ -13,6 +14,8 @@ const KNOWN_ADAPTERS: ReplicaAdapter[] = [ dictionaryAdapter as unknown as ReplicaAdapter, fontAdapter as unknown as ReplicaAdapter, textureAdapter as unknown as ReplicaAdapter, + // Metadata-only — no binary download handler needed. + opdsCatalogAdapter as unknown as ReplicaAdapter, ]; let didBootstrap = false; diff --git a/apps/readest-app/src/services/sync/replicaPullAndApply.ts b/apps/readest-app/src/services/sync/replicaPullAndApply.ts index 02fd939c..ab899e5f 100644 --- a/apps/readest-app/src/services/sync/replicaPullAndApply.ts +++ b/apps/readest-app/src/services/sync/replicaPullAndApply.ts @@ -109,22 +109,29 @@ const applyRow = async ( // Decide bundleDir + display name. If a local entry already maps this // contentId, reuse its bundleDir so we don't orphan the previously // downloaded binaries; otherwise mint a fresh dir and apply the remote - // record to the local store. Legacy local records (pre-replica-sync) - // may carry no bundleDir — skip them; they aren't sync-eligible. + // record to the local store. Legacy binary-kind records (pre-replica- + // sync) may carry no bundleDir — skip them; they aren't sync-eligible. + // Metadata-only kinds (no `binary` capability, e.g. opds_catalog) have + // no on-disk anchor at all, so the bundleDir requirement is dropped. + const needsBundleDir = !!deps.adapter.binary; let bundleDir: string; let displayName: string; if (local) { - if (!local.bundleDir) return; - bundleDir = local.bundleDir; + if (needsBundleDir && !local.bundleDir) return; + bundleDir = local.bundleDir ?? ''; displayName = deps.adapter.getDisplayName?.(local) ?? local.name; } else { - bundleDir = await deps.createBundleDir(); + bundleDir = needsBundleDir ? await deps.createBundleDir() : ''; const record = deps.adapter.unpackRow(row, bundleDir); if (!record) return; deps.applyRemote(record); displayName = deps.adapter.getDisplayName?.(record) ?? record.name; } + // Metadata-only kinds: nothing more to do. The orchestrator's manifest + // / binary-download path is a no-op for them. + if (!needsBundleDir) return; + if (!row.manifest_jsonb || row.manifest_jsonb.files.length === 0) { // Server row has no manifest yet — typically the device that // wrote the metadata never finished the binary upload (TM wasn't diff --git a/apps/readest-app/src/store/customOPDSStore.ts b/apps/readest-app/src/store/customOPDSStore.ts new file mode 100644 index 00000000..1c5b2c7f --- /dev/null +++ b/apps/readest-app/src/store/customOPDSStore.ts @@ -0,0 +1,250 @@ +import { create } from 'zustand'; +import type { EnvConfigType } from '@/services/environment'; +import type { OPDSCatalog } from '@/types/opds'; +import { useSettingsStore } from './settingsStore'; +import { getReplicaPersistEnv } from '@/services/sync/replicaPersist'; +import { publishReplicaDelete, publishReplicaUpsert } from '@/services/sync/replicaPublish'; +import { + computeOpdsCatalogContentId, + OPDS_CATALOG_KIND, +} from '@/services/sync/adapters/opdsCatalog'; + +const publishOpdsUpsert = (catalog: OPDSCatalog): void => { + if (!catalog.contentId) return; + void publishReplicaUpsert(OPDS_CATALOG_KIND, catalog, catalog.contentId, catalog.reincarnation); +}; + +const publishOpdsDelete = (contentId: string): void => { + void publishReplicaDelete(OPDS_CATALOG_KIND, contentId); +}; + +/** + * Backfill `contentId` (and `addedAt`) on legacy catalogs that predate + * replica sync. Returns the same array reference if no changes were + * required so callers can cheaply detect a no-op. + * + * `addedAt` is assigned per array index so the existing display order + * survives the migration: index 0 (newest in the legacy array) gets + * the largest timestamp, index N gets the smallest. The total span is + * tiny (≤ N ms) so newly-imported catalogs (with `Date.now()`) still + * sort above the migrated set, which matches the legacy "prepend new + * entries" UX. + */ +const backfillSyncFields = (catalogs: OPDSCatalog[]): OPDSCatalog[] => { + let mutated = false; + const baseTime = Date.now(); + const next = catalogs.map((c, i) => { + if (c.contentId && c.addedAt !== undefined) return c; + mutated = true; + return { + ...c, + contentId: c.contentId ?? computeOpdsCatalogContentId(c.url), + addedAt: c.addedAt ?? baseTime - i, + }; + }); + return mutated ? next : catalogs; +}; + +interface OPDSStoreState { + catalogs: OPDSCatalog[]; + loading: boolean; + + /** Visible catalogs sorted by `addedAt` descending (newest first). */ + getAvailableCatalogs(): OPDSCatalog[]; + getCatalog(id: string): OPDSCatalog | undefined; + /** Look up by URL — used for popular-catalog dedup (independent of contentId). */ + findByUrl(url: string): OPDSCatalog | undefined; + /** Look up by stable cross-device content id. */ + findByContentId(contentId: string): OPDSCatalog | undefined; + + /** + * Add (or revive) a catalog. Computes `contentId` from URL if absent. + * Re-import of a previously soft-deleted entry mints a reincarnation + * token so the server-side tombstone is replaced with a fresh row. + */ + addCatalog(catalog: Omit & { contentId?: string }): OPDSCatalog; + /** + * Patch a catalog's mutable fields. Only the patched fields are + * republished — credentials (username/password) are NOT in the + * synced field set yet, so editing them stays local-only until the + * encrypted-field PR lands. + */ + updateCatalog(id: string, patch: Partial): OPDSCatalog | undefined; + /** Soft-delete by id; pushes a tombstone if the entry has a contentId. */ + removeCatalog(id: string): boolean; + + /** + * Add a catalog received via replica sync from another device. Same + * effect on local state as addCatalog, but does NOT republish. + */ + applyRemoteCatalog(catalog: OPDSCatalog): void; + /** Mirror a server-side tombstone locally without re-publishing. */ + softDeleteByContentId(contentId: string): void; + + /** Hydrate from `settings.opdsCatalogs`. Backfills sync fields if needed. */ + loadCustomOPDSCatalogs(envConfig: EnvConfigType): Promise; + /** Persist current state back into settings. */ + saveCustomOPDSCatalogs(envConfig: EnvConfigType): Promise; +} + +export const useCustomOPDSStore = create((set, get) => ({ + catalogs: [], + loading: false, + + getAvailableCatalogs: () => + get() + .catalogs.filter((c) => !c.deletedAt) + .sort((a, b) => (b.addedAt || 0) - (a.addedAt || 0)), + + getCatalog: (id) => get().catalogs.find((c) => c.id === id), + + findByUrl: (url) => { + const normalized = url.trim().toLowerCase(); + return get().catalogs.find((c) => c.url.trim().toLowerCase() === normalized && !c.deletedAt); + }, + + findByContentId: (contentId) => + contentId ? get().catalogs.find((c) => c.contentId === contentId) : undefined, + + addCatalog: (input) => { + const contentId = input.contentId ?? computeOpdsCatalogContentId(input.url); + // Revive tombstoned entry under a reincarnation token so the + // server-side tombstone gets replaced rather than stuck. + const existing = get().catalogs.find((c) => c.contentId === contentId); + const reincarnation = + existing?.deletedAt && !input.reincarnation + ? Math.random().toString(36).slice(2) + : input.reincarnation; + const catalog: OPDSCatalog = { + ...input, + contentId, + addedAt: input.addedAt ?? existing?.addedAt ?? Date.now(), + deletedAt: undefined, + reincarnation, + }; + set((state) => { + const idx = state.catalogs.findIndex((c) => c.contentId === contentId); + const catalogs = + idx >= 0 + ? state.catalogs.map((c, i) => (i === idx ? catalog : c)) + : [...state.catalogs, catalog]; + return { catalogs }; + }); + publishOpdsUpsert(catalog); + return catalog; + }, + + updateCatalog: (id, patch) => { + let updated: OPDSCatalog | undefined; + set((state) => { + const idx = state.catalogs.findIndex((c) => c.id === id); + if (idx < 0) return state; + const old = state.catalogs[idx]!; + if (old.deletedAt) return state; + updated = { ...old, ...patch }; + // Recompute contentId only if the URL itself changed; otherwise + // preserve the existing one so we keep the same server row. + if (patch.url && patch.url !== old.url) { + updated.contentId = computeOpdsCatalogContentId(patch.url); + } + return { + catalogs: state.catalogs.map((c, i) => (i === idx ? updated! : c)), + }; + }); + if (updated) publishOpdsUpsert(updated); + return updated; + }, + + removeCatalog: (id) => { + const catalog = get().catalogs.find((c) => c.id === id); + if (!catalog) return false; + set((state) => ({ + catalogs: state.catalogs.map((c) => (c.id === id ? { ...c, deletedAt: Date.now() } : c)), + })); + if (catalog.contentId) publishOpdsDelete(catalog.contentId); + return true; + }, + + applyRemoteCatalog: (catalog) => { + set((state) => { + const idx = state.catalogs.findIndex((c) => c.contentId === catalog.contentId); + if (idx >= 0) { + // Preserve local-only fields (username/password — not yet in the + // synced field set) when overlaying a remote update. + const old = state.catalogs[idx]!; + const merged: OPDSCatalog = { + ...catalog, + username: old.username, + password: old.password, + deletedAt: undefined, + }; + return { catalogs: state.catalogs.map((c, i) => (i === idx ? merged : c)) }; + } + return { catalogs: [...state.catalogs, catalog] }; + }); + const env = getReplicaPersistEnv(); + if (env) void get().saveCustomOPDSCatalogs(env); + }, + + softDeleteByContentId: (contentId) => { + const target = get().catalogs.find((c) => c.contentId === contentId && !c.deletedAt); + if (!target) return; + set((state) => ({ + catalogs: state.catalogs.map((c) => + c.id === target.id ? { ...c, deletedAt: Date.now() } : c, + ), + })); + const env = getReplicaPersistEnv(); + if (env) void get().saveCustomOPDSCatalogs(env); + }, + + loadCustomOPDSCatalogs: async (_envConfig) => { + try { + const { settings } = useSettingsStore.getState(); + const persisted = settings?.opdsCatalogs ?? []; + const backfilled = backfillSyncFields(persisted); + set({ catalogs: backfilled }); + // If backfill mutated anything, persist + publish the fresh + // contentIds so existing catalogs start syncing on next push. + if (backfilled !== persisted) { + await get().saveCustomOPDSCatalogs(_envConfig); + for (const c of backfilled) { + if (c.contentId && !c.deletedAt) publishOpdsUpsert(c); + } + } + } catch (error) { + console.error('Failed to load OPDS catalogs:', error); + } + }, + + saveCustomOPDSCatalogs: async (_envConfig) => { + try { + const { settings, setSettings, saveSettings } = useSettingsStore.getState(); + const { catalogs } = get(); + // Tombstoned entries stay in memory so the orchestrator can detect + // re-import / reincarnation, but they're stripped at the + // persistence boundary. The next pull will mirror server-side + // tombstones back into memory if the row is still deleted. + settings.opdsCatalogs = catalogs.filter((c) => !c.deletedAt); + setSettings(settings); + saveSettings(_envConfig, settings); + } catch (error) { + console.error('Failed to save OPDS catalogs:', error); + throw error; + } + }, +})); + +/** + * Look up an OPDS catalog by its cross-device contentId, falling back to + * the persisted settings when the in-memory store is empty. The pull-side + * orchestrator runs before the OPDS page mounts; without the fallback + * every refresh would treat existing catalogs as new and double up. + */ +export const findOPDSCatalogByContentId = (contentId: string): OPDSCatalog | undefined => { + if (!contentId) return undefined; + const inMemory = useCustomOPDSStore.getState().findByContentId(contentId); + if (inMemory) return inMemory; + const persisted = useSettingsStore.getState().settings?.opdsCatalogs ?? []; + return persisted.find((c) => c.contentId === contentId && !c.deletedAt); +}; diff --git a/apps/readest-app/src/types/opds.ts b/apps/readest-app/src/types/opds.ts index cf1f5e0c..89edb114 100644 --- a/apps/readest-app/src/types/opds.ts +++ b/apps/readest-app/src/types/opds.ts @@ -30,6 +30,19 @@ export interface OPDSCatalog { password?: string; customHeaders?: Record; autoDownload?: boolean; + /** + * Stable cross-device identifier derived from the URL. Used as the + * replica_id so two devices that import the same OPDS URL converge to a + * single row instead of duplicating. Absent on legacy entries imported + * before replica sync shipped — they get backfilled at next save. + */ + contentId?: string; + /** Wall-clock ms of first import, used for ordering. */ + addedAt?: number; + /** Soft-delete timestamp; non-null entries are hidden from the UI. */ + deletedAt?: number; + /** Reincarnation token (re-import after server tombstone). */ + reincarnation?: string; } export interface OPDSFeed { diff --git a/docker/volumes/db/migrations/009_replica_opds_catalog.sql b/docker/volumes/db/migrations/009_replica_opds_catalog.sql new file mode 100644 index 00000000..4eed3506 --- /dev/null +++ b/docker/volumes/db/migrations/009_replica_opds_catalog.sql @@ -0,0 +1,11 @@ +-- Migration 009: Extend the replicas.kind allowlist with 'opds_catalog'. +-- The DB CHECK is belt-and-suspenders; src/libs/replicaSchemas.ts +-- (KIND_ALLOWLIST) is the actual gate that decides which kinds the +-- server accepts on push. + +ALTER TABLE public.replicas + DROP CONSTRAINT IF EXISTS replicas_kind_allowlist; + +ALTER TABLE public.replicas + ADD CONSTRAINT replicas_kind_allowlist + CHECK (kind IN ('dictionary', 'font', 'texture', 'opds_catalog'));