From 5f44c955929e29e005495eba2661c0301dd59332 Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Sun, 28 Jun 2026 17:20:32 +0800 Subject: [PATCH] feat(sync): library-scoped auto-sync for third-party cloud (WebDAV / Drive) (#4835) Parity with native useBooksSync: keep library.json current on import, delete, and book-close, not just on a manual "Sync now". - useLibraryFileSync: new library-scoped hook (counterpart of useBooksSync), mounted once on the library page. Builds the active provider's engine async and runs engine.syncLibrary on every library change (import adds a row, delete sets deletedAt, closing a book bumps updatedAt), debounced 5s and gated on the global file-sync mutex + Sync Strategy + Upload Book Files. The reader's per-book useFileSync is unchanged (it's the per-book progress sync). - Pass the FULL library (incl. soft-deleted books) to engine.syncLibrary, in both the new hook and the manual FileSyncForm "Sync now": the engine tombstones deleted books in library.json so deletions propagate, and keeping them in the input set stops the discovery pass from re-downloading a book the user just deleted (its remote hash dir lingers until the GC sweep). - Tests: engine tombstones a soft-deleted book in the pushed index and does not re-download one whose remote dir still exists. Gated only by the active provider's enabled flag + strategy (cloud sync is currently ungated from premium). Never runs before the library loads from disk, so it can't push an empty index over the remote. Co-authored-by: Claude Opus 4.8 (1M context) --- .../sync/file/engine-metadata-sync.test.ts | 48 +++++ .../app/library/hooks/useLibraryFileSync.ts | 188 ++++++++++++++++++ apps/readest-app/src/app/library/page.tsx | 5 + .../settings/integrations/FileSyncForm.tsx | 10 +- 4 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 apps/readest-app/src/app/library/hooks/useLibraryFileSync.ts diff --git a/apps/readest-app/src/__tests__/services/sync/file/engine-metadata-sync.test.ts b/apps/readest-app/src/__tests__/services/sync/file/engine-metadata-sync.test.ts index 02068867..98e0a300 100644 --- a/apps/readest-app/src/__tests__/services/sync/file/engine-metadata-sync.test.ts +++ b/apps/readest-app/src/__tests__/services/sync/file/engine-metadata-sync.test.ts @@ -152,6 +152,54 @@ describe('FileSyncEngine metadata reconciliation (#4756)', () => { }); }); +describe('FileSyncEngine soft-delete propagation', () => { + test('tombstones a soft-deleted book in the pushed index', async () => { + const deleted = makeLocalBook({ deletedAt: 500, updatedAt: 100 }); + const capture: { index?: RemoteLibraryIndex | null } = {}; + const provider = makeProvider(null, null, capture); + + const engine = new FileSyncEngine(provider, makeStore()); + await engine.syncLibrary([deleted], { + strategy: 'silent', + syncBooks: false, + deviceId: 'd1', + }); + + // The deletion travels to peers as a tombstone in library.json. + const indexed = capture.index!.books.find((b) => b.hash === 'h1')!; + expect(indexed.deletedAt).toBe(500); + }); + + test('does not re-download a soft-deleted book whose remote dir still exists', async () => { + const deleted = makeLocalBook({ deletedAt: 500, updatedAt: 100 }); + const capture: { index?: RemoteLibraryIndex | null } = {}; + const provider = makeProvider(null, null, capture); + // The GC sweep is separate, so the deleted book's hash dir + file still + // exist remotely. Passing the deleted book in (it stays in allBooksMap) + // must keep the discovery pass from re-adding it as a download candidate. + provider.list = vi.fn(async (path: string) => { + if (path.endsWith('/books')) + return [{ name: 'h1', path: '/Readest/books/h1', isDirectory: true }]; + if (path.endsWith('/h1')) + return [{ name: 'book.epub', path: '/Readest/books/h1/book.epub', isDirectory: false }]; + return []; + }); + const addBookToLibrary = vi.fn(async (_book: Book) => {}); + const store = makeStore({ addBookToLibrary }); + + const engine = new FileSyncEngine(provider, store); + const result = await engine.syncLibrary([deleted], { + strategy: 'silent', + syncBooks: false, + deviceId: 'd1', + }); + + expect(addBookToLibrary).not.toHaveBeenCalled(); + expect(result.booksDownloaded).toBe(0); + expect(capture.index!.books.find((b) => b.hash === 'h1')!.deletedAt).toBe(500); + }); +}); + describe('FileSyncEngine config merge before push (Sync now must not blind-overwrite)', () => { test('unions remote booknotes into the pushed config instead of clobbering them', async () => { // Local book is newer than the index, so incremental includes it in the diff --git a/apps/readest-app/src/app/library/hooks/useLibraryFileSync.ts b/apps/readest-app/src/app/library/hooks/useLibraryFileSync.ts new file mode 100644 index 00000000..2a69b765 --- /dev/null +++ b/apps/readest-app/src/app/library/hooks/useLibraryFileSync.ts @@ -0,0 +1,188 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { v4 as uuidv4 } from 'uuid'; +import { useEnv } from '@/context/EnvContext'; +import { useTranslation } from '@/hooks/useTranslation'; +import { useQuotaStats } from '@/hooks/useQuotaStats'; +import { useSettingsStore } from '@/store/settingsStore'; +import { useLibraryStore } from '@/store/libraryStore'; +import { useFileSyncStore } from '@/store/fileSyncStore'; +import { isCloudSyncAllowed } from '@/utils/access'; +import { debounce } from '@/utils/debounce'; +import { FileSyncEngine } from '@/services/sync/file/engine'; +import { createAppLocalStore } from '@/services/sync/file/appLocalStore'; +import { + createFileSyncProvider, + type FileSyncBackendKind, +} from '@/services/sync/file/providerRegistry'; + +/** + * Library-scoped auto-sync for the active third-party cloud provider (WebDAV / + * Google Drive) — the parity counterpart of {@link useBooksSync} (native cloud). + * + * The reader's per-book `useFileSync` keeps a book's progress/notes in sync while + * reading, but never touches the shared `library.json` index. This hook fills + * that gap: it runs the active provider's `engine.syncLibrary` whenever the + * library changes — importing, deleting, or closing a book all mutate the + * library array — so `library.json` (book metadata + tombstones) stays current + * across devices, exactly as native sync does. Mount once on the library page. + * + * Behaviour: + * - Debounced to collapse import bursts (and the engine's own + * pull-merge writes, which re-fire this effect — debounce + the engine's + * incremental cursor make repeats converge to a no-op). + * - Gated on the global file-sync mutex so it never collides with a manual + * "Sync now" or another backend reconciling the same local library. + * - Honours the provider's Sync Strategy (send / receive / silent) and the + * "Upload Book Files" toggle, same as the manual sync. + * - Passes the FULL library, including soft-deleted books, so deletions + * propagate as tombstones in `library.json` and deleted books are not + * re-discovered and re-downloaded. + */ + +/** Quiet window before an auto library sync fires; collapses import bursts. */ +const SYNC_DEBOUNCE_MS = 5_000; + +const settingsKeyFor = (kind: FileSyncBackendKind): 'webdav' | 'googleDrive' => + kind === 'gdrive' ? 'googleDrive' : 'webdav'; + +export const useLibraryFileSync = () => { + const _ = useTranslation(); + const { envConfig, appService } = useEnv(); + const { setSettings, saveSettings } = useSettingsStore(); + const settings = useSettingsStore((s) => s.settings); + const library = useLibraryStore((s) => s.library); + const libraryLoaded = useLibraryStore((s) => s.libraryLoaded); + const { userProfilePlan } = useQuotaStats(); + + // The single active cloud provider (WebDAV and Google Drive are exclusive). + const activeKind: FileSyncBackendKind | null = settings.webdav?.enabled + ? 'webdav' + : settings.googleDrive?.enabled + ? 'gdrive' + : null; + + const isAllowed = isCloudSyncAllowed(userProfilePlan ?? 'free'); + const isReady = useMemo(() => { + if (!isAllowed) return false; + if (activeKind === 'webdav') { + const w = settings.webdav; + return !!(w?.enabled && w?.serverUrl && w?.username); + } + if (activeKind === 'gdrive') return !!settings.googleDrive?.enabled; + return false; + }, [isAllowed, activeKind, settings.webdav, settings.googleDrive]); + + // Build the engine async (Drive probes the OS keychain). Keyed on the + // connection-relevant settings so an unrelated write (e.g. lastSyncedAt) + // doesn't rebuild it — which for Drive would re-probe the keychain. + const engineKey = useMemo(() => { + if (activeKind === 'webdav') { + const w = settings.webdav; + return `webdav:${w?.enabled}:${w?.serverUrl}:${w?.username}:${w?.password}:${w?.rootPath}`; + } + if (activeKind === 'gdrive') return `gdrive:${settings.googleDrive?.enabled}`; + return 'none'; + }, [activeKind, settings.webdav, settings.googleDrive]); + + const [engine, setEngine] = useState(null); + useEffect(() => { + let cancelled = false; + setEngine(null); + if (!isReady || !appService || activeKind === null) return; + (async () => { + const current = useSettingsStore.getState().settings; + const provider = await createFileSyncProvider(activeKind, current); + if (cancelled || !provider) return; + const store = createAppLocalStore({ appService, settings: current, envConfig }); + setEngine(new FileSyncEngine(provider, store)); + })(); + return () => { + cancelled = true; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [engineKey, isReady, activeKind, appService, envConfig]); + + const ensureDeviceId = useCallback((): string => { + const latest = useSettingsStore.getState().settings; + const key = activeKind ? settingsKeyFor(activeKind) : 'webdav'; + let id = latest[key]?.deviceId; + if (!id) { + id = uuidv4(); + const next = { ...latest, [key]: { ...latest[key], deviceId: id } }; + setSettings(next); + saveSettings(envConfig, next); + } + return id; + }, [activeKind, envConfig, setSettings, saveSettings]); + + const updateLastSyncedAt = useCallback( + async (ts: number) => { + const latest = useSettingsStore.getState().settings; + const key = activeKind ? settingsKeyFor(activeKind) : 'webdav'; + const next = { ...latest, [key]: { ...latest[key], lastSyncedAt: ts } }; + setSettings(next); + await saveSettings(envConfig, next); + }, + [activeKind, envConfig, setSettings, saveSettings], + ); + + const runSync = useCallback(async () => { + if (!engine || activeKind === null || !isReady) return; + // NEVER sync before the library is loaded from disk: syncing a transient + // empty library could push an empty index and clobber the remote. + if (!useLibraryStore.getState().libraryLoaded) return; + + const kind = activeKind; + const latest = useSettingsStore.getState().settings; + const ps = kind === 'gdrive' ? latest.googleDrive : latest.webdav; + const strategy = ps?.strategy ?? 'silent'; + + const syncStore = useFileSyncStore.getState(); + // Honour the global library-sync mutex: a manual "Sync now" (or another + // backend) already reconciling the same local library wins; this auto-run + // skips and the next library change re-triggers it. + if (!syncStore.beginSync(kind, _('Syncing…'))) return; + try { + const books = useLibraryStore.getState().library; + const deviceId = ensureDeviceId(); + await engine.syncLibrary(books, { + strategy: strategy === 'prompt' ? 'silent' : strategy, + syncBooks: ps?.syncBooks ?? false, + fullSync: false, + deviceId, + onProgress: ({ index, total, action }) => { + const label = action === 'downloading' ? _('Downloading') : _('Uploading'); + useFileSyncStore + .getState() + .updateProgress( + kind, + _('{{action}} {{n}} / {{total}}', { action: label, n: index + 1, total }), + ); + }, + }); + await updateLastSyncedAt(Date.now()); + } catch (e) { + console.warn('library file sync failed', e); + } finally { + useFileSyncStore.getState().endSync(kind); + } + }, [engine, activeKind, isReady, ensureDeviceId, updateLastSyncedAt, _]); + + // Keep one stable debounced trigger that always calls the latest runSync (via + // ref), so it isn't recreated — and lost — on every settings/engine change. + const runSyncRef = useRef(runSync); + runSyncRef.current = runSync; + const debouncedSync = useMemo( + () => debounce(() => void runSyncRef.current(), SYNC_DEBOUNCE_MS), + [], + ); + useEffect(() => () => debouncedSync.cancel(), [debouncedSync]); + + // Library changes — import (adds a row), delete (sets deletedAt), book close + // (bumps updatedAt) — all mutate `library`, so this single effect covers them + // plus the initial load pull. + useEffect(() => { + if (!isReady || !engine || !libraryLoaded) return; + debouncedSync(); + }, [library, libraryLoaded, isReady, engine, debouncedSync]); +}; diff --git a/apps/readest-app/src/app/library/page.tsx b/apps/readest-app/src/app/library/page.tsx index 01f3f898..46bbd9fa 100644 --- a/apps/readest-app/src/app/library/page.tsx +++ b/apps/readest-app/src/app/library/page.tsx @@ -36,6 +36,7 @@ import { useTheme } from '@/hooks/useTheme'; import { useUICSS } from '@/hooks/useUICSS'; import { useDemoBooks } from './hooks/useDemoBooks'; import { useBooksSync } from './hooks/useBooksSync'; +import { useLibraryFileSync } from './hooks/useLibraryFileSync'; import { useInboxDrainer } from '@/hooks/useInboxDrainer'; import { useOPDSSubscriptions } from '@/hooks/useOPDSSubscriptions'; import { useBookDataStore } from '@/store/bookDataStore'; @@ -270,6 +271,10 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP useTransferQueue(libraryLoaded); const { pullLibrary, pushLibrary } = useBooksSync(); + // Library-scoped auto-sync for the active third-party cloud provider (WebDAV / + // Google Drive): keeps library.json current on import / delete / book-close, + // parity with useBooksSync. No-op when no provider is enabled. + useLibraryFileSync(); const { checkOPDSSubscriptions } = useOPDSSubscriptions(); useInboxDrainer(); const { isDragging } = useDragDropImport(); diff --git a/apps/readest-app/src/components/settings/integrations/FileSyncForm.tsx b/apps/readest-app/src/components/settings/integrations/FileSyncForm.tsx index dc1acefb..56cebe56 100644 --- a/apps/readest-app/src/components/settings/integrations/FileSyncForm.tsx +++ b/apps/readest-app/src/components/settings/integrations/FileSyncForm.tsx @@ -106,7 +106,11 @@ const FileSyncForm: React.FC = ({ kind, stored, persist }) => useLibraryStore.getState().setLibrary(currentLibrary); } - const eligibleBooks = currentLibrary.filter((b) => !b.deletedAt); + // Count only live books for the progress label, but sync the FULL library + // (including soft-deleted books): the engine tombstones deleted books in + // library.json so deletions propagate, and keeping them in the input set + // stops the discovery pass from re-downloading a book the user just deleted. + const liveBookCount = currentLibrary.filter((b) => !b.deletedAt).length; // Lazily ensure a deviceId so the first cross-device sync attributes its // rows correctly (the reader hook also touches this on first push). @@ -118,7 +122,7 @@ const FileSyncForm: React.FC = ({ kind, stored, persist }) => // Acquire the global library-sync mutex; bail if another backend's Sync now // is already mutating the local library. - if (!beginSync(kind, _('Syncing {{n}} / {{total}}', { n: 0, total: eligibleBooks.length }))) { + if (!beginSync(kind, _('Syncing {{n}} / {{total}}', { n: 0, total: liveBookCount }))) { return; } @@ -129,7 +133,7 @@ const FileSyncForm: React.FC = ({ kind, stored, persist }) => } const store = createAppLocalStore({ appService, settings, envConfig }); const engine = new FileSyncEngine(provider, store); - const result = await engine.syncLibrary(eligibleBooks, { + const result = await engine.syncLibrary(currentLibrary, { strategy: stored.strategy === 'prompt' ? 'silent' : stored.strategy, syncBooks: stored.syncBooks ?? false, fullSync: stored.fullSync ?? false,