diff --git a/apps/readest-app/src/__tests__/services/sync/file/appLocalStore.test.ts b/apps/readest-app/src/__tests__/services/sync/file/appLocalStore.test.ts new file mode 100644 index 00000000..fd2226f5 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/file/appLocalStore.test.ts @@ -0,0 +1,87 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +import type { Book } from '@/types/book'; +import type { AppService } from '@/types/system'; +import type { SystemSettings } from '@/types/settings'; +import type { EnvConfigType } from '@/services/environment'; +import { useLibraryStore } from '@/store/libraryStore'; +import { createAppLocalStore } from '@/services/sync/file/appLocalStore'; + +/** + * Regression test for the library-clobber data-loss path: when "Sync now" + * runs while the library store hasn't loaded yet (the app launched straight + * into the reader / settings, never mounting the Library view), the engine's + * addBookToLibrary / updateBookMetadata used to merge against the EMPTY + * in-memory library and persist a downloaded book (or a metadata update) as + * the *entire* library, wiping everything already on disk. The store bridge + * now hydrates from disk first. + */ + +const makeBook = (hash: string, overrides: Partial = {}): Book => ({ + hash, + format: 'EPUB', + title: `Book ${hash}`, + sourceTitle: `Book ${hash}`, + author: 'A', + createdAt: 1, + updatedAt: 1, + ...overrides, +}); + +let savedLibrary: Book[] | null; +let appService: AppService; +let envConfig: EnvConfigType; + +const onDisk = [makeBook('a'), makeBook('b')]; + +beforeEach(() => { + savedLibrary = null; + // Simulate the unloaded store: the user hasn't visited the Library view. + useLibraryStore.setState({ library: [], libraryLoaded: false, hashIndex: new Map() }); + appService = { + loadLibraryBooks: vi.fn(async () => onDisk.map((b) => ({ ...b }))), + saveLibraryBooks: vi.fn(async (books: Book[]) => { + savedLibrary = books; + }), + generateCoverImageUrl: vi.fn(async () => 'blob:cover'), + } as unknown as AppService; + envConfig = { getAppService: async () => appService } as unknown as EnvConfigType; +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +const makeStore = () => + createAppLocalStore({ appService, settings: {} as SystemSettings, envConfig }); + +describe('createAppLocalStore — library hydration (data-loss guard)', () => { + test('addBookToLibrary keeps existing on-disk books when the store is unloaded', async () => { + await makeStore().addBookToLibrary(makeBook('c')); + + expect(appService.loadLibraryBooks).toHaveBeenCalledTimes(1); + expect(savedLibrary).not.toBeNull(); + const hashes = savedLibrary!.map((b) => b.hash).sort(); + // The downloaded book is appended; a, b must survive (no clobber to [c]). + expect(hashes).toEqual(['a', 'b', 'c']); + }); + + test('updateBookMetadata does not wipe the library when the store is unloaded', async () => { + await makeStore().updateBookMetadata(makeBook('a', { title: 'New Title', updatedAt: 9 })); + + expect(appService.loadLibraryBooks).toHaveBeenCalled(); + expect(savedLibrary).not.toBeNull(); + const hashes = savedLibrary!.map((b) => b.hash).sort(); + // Both books survive; a is updated in place, b is untouched. + expect(hashes).toEqual(['a', 'b']); + expect(savedLibrary!.find((b) => b.hash === 'a')!.title).toBe('New Title'); + }); + + test('addBookToLibrary merges against an already-loaded store without reloading', async () => { + useLibraryStore.getState().setLibrary([makeBook('a'), makeBook('b')]); + await makeStore().addBookToLibrary(makeBook('c')); + + expect(appService.loadLibraryBooks).not.toHaveBeenCalled(); + expect(savedLibrary!.map((b) => b.hash).sort()).toEqual(['a', 'b', 'c']); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/file/engine-metadata-sync.test.ts b/apps/readest-app/src/__tests__/services/sync/file/engine-metadata-sync.test.ts new file mode 100644 index 00000000..02068867 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/file/engine-metadata-sync.test.ts @@ -0,0 +1,228 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +import type { Book, BookConfig, BookNote } from '@/types/book'; +import { FileSyncEngine } from '@/services/sync/file/engine'; +import type { FileSyncProvider } from '@/services/sync/file/provider'; +import type { LocalStore } from '@/services/sync/file/localStore'; +import type { RemoteBookConfig, RemoteLibraryIndex } from '@/services/sync/file/wire'; + +/** + * Behavior-preservation gate for the WebDAV→FileSyncEngine port (originally + * webdav-metadata-sync.test.ts for issue #4756). Instead of mocking the + * WebDAV transport client, we drive `FileSyncEngine` with a fake + * `FileSyncProvider` (routed by path) and a fake `LocalStore`, asserting the + * same last-writer-wins reconciliation on `book.updatedAt` and the same + * pull-merge-before-push discipline. + */ + +const makeLocalBook = (overrides: Partial = {}): Book => ({ + hash: 'h1', + format: 'EPUB', + title: 'Old Title', + sourceTitle: 'Old Title', + author: 'Old Author', + createdAt: 1, + updatedAt: 100, + ...overrides, +}); + +const makeRemoteIndex = (book: Book, updatedAt = book.updatedAt): RemoteLibraryIndex => ({ + schemaVersion: 1, + updatedAt, + books: [book], +}); + +const makeNote = (id: string, updatedAt: number): BookNote => ({ + id, + type: 'annotation', + cfi: `cfi-${id}`, + note: '', + createdAt: updatedAt, + updatedAt, +}); + +const makeRemoteConfig = (overrides: Partial = {}): RemoteBookConfig => ({ + schemaVersion: 1, + bookHash: 'h1', + config: { updatedAt: 100 }, + booknotes: [], + writerDeviceId: 'mobile', + writerVersion: 'readest-webdav-1', + updatedAt: 100, + ...overrides, +}); + +/** + * A fake provider routed by path: readText resolves the index for + * library.json and the supplied envelope for config.json. writeText captures + * whichever artefact a test cares about. No streaming methods, so the engine + * uses the buffered path (and the store's null loaders skip uploads). + */ +const makeProvider = ( + index: RemoteLibraryIndex | null, + remoteConfig: RemoteBookConfig | null, + capture: { index?: RemoteLibraryIndex | null; config?: RemoteBookConfig | null }, +): FileSyncProvider => ({ + rootPath: '/', + readText: vi.fn(async (path: string) => { + if (path.endsWith('library.json')) return index ? JSON.stringify(index) : null; + if (path.endsWith('config.json')) return remoteConfig ? JSON.stringify(remoteConfig) : null; + return null; + }), + readBinary: vi.fn(async () => new ArrayBuffer(8)), + head: vi.fn(async () => null), + list: vi.fn(async () => []), + writeText: vi.fn(async (path: string, body: string) => { + if (path.endsWith('library.json')) capture.index = JSON.parse(body) as RemoteLibraryIndex; + if (path.endsWith('config.json')) capture.config = JSON.parse(body) as RemoteBookConfig; + }), + writeBinary: vi.fn(async () => {}), + ensureDir: vi.fn(async () => {}), + deleteDir: vi.fn(async () => {}), +}); + +const makeStore = (overrides: Partial = {}): LocalStore => ({ + loadConfig: async (): Promise => ({ updatedAt: 50, booknotes: [] }), + saveBookConfig: async () => {}, + loadBookFile: async () => null, + resolveLocalBookPath: async () => null, + saveBookFile: async () => {}, + prepareLocalBookPath: async () => '/local/path', + loadBookCover: async () => null, + saveBookCover: async () => {}, + addBookToLibrary: async () => {}, + updateBookMetadata: async () => {}, + ...overrides, +}); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe('FileSyncEngine metadata reconciliation (#4756)', () => { + test('pulls newer remote metadata for a book the device already has', async () => { + const local = makeLocalBook({ updatedAt: 100 }); + const remote = makeLocalBook({ title: 'New Title', author: 'New Author', updatedAt: 200 }); + const capture: { index?: RemoteLibraryIndex | null } = {}; + const provider = makeProvider(makeRemoteIndex(remote, 200), null, capture); + + const updateBookMetadata = vi.fn(async (_book: Book) => {}); + const saveBookCover = vi.fn(async (_book: Book, _bytes: ArrayBuffer) => {}); + const store = makeStore({ updateBookMetadata, saveBookCover }); + + const engine = new FileSyncEngine(provider, store); + const result = await engine.syncLibrary([local], { + strategy: 'silent', + syncBooks: false, + deviceId: 'pc-device', + }); + + expect(updateBookMetadata).toHaveBeenCalledTimes(1); + const merged = updateBookMetadata.mock.calls[0]![0]; + expect(merged.title).toBe('New Title'); + expect(merged.author).toBe('New Author'); + expect(result.metadataUpdated).toBe(1); + expect(saveBookCover).toHaveBeenCalledTimes(1); + + expect(capture.index).not.toBeNull(); + const indexedBook = capture.index!.books.find((b) => b.hash === 'h1')!; + expect(indexedBook.title).toBe('New Title'); + }); + + test('does not overwrite local metadata when the local copy is newer', async () => { + const local = makeLocalBook({ title: 'Local Newer', updatedAt: 300 }); + const remote = makeLocalBook({ title: 'Remote Older', updatedAt: 200 }); + const capture: { index?: RemoteLibraryIndex | null } = {}; + const provider = makeProvider(makeRemoteIndex(remote, 200), null, capture); + + const updateBookMetadata = vi.fn(async (_book: Book) => {}); + const store = makeStore({ updateBookMetadata }); + + const engine = new FileSyncEngine(provider, store); + const result = await engine.syncLibrary([local], { + strategy: 'silent', + syncBooks: false, + deviceId: 'pc-device', + }); + + expect(updateBookMetadata).not.toHaveBeenCalled(); + expect(result.metadataUpdated).toBe(0); + const indexedBook = capture.index!.books.find((b) => b.hash === 'h1')!; + expect(indexedBook.title).toBe('Local Newer'); + }); +}); + +describe('FileSyncEngine config merge before push (Sync now must not blind-overwrite)', () => { + test('unions remote booknotes into the pushed config instead of clobbering them', async () => { + // Local book is newer than the index, so incremental includes it in the + // push set (the only case where merge-before-push matters); the metadata + // pass stays a no-op since the index copy isn't newer. + const local = makeLocalBook({ updatedAt: 200 }); + const capture: { config?: RemoteBookConfig | null } = {}; + const provider = makeProvider( + makeRemoteIndex(makeLocalBook({ updatedAt: 100 }), 100), + makeRemoteConfig({ config: { updatedAt: 100 }, booknotes: [makeNote('remote-note', 100)] }), + capture, + ); + const store = makeStore({ + loadConfig: async (): Promise => ({ + updatedAt: 50, + booknotes: [makeNote('local-note', 50)], + }), + }); + + const engine = new FileSyncEngine(provider, store); + await engine.syncLibrary([local], { strategy: 'silent', syncBooks: false, deviceId: 'pc' }); + + expect(capture.config).not.toBeNull(); + const ids = capture.config!.booknotes.map((n) => n.id).sort(); + expect(ids).toEqual(['local-note', 'remote-note']); + }); + + test('does not regress newer remote progress with an older local push', async () => { + // Local newer than the index -> pushed under incremental. The remote + // config is nonetheless ahead on progress, so the pull-merge must carry the + // remote page, not regress it. + const local = makeLocalBook({ updatedAt: 200 }); + const capture: { config?: RemoteBookConfig | null } = {}; + const provider = makeProvider( + makeRemoteIndex(makeLocalBook({ updatedAt: 100 }), 100), + makeRemoteConfig({ config: { updatedAt: 200, progress: [50, 100] }, booknotes: [] }), + capture, + ); + const store = makeStore({ + loadConfig: async (): Promise => ({ + updatedAt: 50, + progress: [10, 100], + booknotes: [], + }), + }); + + const engine = new FileSyncEngine(provider, store); + await engine.syncLibrary([local], { strategy: 'silent', syncBooks: false, deviceId: 'pc' }); + + expect(capture.config!.config.progress).toEqual([50, 100]); + }); + + test('send strategy keeps the blind push (local authoritative, no pull-merge)', async () => { + const local = makeLocalBook({ updatedAt: 100 }); + const capture: { config?: RemoteBookConfig | null } = {}; + const provider = makeProvider( + null, + makeRemoteConfig({ config: { updatedAt: 200 }, booknotes: [makeNote('remote-note', 200)] }), + capture, + ); + const store = makeStore({ + loadConfig: async (): Promise => ({ + updatedAt: 50, + booknotes: [makeNote('local-note', 50)], + }), + }); + + const engine = new FileSyncEngine(provider, store); + await engine.syncLibrary([local], { strategy: 'send', syncBooks: false, deviceId: 'pc' }); + + const ids = capture.config!.booknotes.map((n) => n.id); + expect(ids).toEqual(['local-note']); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/file/engine-sync-paths.test.ts b/apps/readest-app/src/__tests__/services/sync/file/engine-sync-paths.test.ts new file mode 100644 index 00000000..0f151720 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/file/engine-sync-paths.test.ts @@ -0,0 +1,346 @@ +import { describe, expect, test, vi } from 'vitest'; + +import type { Book } from '@/types/book'; +import { FileSyncEngine } from '@/services/sync/file/engine'; +import type { FileSyncProvider } from '@/services/sync/file/provider'; +import type { LocalStore } from '@/services/sync/file/localStore'; +import type { RemoteBookConfig, RemoteLibraryIndex } from '@/services/sync/file/wire'; + +/** + * Coverage for the engine paths the behavior-preservation gate + * (engine-metadata-sync) does not execute: streaming upload + HEAD + * short-circuit, remote-only discovery -> streaming download -> addBook, and + * the receive (pull-only) strategy. These carry the OOM-avoidance and + * idempotency value of the original WebDAV sync. + */ + +const makeBook = (hash: string, overrides: Partial = {}): Book => ({ + hash, + format: 'EPUB', + title: `Book ${hash}`, + sourceTitle: `Book ${hash}`, + author: 'A', + createdAt: 1, + updatedAt: 1, + ...overrides, +}); + +type Captured = { writes: { path: string; body: string }[] }; + +const fakeProvider = ( + opts: Partial & { captured?: Captured } = {}, +): FileSyncProvider => ({ + rootPath: '/', + readText: opts.readText ?? (async () => null), + readBinary: opts.readBinary ?? (async () => new ArrayBuffer(8)), + head: opts.head ?? (async () => null), + list: opts.list ?? (async () => []), + writeText: + opts.writeText ?? + (async (path: string, body: string) => { + opts.captured?.writes.push({ path, body }); + }), + writeBinary: opts.writeBinary ?? (async () => {}), + ensureDir: opts.ensureDir ?? (async () => {}), + deleteDir: opts.deleteDir ?? (async () => {}), + uploadStream: opts.uploadStream, + downloadStream: opts.downloadStream, +}); + +const fakeStore = (opts: Partial = {}): LocalStore => ({ + loadConfig: opts.loadConfig ?? (async () => null), + saveBookConfig: opts.saveBookConfig ?? (async () => {}), + loadBookFile: opts.loadBookFile ?? (async () => null), + resolveLocalBookPath: opts.resolveLocalBookPath ?? (async () => null), + saveBookFile: opts.saveBookFile ?? (async () => {}), + prepareLocalBookPath: opts.prepareLocalBookPath ?? (async () => '/local/dst'), + loadBookCover: opts.loadBookCover ?? (async () => null), + saveBookCover: opts.saveBookCover ?? (async () => {}), + addBookToLibrary: opts.addBookToLibrary ?? (async () => {}), + updateBookMetadata: opts.updateBookMetadata ?? (async () => {}), +}); + +describe('FileSyncEngine.pushBookFile — streaming upload', () => { + test('streams via provider.uploadStream when remote is missing', async () => { + const uploadStream = vi.fn(async () => true); + const provider = fakeProvider({ head: async () => null, uploadStream }); + const store = fakeStore({ + resolveLocalBookPath: async () => ({ path: '/local/x.epub', size: 100 }), + }); + + const res = await new FileSyncEngine(provider, store).pushBookFile(makeBook('h1')); + + expect(res).toEqual({ uploaded: true }); + expect(uploadStream).toHaveBeenCalledTimes(1); + expect(uploadStream).toHaveBeenCalledWith( + expect.stringContaining('/Readest/books/h1/'), + '/local/x.epub', + ); + }); + + test('HEAD size match short-circuits without uploading', async () => { + const uploadStream = vi.fn(async () => true); + const provider = fakeProvider({ head: async () => ({ size: 100 }), uploadStream }); + const store = fakeStore({ + resolveLocalBookPath: async () => ({ path: '/local/x.epub', size: 100 }), + }); + + const res = await new FileSyncEngine(provider, store).pushBookFile(makeBook('h1')); + + expect(res).toEqual({ uploaded: false, reason: 'remote-matches' }); + expect(uploadStream).not.toHaveBeenCalled(); + }); + + test('retries the stream once before failing', async () => { + const uploadStream = vi + .fn<(remotePath: string, localPath: string) => Promise>() + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(true); + const provider = fakeProvider({ head: async () => null, uploadStream }); + const store = fakeStore({ + resolveLocalBookPath: async () => ({ path: '/local/x.epub', size: 100 }), + }); + + const res = await new FileSyncEngine(provider, store).pushBookFile(makeBook('h1')); + + expect(res).toEqual({ uploaded: true }); + expect(uploadStream).toHaveBeenCalledTimes(2); + }); +}); + +describe('FileSyncEngine.syncLibrary — remote discovery + streaming download', () => { + test('discovers a remote-only book, streams it down, and adds it to the library', async () => { + const downloadStream = vi.fn(async () => true); + const provider = fakeProvider({ + list: async (path: string) => + path.endsWith('/books') + ? [{ name: 'h2', path: '/Readest/books/h2', isDirectory: true }] + : [ + { + name: 'Remote.epub', + path: '/Readest/books/h2/Remote.epub', + isDirectory: false, + size: 50, + }, + ], + downloadStream, + }); + const addBookToLibrary = vi.fn<(book: Book) => Promise>(async () => {}); + const prepareLocalBookPath = vi.fn(async () => '/local/h2/Remote.epub'); + const store = fakeStore({ addBookToLibrary, prepareLocalBookPath }); + + const res = await new FileSyncEngine(provider, store).syncLibrary([], { + strategy: 'silent', + syncBooks: false, + deviceId: 'd', + }); + + expect(downloadStream).toHaveBeenCalledWith( + '/Readest/books/h2/Remote.epub', + '/local/h2/Remote.epub', + ); + expect(addBookToLibrary).toHaveBeenCalledTimes(1); + expect(addBookToLibrary.mock.calls[0]![0].hash).toBe('h2'); + expect(res.booksDownloaded).toBe(1); + }); +}); + +describe('FileSyncEngine.syncLibrary — receive strategy is pull-only', () => { + test('never writes (no config push, no index re-push) under receive', async () => { + const captured: Captured = { writes: [] }; + const provider = fakeProvider({ captured }); + const store = fakeStore({ loadConfig: async () => ({ updatedAt: 1, booknotes: [] }) }); + + const res = await new FileSyncEngine(provider, store).syncLibrary([makeBook('h1')], { + strategy: 'receive', + syncBooks: true, + deviceId: 'd', + }); + + expect(captured.writes).toHaveLength(0); + expect(res.configsUploaded).toBe(0); + expect(res.filesUploaded).toBe(0); + }); +}); + +const makeIndex = (books: Book[]): RemoteLibraryIndex => ({ + schemaVersion: 1, + updatedAt: 1, + books, +}); + +const makeEnvelope = (over: Partial = {}): RemoteBookConfig => ({ + schemaVersion: 1, + bookHash: 'h1', + config: { updatedAt: 100 }, + booknotes: [], + writerDeviceId: 'peer', + writerVersion: 'readest-webdav-1', + updatedAt: 100, + ...over, +}); + +const configWrites = (captured: Captured) => + captured.writes.filter((w) => w.path.endsWith('config.json')); + +describe('FileSyncEngine.syncLibrary — incremental diff (default)', () => { + test('skips a book whose updatedAt matches the remote index', async () => { + const captured: Captured = { writes: [] }; + const provider = fakeProvider({ + readText: async (p) => + p.endsWith('library.json') + ? JSON.stringify(makeIndex([makeBook('h1', { updatedAt: 100 })])) + : null, + captured, + }); + const store = fakeStore({ loadConfig: async () => ({ updatedAt: 1, booknotes: [] }) }); + + const res = await new FileSyncEngine(provider, store).syncLibrary( + [makeBook('h1', { updatedAt: 100 })], + { + strategy: 'silent', + syncBooks: false, + deviceId: 'd', + }, + ); + + expect(configWrites(captured)).toHaveLength(0); + expect(res.configsUploaded).toBe(0); + expect(res.booksSynced).toBe(0); + // The index itself is still re-pushed. + expect(captured.writes.some((w) => w.path.endsWith('library.json'))).toBe(true); + }); + + test('pushes a book that is newer locally than the index', async () => { + const captured: Captured = { writes: [] }; + const provider = fakeProvider({ + readText: async (p) => + p.endsWith('library.json') + ? JSON.stringify(makeIndex([makeBook('h1', { updatedAt: 100 })])) + : null, + captured, + }); + const store = fakeStore({ loadConfig: async () => ({ updatedAt: 1, booknotes: [] }) }); + + const res = await new FileSyncEngine(provider, store).syncLibrary( + [makeBook('h1', { updatedAt: 200 })], + { + strategy: 'silent', + syncBooks: false, + deviceId: 'd', + }, + ); + + expect(res.configsUploaded).toBe(1); + expect(res.booksSynced).toBe(1); + expect(configWrites(captured)).toHaveLength(1); + }); + + test('pulls config + metadata for a book newer in the index', async () => { + const captured: Captured = { writes: [] }; + const provider = fakeProvider({ + readText: async (p) => { + if (p.endsWith('library.json')) + return JSON.stringify(makeIndex([makeBook('h1', { updatedAt: 200, title: 'Remote' })])); + if (p.endsWith('config.json')) + return JSON.stringify(makeEnvelope({ config: { updatedAt: 200, progress: [9, 10] } })); + return null; + }, + captured, + }); + const saveBookConfig = vi.fn(async () => {}); + const updateBookMetadata = vi.fn(async () => {}); + const store = fakeStore({ + loadConfig: async () => ({ updatedAt: 1, booknotes: [] }), + saveBookConfig, + updateBookMetadata, + }); + + const res = await new FileSyncEngine(provider, store).syncLibrary( + [makeBook('h1', { updatedAt: 100 })], + { + strategy: 'silent', + syncBooks: false, + deviceId: 'd', + }, + ); + + expect(res.metadataUpdated).toBe(1); + expect(res.configsDownloaded).toBe(1); + expect(res.booksSynced).toBe(1); + expect(saveBookConfig).toHaveBeenCalledTimes(1); + // Remote is newer, so the book is NOT in the push set — no config.json PUT. + expect(configWrites(captured)).toHaveLength(0); + }); + + test('fullSync re-pushes an in-sync book', async () => { + const captured: Captured = { writes: [] }; + const provider = fakeProvider({ + readText: async (p) => + p.endsWith('library.json') + ? JSON.stringify(makeIndex([makeBook('h1', { updatedAt: 100 })])) + : null, + captured, + }); + const store = fakeStore({ loadConfig: async () => ({ updatedAt: 1, booknotes: [] }) }); + + const res = await new FileSyncEngine(provider, store).syncLibrary( + [makeBook('h1', { updatedAt: 100 })], + { + strategy: 'silent', + syncBooks: false, + deviceId: 'd', + fullSync: true, + }, + ); + + expect(res.configsUploaded).toBe(1); + expect(configWrites(captured)).toHaveLength(1); + }); +}); + +describe('FileSyncEngine.syncLibrary — bounded concurrency', () => { + const runWithConcurrency = async (concurrency: number | undefined, bookCount: number) => { + let inFlight = 0; + let maxInFlight = 0; + const loadConfig = async () => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((r) => setTimeout(r, 5)); + inFlight -= 1; + return { updatedAt: 1, booknotes: [] }; + }; + const books = Array.from({ length: bookCount }, (_, i) => + makeBook(`h${i}`, { updatedAt: 100 }), + ); + // No remote index -> every book is local-only -> all pushed. + const provider = fakeProvider({ captured: { writes: [] } }); + const store = fakeStore({ loadConfig }); + const res = await new FileSyncEngine(provider, store).syncLibrary(books, { + strategy: 'silent', + syncBooks: false, + deviceId: 'd', + concurrency, + }); + return { res, maxInFlight }; + }; + + test('caps in-flight work at the configured concurrency', async () => { + const { res, maxInFlight } = await runWithConcurrency(3, 8); + expect(res.configsUploaded).toBe(8); + expect(res.booksSynced).toBe(8); + expect(maxInFlight).toBe(3); + }); + + test('defaults to 4 when concurrency is omitted', async () => { + const { res, maxInFlight } = await runWithConcurrency(undefined, 8); + expect(res.configsUploaded).toBe(8); + expect(maxInFlight).toBe(4); + }); + + test('concurrency 1 runs strictly sequentially', async () => { + const { res, maxInFlight } = await runWithConcurrency(1, 4); + expect(res.configsUploaded).toBe(4); + expect(maxInFlight).toBe(1); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/file/layout.test.ts b/apps/readest-app/src/__tests__/services/sync/file/layout.test.ts new file mode 100644 index 00000000..6fbb85db --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/file/layout.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, test } from 'vitest'; +import { + buildBasePath, + buildBookDirPath, + buildBookConfigPath, + buildLibraryPath, + buildBookFilePath, + buildBookCoverPath, + ancestorsOf, + normalizeRoot, +} from '@/services/sync/file/layout'; +import type { Book } from '@/types/book'; + +const book = { + hash: 'h1', + format: 'EPUB', + title: 'My Book', + sourceTitle: 'My Book', + author: 'A', + createdAt: 1, + updatedAt: 1, +} as Book; + +describe('sync layout (frozen)', () => { + test('book tree paths under /Readest/books/', () => { + expect(buildBasePath('/')).toBe('/Readest'); + expect(buildBasePath('/MyDav')).toBe('/MyDav/Readest'); + expect(buildBookDirPath('/', 'h1')).toBe('/Readest/books/h1'); + expect(buildBookConfigPath('/', 'h1')).toBe('/Readest/books/h1/config.json'); + expect(buildBookCoverPath('/', 'h1')).toBe('/Readest/books/h1/cover.png'); + expect(buildLibraryPath('/')).toBe('/Readest/library.json'); + }); + + test('book file path uses safe title + ext inside the hash dir', () => { + expect(buildBookFilePath('/', book)).toBe('/Readest/books/h1/My Book.epub'); + }); + + test('normalizeRoot strips trailing slash, adds leading', () => { + expect(normalizeRoot('')).toBe('/'); + expect(normalizeRoot('books/')).toBe('/books'); + expect(normalizeRoot('/a/b/')).toBe('/a/b'); + }); + + test('ancestorsOf walks parents top-down excluding the leaf', () => { + expect(ancestorsOf('/a/b/c/file.json')).toEqual(['/a', '/a/b', '/a/b/c']); + expect(ancestorsOf('/file.json')).toEqual([]); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/file/merge.test.ts b/apps/readest-app/src/__tests__/services/sync/file/merge.test.ts new file mode 100644 index 00000000..fde4fd12 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/file/merge.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, test } from 'vitest'; +import { + mergeNotes, + mergeBookConfig, + mergeBookMetadata, + isRemoteBookMetadataNewer, +} from '@/services/sync/file/merge'; +import type { Book, BookConfig, BookNote } from '@/types/book'; +import type { RemoteBookConfig } from '@/services/sync/file/wire'; + +const note = (id: string, updatedAt: number, deletedAt?: number): BookNote => + ({ + id, + type: 'annotation', + cfi: `c-${id}`, + note: '', + createdAt: updatedAt, + updatedAt, + deletedAt, + }) as BookNote; + +const envelope = (over: Partial = {}): RemoteBookConfig => ({ + schemaVersion: 1, + bookHash: 'h1', + config: { updatedAt: 100 }, + booknotes: [], + writerDeviceId: 'd', + writerVersion: 'readest-webdav-1', + updatedAt: 100, + ...over, +}); + +describe('mergeNotes (element-set CRDT)', () => { + test('union keeps ids from both sides', () => { + const out = mergeNotes([note('a', 1)], [note('b', 1)]) + .map((n) => n.id) + .sort(); + expect(out).toEqual(['a', 'b']); + }); + + test('newer updatedAt wins', () => { + const out = mergeNotes([note('a', 1)], [{ ...note('a', 5), note: 'remote' }]); + expect(out.find((n) => n.id === 'a')!.note).toBe('remote'); + }); + + test('local-newer keeps local fields', () => { + const out = mergeNotes( + [{ ...note('a', 9), note: 'local' }], + [{ ...note('a', 3), note: 'remote' }], + ); + expect(out.find((n) => n.id === 'a')!.note).toBe('local'); + }); + + test('deletedAt tombstone wins on updatedAt tie', () => { + const out = mergeNotes([note('a', 5)], [note('a', 5, 9)]); + expect(out.find((n) => n.id === 'a')!.deletedAt).toBe(9); + }); + + test('idempotent on identical input (id set + field values stable)', () => { + const a = [note('a', 1), note('b', 2)]; + const once = mergeNotes(a, a); + expect(once.map((n) => n.id).sort()).toEqual(['a', 'b']); + const twice = mergeNotes(once, once); + expect(twice.map((n) => n.id).sort()).toEqual(['a', 'b']); + }); + + test('commutative on the winning value per id', () => { + const l = [note('a', 5)]; + const r = [{ ...note('a', 9), note: 'remote' }]; + const lr = mergeNotes(l, r).find((n) => n.id === 'a')!; + const rl = mergeNotes(r, l).find((n) => n.id === 'a')!; + // Whichever order, the strictly-newer side (updatedAt 9) supplies `note`. + expect(lr.note).toBe('remote'); + expect(rl.note).toBe('remote'); + }); +}); + +describe('mergeBookConfig (LWW scalars + CRDT notes)', () => { + test('remote newer overrides scalars but unions notes', () => { + const local: BookConfig = { updatedAt: 50, progress: [1, 10], booknotes: [note('l', 50)] }; + const r = envelope({ + config: { updatedAt: 100, progress: [5, 10] }, + booknotes: [note('r', 100)], + }); + const { config, notes } = mergeBookConfig(local, r); + expect(config.progress).toEqual([5, 10]); + expect(notes.map((n) => n.id).sort()).toEqual(['l', 'r']); + expect(config.booknotes!.map((n) => n.id).sort()).toEqual(['l', 'r']); + }); + + test('local newer keeps local scalars, still unions notes', () => { + const local: BookConfig = { updatedAt: 200, progress: [9, 10], booknotes: [note('l', 200)] }; + const r = envelope({ + config: { updatedAt: 100, progress: [1, 10] }, + booknotes: [note('r', 100)], + }); + const { config, notes } = mergeBookConfig(local, r); + expect(config.progress).toEqual([9, 10]); + expect(notes.map((n) => n.id).sort()).toEqual(['l', 'r']); + }); + + test('null/undefined remote scalars are dropped (never clobber local)', () => { + const local: BookConfig = { updatedAt: 50, location: 'keepme', booknotes: [] }; + const r = envelope({ + config: { updatedAt: 100, location: undefined, xpointer: undefined }, + }); + const { config } = mergeBookConfig(local, r); + expect(config.location).toBe('keepme'); + }); +}); + +describe('mergeBookMetadata (LWW field subset)', () => { + test('overlays only metadata fields, preserves local file/progress fields', () => { + const local = { + hash: 'h', + title: 'L', + author: 'L', + sourceTitle: 'src', + filePath: '/p', + progress: [1, 2], + updatedAt: 1, + } as Book; + const remote = { hash: 'h', title: 'R', author: 'R', updatedAt: 9 } as Book; + const m = mergeBookMetadata(local, remote); + expect(m.title).toBe('R'); + expect(m.author).toBe('R'); + expect(m.sourceTitle).toBe('src'); + expect(m.filePath).toBe('/p'); + expect(m.progress).toEqual([1, 2]); + expect(m.updatedAt).toBe(9); + }); +}); + +describe('isRemoteBookMetadataNewer', () => { + test('strictly newer remote only', () => { + expect(isRemoteBookMetadataNewer({ updatedAt: 1 } as Book, { updatedAt: 2 } as Book)).toBe( + true, + ); + expect(isRemoteBookMetadataNewer({ updatedAt: 2 } as Book, { updatedAt: 2 } as Book)).toBe( + false, + ); + expect(isRemoteBookMetadataNewer({ updatedAt: 3 } as Book, { updatedAt: 2 } as Book)).toBe( + false, + ); + }); + + test('tombstone on either side disqualifies', () => { + expect( + isRemoteBookMetadataNewer({ updatedAt: 1 } as Book, { updatedAt: 9, deletedAt: 9 } as Book), + ).toBe(false); + expect( + isRemoteBookMetadataNewer({ updatedAt: 1, deletedAt: 1 } as Book, { updatedAt: 9 } as Book), + ).toBe(false); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/file/provider-conformance.test.ts b/apps/readest-app/src/__tests__/services/sync/file/provider-conformance.test.ts new file mode 100644 index 00000000..40e41214 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/file/provider-conformance.test.ts @@ -0,0 +1,111 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { createWebDAVProvider } from '@/services/sync/providers/webdav/WebDAVProvider'; +import { FileSyncError } from '@/services/sync/file/provider'; +import type { FileSyncProvider } from '@/services/sync/file/provider'; +import type { WebDAVSettings } from '@/types/settings'; + +/** + * Contract every {@link FileSyncProvider} must satisfy, exercised here against + * {@link createWebDAVProvider} over a `fetch` mock. When a second backend + * (Drive, Dropbox, …) arrives, lift this into a shared helper and pass a + * factory that builds the new provider over whatever transport it stubs. + */ +const runProviderConformance = ( + name: string, + makeProvider: () => FileSyncProvider, + mock: { ok: (status: number, body?: string | null, headers?: Record) => void }, +) => { + describe(`${name} — FileSyncProvider conformance`, () => { + test('readText resolves null on 404', async () => { + mock.ok(404, null); + expect(await makeProvider().readText('/Readest/x.json')).toBeNull(); + }); + + test('readText maps 401 to FileSyncError AUTH_FAILED', async () => { + mock.ok(401, ''); + const err = await makeProvider() + .readText('/Readest/x.json') + .catch((e) => e); + expect(err).toBeInstanceOf(FileSyncError); + expect(err).toMatchObject({ code: 'AUTH_FAILED', status: 401 }); + }); + + test('readBinary resolves null on 404', async () => { + mock.ok(404, null); + expect(await makeProvider().readBinary('/Readest/x.bin')).toBeNull(); + }); + + test('head reads content-length, null on 404', async () => { + mock.ok(200, null, { 'content-length': '512' }); + expect(await makeProvider().head('/Readest/x')).toEqual({ size: 512, etag: undefined }); + mock.ok(404, null); + expect(await makeProvider().head('/Readest/x')).toBeNull(); + }); + + test('writeText succeeds on 201 Created', async () => { + mock.ok(201, ''); + await expect(makeProvider().writeText('/Readest/x.json', '{}')).resolves.toBeUndefined(); + }); + + test('deleteDir treats 404 as success', async () => { + mock.ok(404, null); + await expect(makeProvider().deleteDir('/Readest/books/gone')).resolves.toBeUndefined(); + }); + + test('list maps 401 to FileSyncError AUTH_FAILED', async () => { + mock.ok(401, ''); + const err = await makeProvider() + .list('/Readest/books') + .catch((e) => e); + expect(err).toBeInstanceOf(FileSyncError); + expect(err).toMatchObject({ code: 'AUTH_FAILED', status: 401 }); + }); + + test('list maps 404 to FileSyncError NOT_FOUND', async () => { + mock.ok(404, ''); + const err = await makeProvider() + .list('/Readest/books') + .catch((e) => e); + expect(err).toBeInstanceOf(FileSyncError); + expect(err).toMatchObject({ code: 'NOT_FOUND', status: 404 }); + }); + }); +}; + +const settings: WebDAVSettings = { + enabled: true, + serverUrl: 'https://dav.example.com', + username: 'alice', + password: 'secret', + rootPath: '/', +}; + +const ORIGINAL_FETCH = globalThis.fetch; +let fetchMock: ReturnType; + +beforeEach(() => { + fetchMock = vi.fn(); + globalThis.fetch = fetchMock as unknown as typeof fetch; +}); + +afterEach(() => { + globalThis.fetch = ORIGINAL_FETCH; + vi.restoreAllMocks(); +}); + +runProviderConformance('WebDAVProvider', () => createWebDAVProvider(settings), { + ok: (status, body = '', headers) => + fetchMock.mockResolvedValueOnce(new Response(body, { status, headers })), +}); + +describe('WebDAVProvider construction', () => { + test('rootPath is normalised', () => { + expect(createWebDAVProvider({ ...settings, rootPath: '/MyDav/' }).rootPath).toBe('/MyDav'); + }); + + test('streaming methods are absent off-Tauri (web fallback)', () => { + const provider = createWebDAVProvider(settings); + expect(provider.uploadStream).toBeUndefined(); + expect(provider.downloadStream).toBeUndefined(); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/sync/file/wire.test.ts b/apps/readest-app/src/__tests__/services/sync/file/wire.test.ts new file mode 100644 index 00000000..2f2999ad --- /dev/null +++ b/apps/readest-app/src/__tests__/services/sync/file/wire.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, test } from 'vitest'; +import { + buildRemotePayload, + parseRemotePayload, + parseRemoteLibraryIndex, +} from '@/services/sync/file/wire'; +import type { Book, BookConfig } from '@/types/book'; + +const book = { + hash: 'h1', + metaHash: 'm1', + format: 'EPUB', + title: 'T', + author: 'A', + createdAt: 1, + updatedAt: 1, +} as Book; + +// Cast through unknown so a device-local field (viewSettings) can be present +// on the source config without fighting the BookConfig type — the point of +// the test is that it never reaches the wire. +const config = { + updatedAt: 42, + progress: [3, 10], + location: 'loc', + xpointer: 'xp', + booknotes: [], + viewSettings: { fontSize: 14 }, +} as unknown as BookConfig; + +describe('wire envelope (frozen)', () => { + test('buildRemotePayload trims to reading state + stable header', () => { + const p = buildRemotePayload(book, config, 'dev-1'); + expect(p.schemaVersion).toBe(1); + expect(p.writerVersion).toBe('readest-webdav-1'); + expect(p.writerDeviceId).toBe('dev-1'); + expect(p.bookHash).toBe('h1'); + expect(p.metaHash).toBe('m1'); + expect(p.config).toEqual({ progress: [3, 10], location: 'loc', xpointer: 'xp', updatedAt: 42 }); + // Device-local fields never travel. + expect('viewSettings' in p.config).toBe(false); + }); + + test('parseRemotePayload rejects null / non-JSON / wrong schema', () => { + expect(parseRemotePayload(null)).toBeNull(); + expect(parseRemotePayload('not json')).toBeNull(); + expect(parseRemotePayload(JSON.stringify({ schemaVersion: 2 }))).toBeNull(); + const ok = parseRemotePayload(JSON.stringify(buildRemotePayload(book, config, 'd'))); + expect(ok?.bookHash).toBe('h1'); + }); + + test('parseRemoteLibraryIndex rejects null / malformed / wrong schema', () => { + expect(parseRemoteLibraryIndex(null)).toBeNull(); + expect(parseRemoteLibraryIndex('{')).toBeNull(); + expect(parseRemoteLibraryIndex(JSON.stringify({ schemaVersion: 9, books: [] }))).toBeNull(); + const ok = parseRemoteLibraryIndex( + JSON.stringify({ schemaVersion: 1, books: [book], updatedAt: 5 }), + ); + expect(ok?.books).toHaveLength(1); + expect(ok?.updatedAt).toBe(5); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/webdav-connect-settings.test.ts b/apps/readest-app/src/__tests__/services/webdav-connect-settings.test.ts index 14ae9410..03ae8e1c 100644 --- a/apps/readest-app/src/__tests__/services/webdav-connect-settings.test.ts +++ b/apps/readest-app/src/__tests__/services/webdav-connect-settings.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect } from 'vitest'; -import { buildWebDAVConnectSettings } from '@/services/webdav/webdavConnectSettings'; +import { buildWebDAVConnectSettings } from '@/services/sync/providers/webdav/connectSettings'; import type { WebDAVSettings } from '@/types/settings'; describe('buildWebDAVConnectSettings', () => { diff --git a/apps/readest-app/src/__tests__/services/webdav-delete.test.ts b/apps/readest-app/src/__tests__/services/webdav-delete.test.ts index a94d4a9e..cbb5923a 100644 --- a/apps/readest-app/src/__tests__/services/webdav-delete.test.ts +++ b/apps/readest-app/src/__tests__/services/webdav-delete.test.ts @@ -1,10 +1,8 @@ import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; -import { - deleteDirectory, - WebDAVRequestError, - type WebDAVConfig, -} from '@/services/webdav/WebDAVClient'; -import { deleteRemoteBookDir } from '@/services/webdav/WebDAVSync'; +import { deleteDirectory, type WebDAVConfig } from '@/services/sync/providers/webdav/client'; +import { createWebDAVProvider } from '@/services/sync/providers/webdav/WebDAVProvider'; +import { deleteRemoteBookDir } from '@/services/sync/file/engine'; +import { FileSyncError } from '@/services/sync/file/provider'; import type { WebDAVSettings } from '@/types/settings'; /** @@ -127,15 +125,16 @@ describe('deleteDirectory', () => { describe('deleteRemoteBookDir', () => { const HASH = 'abc123'; + const provider = createWebDAVProvider(settings); test('successful DELETE returns ok=true', async () => { fetchMock.mockResolvedValueOnce(buildResponse(204)); - await expect(deleteRemoteBookDir(settings, HASH)).resolves.toEqual({ ok: true }); + await expect(deleteRemoteBookDir(provider, HASH)).resolves.toEqual({ ok: true }); }); test('non-auth failure (500) returns ok=false with reason populated', async () => { fetchMock.mockResolvedValueOnce(buildResponse(500)); - const result = await deleteRemoteBookDir(settings, HASH); + const result = await deleteRemoteBookDir(provider, HASH); expect(result.ok).toBe(false); // The reason is the underlying error message; we don't pin its // exact wording (could be tweaked) but it must be a non-empty @@ -146,24 +145,23 @@ describe('deleteRemoteBookDir', () => { test('AUTH_FAILED is rethrown so callers can short-circuit batches', async () => { fetchMock.mockResolvedValueOnce(buildResponse(401)); - await expect(deleteRemoteBookDir(settings, HASH)).rejects.toBeInstanceOf(WebDAVRequestError); + await expect(deleteRemoteBookDir(provider, HASH)).rejects.toBeInstanceOf(FileSyncError); // 403 walks the same code path; assert the AUTH_FAILED tag // independently so a regression that lets one status leak // through but not the other still trips a test. fetchMock.mockResolvedValueOnce(buildResponse(403)); - await expect(deleteRemoteBookDir(settings, HASH)).rejects.toMatchObject({ + await expect(deleteRemoteBookDir(provider, HASH)).rejects.toMatchObject({ code: 'AUTH_FAILED', }); }); test('targets the correct per-hash directory under /Readest/books', async () => { - // The remote layout is documented in WebDAVPaths.ts; this test - // pins the contract so neither side can drift. A custom - // rootPath is used to make sure buildBookDirPath honours it - // (a literal '/' would mask a "join always uses leading-slash - // base" regression). + // The remote layout is documented in sync/file/layout.ts; this test + // pins the contract so neither side can drift. A custom rootPath is + // used to make sure buildBookDirPath honours it (a literal '/' would + // mask a "join always uses leading-slash base" regression). fetchMock.mockResolvedValueOnce(buildResponse(204)); - await deleteRemoteBookDir({ ...settings, rootPath: '/MyDav' }, HASH); + await deleteRemoteBookDir(createWebDAVProvider({ ...settings, rootPath: '/MyDav' }), HASH); const [url] = fetchMock.mock.calls[0]!; expect(url).toBe('https://dav.example.com/MyDav/Readest/books/abc123'); }); diff --git a/apps/readest-app/src/__tests__/services/webdav-encode-path.test.ts b/apps/readest-app/src/__tests__/services/webdav-encode-path.test.ts index e9a90bf5..a0d64a74 100644 --- a/apps/readest-app/src/__tests__/services/webdav-encode-path.test.ts +++ b/apps/readest-app/src/__tests__/services/webdav-encode-path.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect } from 'vitest'; -import { buildRequestUrl } from '@/services/webdav/WebDAVClient'; +import { buildRequestUrl } from '@/services/sync/providers/webdav/client'; describe('buildRequestUrl (encodePath)', () => { test('escapes spaces and unicode in each segment', () => { diff --git a/apps/readest-app/src/__tests__/services/webdav-metadata-sync.test.ts b/apps/readest-app/src/__tests__/services/webdav-metadata-sync.test.ts deleted file mode 100644 index a6f9b51e..00000000 --- a/apps/readest-app/src/__tests__/services/webdav-metadata-sync.test.ts +++ /dev/null @@ -1,277 +0,0 @@ -import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; - -import type { WebDAVSettings } from '@/types/settings'; -import type { Book, BookConfig, BookNote } from '@/types/book'; - -/** - * Regression tests for issue #4756: once a device already has a book in its - * local library, the manual "Sync now" path never pulled the book's metadata - * (title / author / cover) back from the shared `library.json`, and the final - * index re-push clobbered the remote with the device's stale copy. - * - * These tests mock the WebDAVClient I/O primitives so we can drive - * `syncLibrary` deterministically without a real server, and assert the - * last-writer-wins reconciliation on `book.updatedAt`. - */ - -vi.mock('@/services/webdav/WebDAVClient', async (importActual) => { - const actual = await importActual(); - return { - ...actual, - getFile: vi.fn(), - getFileBinary: vi.fn(), - headFile: vi.fn(), - listDirectory: vi.fn(), - putFile: vi.fn(), - putFileBinary: vi.fn(), - ensureDirectory: vi.fn(), - deleteDirectory: vi.fn(), - }; -}); - -import { - getFile, - getFileBinary, - headFile, - listDirectory, - putFile, - putFileBinary, - ensureDirectory, -} from '@/services/webdav/WebDAVClient'; -import { - syncLibrary, - type RemoteLibraryIndex, - type RemoteBookConfig, -} from '@/services/webdav/WebDAVSync'; - -const settings: WebDAVSettings = { - enabled: true, - serverUrl: 'https://dav.example.com', - username: 'alice', - password: 'secret', - rootPath: '/', -}; - -const makeLocalBook = (overrides: Partial = {}): Book => ({ - hash: 'h1', - format: 'EPUB', - title: 'Old Title', - sourceTitle: 'Old Title', - author: 'Old Author', - createdAt: 1, - updatedAt: 100, - ...overrides, -}); - -const makeRemoteIndex = (book: Book, updatedAt = book.updatedAt): RemoteLibraryIndex => ({ - schemaVersion: 1, - updatedAt, - books: [book], -}); - -/** - * Route `getFile` by path: library.json → index JSON, config.json → the - * supplied remote envelope (or null when none). - */ -const wireGetFile = ( - index: RemoteLibraryIndex | null, - remoteConfig: RemoteBookConfig | null = null, -) => { - (getFile as ReturnType).mockImplementation(async (_client, path: string) => { - if (path.endsWith('library.json')) return index ? JSON.stringify(index) : null; - if (path.endsWith('config.json')) return remoteConfig ? JSON.stringify(remoteConfig) : null; - return null; - }); -}; - -/** Capture the library index that was re-pushed at the end of the sync. */ -const capturePushedIndex = (): { value: RemoteLibraryIndex | null } => { - const captured: { value: RemoteLibraryIndex | null } = { value: null }; - (putFile as ReturnType).mockImplementation(async (_client, path: string, body) => { - if (path.endsWith('library.json')) captured.value = JSON.parse(body as string); - }); - return captured; -}; - -/** Capture the per-book config envelope pushed to /config.json. */ -const capturePushedConfig = (): { value: RemoteBookConfig | null } => { - const captured: { value: RemoteBookConfig | null } = { value: null }; - (putFile as ReturnType).mockImplementation(async (_client, path: string, body) => { - if (path.endsWith('config.json')) captured.value = JSON.parse(body as string); - }); - return captured; -}; - -const makeNote = (id: string, updatedAt: number): BookNote => ({ - id, - type: 'annotation', - cfi: `cfi-${id}`, - note: '', - createdAt: updatedAt, - updatedAt, -}); - -const makeRemoteConfig = (overrides: Partial = {}): RemoteBookConfig => ({ - schemaVersion: 1, - bookHash: 'h1', - config: { updatedAt: 100 }, - booknotes: [], - writerDeviceId: 'mobile', - writerVersion: 'readest-webdav-1', - updatedAt: 100, - ...overrides, -}); - -beforeEach(() => { - vi.clearAllMocks(); - (listDirectory as ReturnType).mockResolvedValue([]); - (getFileBinary as ReturnType).mockResolvedValue(new ArrayBuffer(8)); - (headFile as ReturnType).mockResolvedValue(null); - (putFile as ReturnType).mockResolvedValue(undefined); - (putFileBinary as ReturnType).mockResolvedValue(undefined); - (ensureDirectory as ReturnType).mockResolvedValue(undefined); -}); - -afterEach(() => { - vi.restoreAllMocks(); -}); - -const baseOptions = () => ({ - strategy: 'silent' as const, - syncBooks: false, - deviceId: 'pc-device', - loadConfig: async (): Promise => ({ updatedAt: 50, booknotes: [] }), - loadBookFile: async () => null, - loadBookCover: async () => null, - onProgress: () => {}, -}); - -describe('syncLibrary metadata reconciliation (#4756)', () => { - test('pulls newer remote metadata for a book the device already has', async () => { - const local = makeLocalBook({ updatedAt: 100 }); - const remote = makeLocalBook({ - title: 'New Title', - author: 'New Author', - updatedAt: 200, - }); - wireGetFile(makeRemoteIndex(remote, 200)); - const pushedIndex = capturePushedIndex(); - - const updateBookMetadata = vi.fn(async (_book: Book) => {}); - const saveBookCover = vi.fn(async (_book: Book, _bytes: ArrayBuffer) => {}); - - const result = await syncLibrary(settings, [local], { - ...baseOptions(), - updateBookMetadata, - saveBookCover, - }); - - // The device must learn the remote title/author. - expect(updateBookMetadata).toHaveBeenCalledTimes(1); - const merged = updateBookMetadata.mock.calls[0]![0]; - expect(merged.title).toBe('New Title'); - expect(merged.author).toBe('New Author'); - expect(result.metadataUpdated).toBe(1); - - // The cover must be re-pulled so a changed cover propagates. - expect(saveBookCover).toHaveBeenCalledTimes(1); - - // The re-pushed index must carry the newer metadata, not clobber it. - expect(pushedIndex.value).not.toBeNull(); - const indexedBook = pushedIndex.value!.books.find((b) => b.hash === 'h1')!; - expect(indexedBook.title).toBe('New Title'); - }); - - test('does not overwrite local metadata when the local copy is newer', async () => { - const local = makeLocalBook({ title: 'Local Newer', updatedAt: 300 }); - const remote = makeLocalBook({ title: 'Remote Older', updatedAt: 200 }); - wireGetFile(makeRemoteIndex(remote, 200)); - const pushedIndex = capturePushedIndex(); - - const updateBookMetadata = vi.fn(async (_book: Book) => {}); - - const result = await syncLibrary(settings, [local], { - ...baseOptions(), - updateBookMetadata, - }); - - expect(updateBookMetadata).not.toHaveBeenCalled(); - expect(result.metadataUpdated).toBe(0); - - // Local wins: the re-pushed index keeps the local title. - const indexedBook = pushedIndex.value!.books.find((b) => b.hash === 'h1')!; - expect(indexedBook.title).toBe('Local Newer'); - }); -}); - -describe('syncLibrary config merge before push (Sync now must not blind-overwrite)', () => { - test('unions remote booknotes into the pushed config instead of clobbering them', async () => { - const local = makeLocalBook({ updatedAt: 100 }); - // Same updatedAt in the index so the metadata pass is a no-op — this - // isolates the config/notes path. - wireGetFile( - makeRemoteIndex(makeLocalBook({ updatedAt: 100 }), 100), - makeRemoteConfig({ config: { updatedAt: 100 }, booknotes: [makeNote('remote-note', 100)] }), - ); - const pushedConfig = capturePushedConfig(); - - await syncLibrary(settings, [local], { - ...baseOptions(), - loadConfig: async (): Promise => ({ - updatedAt: 50, - booknotes: [makeNote('local-note', 50)], - }), - }); - - // The pushed envelope must carry BOTH notes — a blind push would have - // dropped the peer's note that this device never pulled. - expect(pushedConfig.value).not.toBeNull(); - const ids = pushedConfig.value!.booknotes.map((n) => n.id).sort(); - expect(ids).toEqual(['local-note', 'remote-note']); - }); - - test('does not regress newer remote progress with an older local push', async () => { - const local = makeLocalBook({ updatedAt: 100 }); - wireGetFile( - makeRemoteIndex(makeLocalBook({ updatedAt: 100 }), 100), - makeRemoteConfig({ config: { updatedAt: 200, progress: [50, 100] }, booknotes: [] }), - ); - const pushedConfig = capturePushedConfig(); - - await syncLibrary(settings, [local], { - ...baseOptions(), - loadConfig: async (): Promise => ({ - updatedAt: 50, - progress: [10, 100], - booknotes: [], - }), - }); - - // Remote progress is newer (updatedAt 200 > 50): the LWW merge must push - // the remote's page, never regress it back to the local page. - expect(pushedConfig.value!.config.progress).toEqual([50, 100]); - }); - - test('send strategy keeps the blind push (local authoritative, no pull-merge)', async () => { - const local = makeLocalBook({ updatedAt: 100 }); - wireGetFile( - null, - makeRemoteConfig({ config: { updatedAt: 200 }, booknotes: [makeNote('remote-note', 200)] }), - ); - const pushedConfig = capturePushedConfig(); - - await syncLibrary(settings, [local], { - ...baseOptions(), - strategy: 'send', - loadConfig: async (): Promise => ({ - updatedAt: 50, - booknotes: [makeNote('local-note', 50)], - }), - }); - - // 'send' deliberately treats local as the source of truth — it must not - // pull the remote note in. - const ids = pushedConfig.value!.booknotes.map((n) => n.id); - expect(ids).toEqual(['local-note']); - }); -}); diff --git a/apps/readest-app/src/__tests__/styles/zIndexScale.test.ts b/apps/readest-app/src/__tests__/styles/zIndexScale.test.ts index b9cc1d58..a34c13ab 100644 --- a/apps/readest-app/src/__tests__/styles/zIndexScale.test.ts +++ b/apps/readest-app/src/__tests__/styles/zIndexScale.test.ts @@ -42,6 +42,7 @@ const RSVP_CONTROLS = firstZ( read('src/app/reader/components/rsvp/RSVPStartDialog.tsx'), /z-\[(\d+)\]/, ); +const TOAST = firstZ(read('src/components/Toast.tsx'), /toast z-\[(\d+)\]/); const APP_LOCK = firstZ(read('src/components/AppLockScreen.tsx'), /z-\[(\d+)\]/); describe('overlay z-index scale', () => { @@ -62,12 +63,21 @@ describe('overlay z-index scale', () => { expect(RSVP_OVERLAY).toBeGreaterThan(PAGE_FRAME); }); - it('keeps the security lock screen on top of every modal', () => { + it('raises toasts above every modal so they show over open dialogs', () => { + // Regression: a sync-complete toast dispatched from the open Settings + // dialog was buried because the toast sat at z-50, below Settings (110) + // and ModalPortal (120). + expect(TOAST).toBeGreaterThan(MODAL); + expect(TOAST).toBeGreaterThan(SETTINGS); + }); + + it('keeps the security lock screen on top of every modal and toast', () => { expect(APP_LOCK).toBeGreaterThan(MODAL); + expect(APP_LOCK).toBeGreaterThan(TOAST); }); it('uses a compact scale with no four-digit z-index', () => { - for (const value of [RSVP_OVERLAY, RSVP_CONTROLS, SETTINGS, MODAL, APP_LOCK]) { + for (const value of [RSVP_OVERLAY, RSVP_CONTROLS, SETTINGS, MODAL, TOAST, APP_LOCK]) { expect(value).toBeLessThan(1000); } }); diff --git a/apps/readest-app/src/app/reader/hooks/useWebDAVSync.ts b/apps/readest-app/src/app/reader/hooks/useWebDAVSync.ts index d9c9281b..66af2351 100644 --- a/apps/readest-app/src/app/reader/hooks/useWebDAVSync.ts +++ b/apps/readest-app/src/app/reader/hooks/useWebDAVSync.ts @@ -8,16 +8,10 @@ import { useSettingsStore } from '@/store/settingsStore'; import { useTranslation } from '@/hooks/useTranslation'; import { debounce } from '@/utils/debounce'; import { eventDispatcher } from '@/utils/event'; -import { - pullBookConfig, - pushBookConfig, - pushBookCover, - pushBookFile, -} from '@/services/webdav/WebDAVSync'; -import { WebDAVRequestError } from '@/services/webdav/WebDAVClient'; -import { isTauriAppPlatform } from '@/services/environment'; -import { tauriUpload } from '@/utils/transfer'; -import { getCoverFilename, getLocalBookFilename } from '@/utils/book'; +import { FileSyncEngine } from '@/services/sync/file/engine'; +import { FileSyncError } from '@/services/sync/file/provider'; +import { createAppLocalStore } from '@/services/sync/file/appLocalStore'; +import { createWebDAVProvider } from '@/services/sync/providers/webdav/WebDAVProvider'; import { removeBookNoteOverlays } from '../utils/annotatorUtil'; import { useWindowActiveChanged } from './useWindowActiveChanged'; @@ -162,6 +156,17 @@ export const useWebDAVSync = (bookKey: string) => { const allowPush = isReady && strategy !== 'receive'; const allowPull = isReady && strategy !== 'send'; + // One engine per (credentials, appService) — the provider owns the WebDAV + // URL + auth and the streaming transport; the shared local-store bridge owns + // the on-disk book/cover I/O. Rebuilt when settings change (cheap closures). + const engine = useMemo(() => { + const w = settings.webdav; + if (!w?.enabled || !w?.serverUrl || !w?.username || !appService) return null; + const provider = createWebDAVProvider(w); + const store = createAppLocalStore({ appService, settings, envConfig }); + return new FileSyncEngine(provider, store); + }, [settings, appService, envConfig]); + /** * Push the latest config (progress + booknotes) to the remote. * Skips while the user is previewing a deep-link target — the in-memory @@ -178,18 +183,18 @@ export const useWebDAVSync = (bookKey: string) => { const config = getConfig(bookKey); const book = getBookData(bookKey)?.book; - if (!config || !book) return; + if (!config || !book || !engine) return; try { const deviceId = ensureDeviceId(); // We always push the full envelope; sub-toggles only gate _which_ // fields the local writer applies on pull. This keeps the wire // schema stable across users with different toggle combinations. - await pushBookConfig(settings.webdav!, book, config, deviceId); + await engine.pushBookConfig(book, config, deviceId); dirtyRef.current = false; await updateLastSyncedAt(Date.now()); } catch (e) { - if (e instanceof WebDAVRequestError && e.code === 'AUTH_FAILED') { + if (e instanceof FileSyncError && e.code === 'AUTH_FAILED') { eventDispatcher.dispatch('toast', { type: 'error', message: _('WebDAV authentication failed. Reconnect in Settings.'), @@ -204,6 +209,7 @@ export const useWebDAVSync = (bookKey: string) => { getConfig, getBookData, ensureDeviceId, + engine, settings.webdav, updateLastSyncedAt, _, @@ -224,68 +230,13 @@ export const useWebDAVSync = (bookKey: string) => { fileSyncedRef.current = true; const book = getBookData(bookKey)?.book; - if (!book || !appService) return; + if (!book || !engine) return; try { - const result = await pushBookFile( - settings.webdav!, - book, - async () => { - // Buffered fallback: read the local book file off disk lazily - // so we only do the expensive ArrayBuffer materialisation when - // the HEAD probe says we actually need to upload. Used on web - // targets where streaming PUTs aren't available. - // In-place imports keep their bytes outside Books//, so - // resolve to (book.filePath, 'None') when the field is set — - // mirrors the same fallback in cloudService.uploadBook so - // syncBooks treats in-place books as first-class. - const fp = book.filePath ?? getLocalBookFilename(book); - const base = book.filePath ? 'None' : 'Books'; - if (!(await appService.exists(fp, base))) return null; - const file = await appService.openFile(fp, base); - const bytes = await file.arrayBuffer(); - return { bytes, size: bytes.byteLength }; - }, - // Tauri-only: stream the book file straight from disk to the - // server via Rust-side `upload_file`, never letting the bytes - // land in the JS heap. Without this, opening a multi-hundred- - // megabyte PDF / scanned book would buffer the whole file into - // V8 just to PUT it, blowing the renderer's heap and freezing - // the reader to a blank screen mid-open. Same flow as the - // library Sync now path in WebDAVForm. - isTauriAppPlatform() - ? async () => { - const fp = book.filePath ?? getLocalBookFilename(book); - const base = book.filePath ? 'None' : 'Books'; - if (!(await appService.exists(fp, base))) return null; - const file = await appService.openFile(fp, base); - const size = file.size; - // Release the FD before streaming so the Tauri side can - // re-open the path for the PUT without contending. - const closable = file as { close?: () => Promise }; - if (closable.close) await closable.close(); - const dst = await appService.resolveFilePath(fp, base); - return { - size, - upload: async (remoteUrl, headers) => { - try { - await tauriUpload( - remoteUrl, - dst, - 'PUT', - undefined, - headers as unknown as Map, - ); - return true; - } catch (e) { - console.warn('WD per-book push: tauriUpload failed', book.hash, e); - return false; - } - }, - }; - } - : undefined, - ); + // The engine prefers the provider's streaming upload (Tauri) and falls + // back to a buffered PUT on web — both resolve the local bytes/path via + // the shared local store, so this hook no longer owns that plumbing. + const result = await engine.pushBookFile(book); if (result.uploaded) { await updateLastSyncedAt(Date.now()); } @@ -294,7 +245,7 @@ export const useWebDAVSync = (bookKey: string) => { // open retries — otherwise a transient hiccup would mark this // book "synced" for the rest of the session. fileSyncedRef.current = false; - if (e instanceof WebDAVRequestError && e.code === 'AUTH_FAILED') { + if (e instanceof FileSyncError && e.code === 'AUTH_FAILED') { eventDispatcher.dispatch('toast', { type: 'error', message: _('WebDAV authentication failed. Reconnect in Settings.'), @@ -303,7 +254,7 @@ export const useWebDAVSync = (bookKey: string) => { console.warn('WD book file push failed', e); } } - }, [allowPush, settings.webdav, getBookData, bookKey, appService, updateLastSyncedAt, _]); + }, [allowPush, settings.webdav, getBookData, bookKey, engine, updateLastSyncedAt, _]); /** * Push the local cover image to the remote, independent of @@ -324,21 +275,15 @@ export const useWebDAVSync = (bookKey: string) => { coverSyncedRef.current = true; const book = getBookData(bookKey)?.book; - if (!book || !appService) return; + if (!book || !engine) return; try { - await pushBookCover(settings.webdav!, book.hash, async () => { - const fp = getCoverFilename(book); - if (!(await appService.exists(fp, 'Books'))) return null; - const file = await appService.openFile(fp, 'Books'); - const bytes = await file.arrayBuffer(); - return { bytes, size: bytes.byteLength }; - }); + await engine.pushBookCover(book); } catch (e) { // Reset the lock so a manual "Sync now" or a subsequent open // can retry, mirroring `pushBookFileNow`'s recovery model. coverSyncedRef.current = false; - if (e instanceof WebDAVRequestError && e.code === 'AUTH_FAILED') { + if (e instanceof FileSyncError && e.code === 'AUTH_FAILED') { eventDispatcher.dispatch('toast', { type: 'error', message: _('WebDAV authentication failed. Reconnect in Settings.'), @@ -347,7 +292,7 @@ export const useWebDAVSync = (bookKey: string) => { console.warn('WD book cover push failed', e); } } - }, [allowPush, settings.webdav, getBookData, bookKey, appService, _]); + }, [allowPush, getBookData, bookKey, engine, _]); /** * Pull, merge, and persist. Uses the same per-config / per-note merge @@ -367,10 +312,10 @@ export const useWebDAVSync = (bookKey: string) => { const config = getConfig(bookKey); const book = getBookData(bookKey)?.book; - if (!config || !book) return false; + if (!config || !book || !engine) return false; try { - const result = await pullBookConfig(settings.webdav!, book, config); + const result = await engine.pullBookConfig(book, config); lastPulledAtRef.current = Date.now(); if (!result.applied || !result.mergedConfig) return false; @@ -416,7 +361,7 @@ export const useWebDAVSync = (bookKey: string) => { await updateLastSyncedAt(Date.now()); return true; } catch (e) { - if (e instanceof WebDAVRequestError && e.code === 'AUTH_FAILED') { + if (e instanceof FileSyncError && e.code === 'AUTH_FAILED') { eventDispatcher.dispatch('toast', { type: 'error', message: _('WebDAV authentication failed. Reconnect in Settings.'), @@ -435,6 +380,7 @@ export const useWebDAVSync = (bookKey: string) => { getViewsById, setConfig, saveConfig, + engine, envConfig, settings, updateLastSyncedAt, diff --git a/apps/readest-app/src/components/Toast.tsx b/apps/readest-app/src/components/Toast.tsx index be92c05c..bb3573e6 100644 --- a/apps/readest-app/src/components/Toast.tsx +++ b/apps/readest-app/src/components/Toast.tsx @@ -118,7 +118,7 @@ export const Toast = () => { toastMessage && (
= ({ settings }) => { // limit. Memoise so we don't recompute on every keystroke. const savedRoot = useMemo(() => normalizeRootPath(settings.rootPath || '/'), [settings.rootPath]); + // Provider for the engine-level cleanup helper (orphan GC). The browse + // pane is WebDAV-specific UI, so it builds the WebDAV provider directly. + const provider = useMemo(() => createWebDAVProvider(settings), [settings]); + // Absolute path of the per-book hash directories. When the user has // drilled into this exact path, each row's `entry.name` is a content // hash and we can swap in the human-readable book title from the @@ -69,7 +74,7 @@ const WebDAVBrowsePane: React.FC = ({ settings }) => { // can be string-compared against `currentPath` without further // normalisation. const booksDirPath = useMemo( - () => `${buildBasePath(settings.rootPath || '/')}/${WEBDAV_BOOKS_DIR}`, + () => `${buildBasePath(settings.rootPath || '/')}/${SYNC_BOOKS_DIR}`, [settings.rootPath], ); @@ -197,7 +202,7 @@ const WebDAVBrowsePane: React.FC = ({ settings }) => { for (let i = 0; i < targets.length; i++) { const t0 = targets[i]!; try { - const res = await deleteRemoteBookDir(settings, t0.hash); + const res = await deleteRemoteBookDir(provider, t0.hash); if (res.ok) { succeeded++; // Splice on success so the listing itself is the progress @@ -217,7 +222,7 @@ const WebDAVBrowsePane: React.FC = ({ settings }) => { }); } } catch (e) { - if (e instanceof WebDAVRequestError && e.code === 'AUTH_FAILED') { + if (e instanceof FileSyncError && e.code === 'AUTH_FAILED') { // Every remaining target would fail identically; stop // and surface a single re-auth toast. authFailed = true; diff --git a/apps/readest-app/src/components/settings/integrations/WebDAVForm.tsx b/apps/readest-app/src/components/settings/integrations/WebDAVForm.tsx index 70d61f25..ac29b8cf 100644 --- a/apps/readest-app/src/components/settings/integrations/WebDAVForm.tsx +++ b/apps/readest-app/src/components/settings/integrations/WebDAVForm.tsx @@ -7,21 +7,18 @@ import { useTranslation } from '@/hooks/useTranslation'; import { useSettingsStore } from '@/store/settingsStore'; import { useLibraryStore } from '@/store/libraryStore'; import { useWebDAVSyncStore } from '@/store/webdavSyncStore'; -import { isTauriAppPlatform } from '@/services/environment'; -import { tauriDownload, tauriUpload } from '@/utils/transfer'; import { eventDispatcher } from '@/utils/event'; import { - buildBasicAuthHeader, - buildRequestUrl, checkConnection, normalizeRootPath, WebDAVConnectResult, - WebDAVRequestError, -} from '@/services/webdav/WebDAVClient'; +} from '@/services/sync/providers/webdav/client'; import { type TranslationFunc } from '@/hooks/useTranslation'; -import { syncLibrary } from '@/services/webdav/WebDAVSync'; -import { buildWebDAVConnectSettings } from '@/services/webdav/webdavConnectSettings'; -import { getCoverFilename, getLocalBookFilename } from '@/utils/book'; +import { createWebDAVProvider } from '@/services/sync/providers/webdav/WebDAVProvider'; +import { buildWebDAVConnectSettings } from '@/services/sync/providers/webdav/connectSettings'; +import { FileSyncEngine } from '@/services/sync/file/engine'; +import { FileSyncError } from '@/services/sync/file/provider'; +import { createAppLocalStore } from '@/services/sync/file/appLocalStore'; import SubPageHeader from '../SubPageHeader'; import { BoxedList, @@ -68,7 +65,7 @@ const formatConnectError = (_: TranslationFunc, result: WebDAVConnectResult): st * showing the raw English `e.message` to the user. */ const formatSyncError = (_: TranslationFunc, e: unknown): string => { - if (e instanceof WebDAVRequestError) { + if (e instanceof FileSyncError) { switch (e.code) { case 'AUTH_FAILED': return _('WebDAV authentication failed. Reconnect in Settings.'); @@ -209,21 +206,23 @@ const WebDAVForm: React.FC = ({ onBack }) => { // enabled — anyone bothering to set up cloud sync wants those. Only // book files stay opt-in because they're bandwidth/storage heavy. const handleToggleSyncBooks = () => persistWebdav({ syncBooks: !(stored?.syncBooks ?? false) }); + const handleToggleFullSync = () => persistWebdav({ fullSync: !(stored?.fullSync ?? false) }); const handleStrategyChange = async (e: React.ChangeEvent) => { await persistWebdav({ strategy: e.target.value as typeof stored.strategy }); }; /** - * Manual "Sync now" — push every book in the local library up to the - * remote in a single sequential pass. We don't pull here; the per- - * book Reader hook handles incoming changes when the user opens a - * book. + * Manual "Sync now" — reconcile the local library with the remote over a + * bounded-concurrency pool. By default this is incremental: only books whose + * local copy differs from the shared library.json index are processed + * (`book.updatedAt` is the per-book change marker). The "Full Sync" toggle + * re-checks every book. The engine pulls peers' changes and pushes ours; the + * per-book Reader hook still handles live changes as the user reads. * - * Why sequential: shared WebDAV servers (NextCloud, Synology, …) are - * not happy with parallel PUTs from one user, and a steady linear - * walk gives us a usable progress indicator. The whole thing runs - * off-thread relative to the UI by virtue of being async — we just - * surface a status string and disable the button. + * Concurrency is capped (engine default 4) so shared WebDAV servers + * (NextCloud, Synology, …) aren't hammered while still hiding per-request + * latency. The run is async relative to the UI — we surface a status string + * and disable the button. */ const handleSyncNow = async () => { // Re-entrancy gate must read the live store, not the closure: a @@ -239,6 +238,12 @@ const WebDAVForm: React.FC = ({ onBack }) => { let currentLibrary = library ?? []; if (!libraryLoaded && appService) { currentLibrary = await appService.loadLibraryBooks(); + // Hydrate the store before syncing. The engine's addBookToLibrary / + // updateBookMetadata merge against the in-memory library; if it were + // still empty here, a downloaded book or a metadata update would persist + // as the *entire* library and clobber what's on disk. setLibrary also + // flips libraryLoaded so the per-book store calls see a loaded store. + useLibraryStore.getState().setLibrary(currentLibrary); } const eligibleBooks = currentLibrary.filter((b) => !b.deletedAt); @@ -257,173 +262,18 @@ const WebDAVForm: React.FC = ({ onBack }) => { beginSync(_('Syncing {{n}} / {{total}}', { n: 0, total: eligibleBooks.length })); try { - const result = await syncLibrary(stored, eligibleBooks, { + // The provider owns the WebDAV URL + auth + streaming transport; the + // shared local-store bridge owns all on-disk book/cover/config I/O + // (including the in-place vs hash-copy path resolution and the Tauri + // streaming fast path). This form no longer knows any WebDAV specifics. + const provider = createWebDAVProvider(stored); + const store = createAppLocalStore({ appService, settings, envConfig }); + const engine = new FileSyncEngine(provider, store); + const result = await engine.syncLibrary(eligibleBooks, { strategy: stored.strategy === 'prompt' ? 'silent' : stored.strategy, syncBooks: stored.syncBooks ?? false, + fullSync: stored.fullSync ?? false, deviceId: deviceId as string, - loadConfig: (book) => - appService ? appService.loadBookConfig(book, settings) : Promise.resolve(null), - loadBookFile: async (book) => { - if (!appService) return null; - // In-place imports live outside Books//; resolve to - // (book.filePath, 'None') when set. Hash-copy books fall - // through to the original Books-relative path. Same fallback - // pattern as cloudService.uploadBook so library Sync now - // treats in-place books as first-class. - const fp = book.filePath ?? getLocalBookFilename(book); - const base = book.filePath ? 'None' : 'Books'; - if (!(await appService.exists(fp, base))) return null; - const file = await appService.openFile(fp, base); - const bytes = await file.arrayBuffer(); - return { bytes, size: bytes.byteLength }; - }, - // Tauri-only: stream the book file straight from disk to the - // WebDAV server via Rust-side `upload_file`, never letting the - // bytes land in the JS heap. Without this, syncing a library - // with multiple multi-hundred-megabyte PDFs accumulates - // ArrayBuffers that V8 can't free fast enough between - // sequential `pushBookFile` calls — the renderer eventually - // hits its heap ceiling and the WebView crashes mid-sync, - // surfacing as a blank white screen on desktop and as a - // binder-OOM kill on Android. The metadata-only fast path - // (open file just to read `.size`) keeps the HEAD short- - // circuit working the same way the buffered path does. - loadBookFileStreaming: isTauriAppPlatform() - ? async (book) => { - if (!appService) return null; - const fp = book.filePath ?? getLocalBookFilename(book); - const base = book.filePath ? 'None' : 'Books'; - if (!(await appService.exists(fp, base))) return null; - const file = await appService.openFile(fp, base); - const size = file.size; - // openFile returns a File-like handle; close eagerly when - // the platform exposes it so the Tauri side can re-open - // the path for the streamed PUT without holding two FDs. - const closable = file as { close?: () => Promise }; - if (closable.close) await closable.close(); - const dst = await appService.resolveFilePath(fp, base); - return { - size, - upload: async (remoteUrl, headers) => { - try { - // tauriUpload's TS type says Map, but its Tauri - // command on the Rust side accepts a JSON object → - // HashMap. The internal `headers ?? - // {}` default already proves a plain object works, - // so cast and pass the headers object directly - // rather than building a Map (which Tauri's IPC - // serialiser handles less consistently). - await tauriUpload( - remoteUrl, - dst, - 'PUT', - undefined, - headers as unknown as Map, - ); - return true; - } catch (e) { - console.warn('WD library sync: tauriUpload failed', book.hash, e); - return false; - } - }, - }; - } - : undefined, - loadBookCover: async (book) => { - // Covers are best-effort — books without one (TXT/MD without - // metadata, custom imports without art) just return null and - // syncLibrary skips them silently. - if (!appService) return null; - const fp = getCoverFilename(book); - if (!(await appService.exists(fp, 'Books'))) return null; - const file = await appService.openFile(fp, 'Books'); - const bytes = await file.arrayBuffer(); - return { bytes, size: bytes.byteLength }; - }, - saveBookFile: async (book, bytes) => { - if (!appService) return; - const fp = getLocalBookFilename(book); - await appService.writeFile(fp, 'Books', bytes); - }, - // Tauri-only: stream the book straight to disk via the Rust - // side instead of slurping it into a JS ArrayBuffer first. The - // WebView<->Tauri IPC bridge cannot handle multi-megabyte - // buffers on Android (the renderer is binder-killed mid-write), - // so for any non-trivial epub/pdf this is the *only* path that - // works reliably on mobile. - downloadBookFile: isTauriAppPlatform() - ? async (book, remotePath) => { - if (!appService) return false; - const url = buildRequestUrl(stored.serverUrl, remotePath); - const headers = { - Authorization: buildBasicAuthHeader(stored.username, stored.password), - }; - // The Rust downloader writes the file verbatim and does - // NOT create parent dirs — make sure the per-hash folder - // under Books exists before kicking off the stream. - try { - if (!(await appService.exists(book.hash, 'Books'))) { - await appService.createDir(book.hash, 'Books', true); - } - } catch (e) { - console.warn('WD library sync: mkdir failed', book.hash, e); - } - const dst = await appService.resolveFilePath(getLocalBookFilename(book), 'Books'); - try { - await tauriDownload(url, dst, undefined, headers); - return true; - } catch (e) { - console.warn('WD library sync: tauriDownload failed', book.hash, e); - return false; - } - } - : undefined, - saveBookCover: async (book, bytes) => { - if (!appService) return; - const fp = getCoverFilename(book); - await appService.writeFile(fp, 'Books', bytes); - }, - saveBookConfig: async (book, config) => { - if (!appService) return; - await appService.saveBookConfig(book, config, settings); - }, - addBookToLibrary: async (book) => { - if (!appService) return; - try { - book.coverImageUrl = await appService.generateCoverImageUrl(book); - } catch (e) { - // Missing or broken cover shouldn't block adding the book — - // the bookshelf renders a placeholder when coverImageUrl - // is empty. - console.warn('WD library sync: cover URL generation failed', book.hash, e); - book.coverImageUrl = null; - } - book.syncedAt = Date.now(); - book.downloadedAt = Date.now(); - if (!book.metaHash) book.metaHash = book.hash; - const { library, setLibrary } = useLibraryStore.getState(); - // Avoid duplicates if the user runs Sync now twice quickly. - if (library.find((b) => b.hash === book.hash)) return; - const newLibrary = [...library, book]; - await appService.saveLibraryBooks(newLibrary); - // Update the store last so subscribers re-render against a - // library that's already persisted on disk. - setLibrary(newLibrary); - }, - updateBookMetadata: async (book) => { - if (!appService) return; - // The cover bytes were just refreshed via saveBookCover, so - // regenerate the device-local blob URL the bookshelf renders. - try { - book.coverImageUrl = await appService.generateCoverImageUrl(book); - } catch (e) { - console.warn('WD library sync: cover URL generation failed', book.hash, e); - } - book.syncedAt = Date.now(); - // updateBook persists via saveLibraryBooks and refreshes the store, - // so the new title / author / cover show up without a reload. - await useLibraryStore.getState().updateBook(envConfig, book); - }, onProgress: ({ book, index, total, action }) => { const actionStr = action === 'downloading' ? _('Downloading') : _('Uploading'); updateProgress( @@ -434,58 +284,23 @@ const WebDAVForm: React.FC = ({ onBack }) => { }); await persistWebdav({ lastSyncedAt: Date.now() }); - // Build a compact, accurate summary. Downloads happen regardless - // of the `syncBooks` toggle, so they're always part of the toast; - // the upload counters are only included when there was anything - // to push (otherwise they'd just be a wall of zeros). - const parts: string[] = []; - if (result.booksDownloaded > 0) { - parts.push(_('downloaded {{n}} book(s)', { n: result.booksDownloaded })); - } - if (result.configsDownloaded > 0) { - parts.push(_('pulled progress for {{n}} book(s)', { n: result.configsDownloaded })); - } - if (result.metadataUpdated > 0) { - parts.push(_('updated metadata for {{n}} book(s)', { n: result.metadataUpdated })); - } - if (result.configsUploaded > 0) { - parts.push(_('pushed {{n}} config(s)', { n: result.configsUploaded })); - } - if (stored.syncBooks && result.filesUploaded > 0) { - parts.push(_('uploaded {{n}} new file(s)', { n: result.filesUploaded })); - } - // Build the toast in two pieces so we can render the details on - // their own lines on mobile. The Toast component truncates - // single-line `info` messages (max-width + `truncate`), which - // chops the long detail string on small screens. Two ways out: - // 1. Use `success` type, which renders multi-line and shows a - // dismiss button — picked when there's actionable detail. - // 2. Stick with `info` for the short "everything up to date" - // string, which always fits in one line anyway. - // The detail bullets are joined with `\n` because Toast's - // renderer (Toast.tsx) already splits on newlines into
s. - let toastType: 'info' | 'success' | 'warning' = 'info'; - let summary: string; + // Keep the toast as simple as the native cloud sync: a single-line + // "{{count}} book(s) synced" info message. Failures still surface as a + // warning so a partial sync isn't reported as a clean success. if (result.failures > 0) { - toastType = 'warning'; - summary = _('Sync finished with {{failed}} failure(s). {{ok}} ok.', { - failed: result.failures, - ok: Math.max(0, result.totalBooks - result.failures), + eventDispatcher.dispatch('toast', { + type: 'warning', + message: _('Sync finished with {{failed}} failure(s). {{ok}} ok.', { + failed: result.failures, + ok: Math.max(0, result.totalBooks - result.failures), + }), }); - if (parts.length > 0) { - summary += '\n' + parts.map((p) => `• ${p}`).join('\n'); - } - } else if (parts.length > 0) { - toastType = 'success'; - const heading = _('Sync complete'); - summary = `${heading}\n${parts.map((p) => `• ${p}`).join('\n')}`; } else { - summary = _('Everything is already up to date.'); + eventDispatcher.dispatch('toast', { + type: 'info', + message: _('{{count}} book(s) synced', { count: result.booksSynced }), + }); } - eventDispatcher.dispatch('toast', { - type: toastType, - message: summary, - }); } catch (e) { const message = formatSyncError(_, e); eventDispatcher.dispatch('toast', { type: 'error', message }); @@ -522,6 +337,12 @@ const WebDAVForm: React.FC = ({ onBack }) => { checked={stored.syncBooks ?? false} onChange={handleToggleSyncBooks} /> + /`, so the book-file + * helpers resolve to `(book.filePath, 'None')` when `filePath` is set and fall + * through to the hash-copy `Books`-relative path otherwise — mirroring + * `cloudService.uploadBook` so sync treats in-place books as first-class. + */ +export const createAppLocalStore = ({ + appService, + settings, + envConfig, +}: { + appService: AppService; + settings: SystemSettings; + envConfig: EnvConfigType; +}): LocalStore => ({ + loadConfig: (book) => appService.loadBookConfig(book, settings), + saveBookConfig: (book, config) => appService.saveBookConfig(book, config, settings), + + loadBookFile: async (book) => { + const fp = book.filePath ?? getLocalBookFilename(book); + const base = book.filePath ? 'None' : 'Books'; + if (!(await appService.exists(fp, base))) return null; + const file = await appService.openFile(fp, base); + const bytes = await file.arrayBuffer(); + return { bytes, size: bytes.byteLength }; + }, + + resolveLocalBookPath: async (book) => { + const fp = book.filePath ?? getLocalBookFilename(book); + const base = book.filePath ? 'None' : 'Books'; + if (!(await appService.exists(fp, base))) return null; + const file = await appService.openFile(fp, base); + const size = file.size; + // Release the FD before streaming so the Tauri side can re-open the path + // for the PUT without contending. + const closable = file as { close?: () => Promise }; + if (closable.close) await closable.close(); + const path = await appService.resolveFilePath(fp, base); + return { path, size }; + }, + + saveBookFile: async (book, bytes) => { + await appService.writeFile(getLocalBookFilename(book), 'Books', bytes); + }, + + prepareLocalBookPath: async (book) => { + // The Rust downloader writes the file verbatim and does NOT create parent + // dirs — make sure the per-hash folder under Books exists first. + try { + if (!(await appService.exists(book.hash, 'Books'))) { + await appService.createDir(book.hash, 'Books', true); + } + } catch (e) { + console.warn('createAppLocalStore: mkdir failed', book.hash, e); + } + return appService.resolveFilePath(getLocalBookFilename(book), 'Books'); + }, + + loadBookCover: async (book) => { + const fp = getCoverFilename(book); + if (!(await appService.exists(fp, 'Books'))) return null; + const file = await appService.openFile(fp, 'Books'); + const bytes = await file.arrayBuffer(); + return { bytes, size: bytes.byteLength }; + }, + + saveBookCover: async (book, bytes) => { + await appService.writeFile(getCoverFilename(book), 'Books', bytes); + }, + + addBookToLibrary: async (book) => { + try { + book.coverImageUrl = await appService.generateCoverImageUrl(book); + } catch (e) { + // Missing/broken cover shouldn't block adding the book — the bookshelf + // renders a placeholder when coverImageUrl is empty. + console.warn('createAppLocalStore: cover URL generation failed', book.hash, e); + book.coverImageUrl = null; + } + book.syncedAt = Date.now(); + book.downloadedAt = Date.now(); + if (!book.metaHash) book.metaHash = book.hash; + // Hydrate from disk if the store hasn't loaded yet. Merging against an + // empty in-memory array would persist this book as the *entire* library + // and clobber whatever is on disk. Mirrors useLibraryStore.updateBooks' + // hardening; the Sync-now caller also hydrates up front, so this is + // belt-and-suspenders for a data-loss path. + let library = useLibraryStore.getState().library; + if (!useLibraryStore.getState().libraryLoaded) { + library = await appService.loadLibraryBooks(); + useLibraryStore.getState().setLibrary(library); + } + // Avoid duplicates if the user runs Sync now twice quickly. + if (library.find((b) => b.hash === book.hash)) return; + const newLibrary = [...library, book]; + await appService.saveLibraryBooks(newLibrary); + // Update the store last so subscribers re-render against a library that's + // already persisted on disk. + useLibraryStore.getState().setLibrary(newLibrary); + }, + + updateBookMetadata: async (book) => { + // The cover bytes were just refreshed via saveBookCover, so regenerate the + // device-local blob URL the bookshelf renders. + try { + book.coverImageUrl = await appService.generateCoverImageUrl(book); + } catch (e) { + console.warn('createAppLocalStore: cover URL generation failed', book.hash, e); + } + book.syncedAt = Date.now(); + // Hydrate before updating: updateBook merges against the in-memory library, + // so an unloaded store would persist an empty (or single-book) library and + // clobber the disk. See the same guard in addBookToLibrary. + if (!useLibraryStore.getState().libraryLoaded) { + useLibraryStore.getState().setLibrary(await appService.loadLibraryBooks()); + } + // updateBook persists via saveLibraryBooks and refreshes the store, so the + // new title / author / cover show up without a reload. + await useLibraryStore.getState().updateBook(envConfig, book); + }, +}); diff --git a/apps/readest-app/src/services/sync/file/engine.ts b/apps/readest-app/src/services/sync/file/engine.ts new file mode 100644 index 00000000..74ddb486 --- /dev/null +++ b/apps/readest-app/src/services/sync/file/engine.ts @@ -0,0 +1,736 @@ +import { Book, BookConfig, BookNote } from '@/types/book'; +import { FileHead, FileSyncError, FileSyncProvider } from './provider'; +import { LocalStore } from './localStore'; +import { + ancestorsOf, + buildBasePath, + buildBookConfigPath, + buildBookCoverPath, + buildBookDirPath, + buildBookFilePath, + buildLibraryPath, + SYNC_BOOKS_DIR, + SYNC_BOOK_CONFIG_FILE, + SYNC_BOOK_COVER_FILE, +} from './layout'; +import { + buildRemotePayload, + parseRemotePayload, + parseRemoteLibraryIndex, + RemoteLibraryIndex, +} from './wire'; +import { isRemoteBookMetadataNewer, mergeBookConfig, mergeBookMetadata } from './merge'; + +export type SyncStrategy = 'silent' | 'send' | 'receive'; + +export interface PullResult { + /** True when the remote had a config and we merged something into local. */ + applied: boolean; + /** The merged config to be written back into the local store. */ + mergedConfig?: BookConfig; + /** When non-empty, these are the notes after merge — use them to update the live view. */ + mergedNotes?: BookNote[]; + /** The remote's writerDeviceId, useful for diagnostics. */ + remoteDeviceId?: string; +} + +export interface PushBookFileResult { + /** True when bytes were uploaded; false when the upload was skipped. */ + uploaded: boolean; + /** Reason for the skip, when applicable — surfaced for diagnostics. */ + reason?: 'remote-matches' | 'no-source' | 'disabled'; +} + +export interface DeleteRemoteBookDirResult { + /** True when the server confirmed deletion (or the dir was already gone). */ + ok: boolean; + /** Compact reason string when `ok === false`, for the failure toast. */ + reason?: string; +} + +export interface SyncFailureEntry { + hash: string; + title: string; + reason: string; + /** Which phase of the per-book pipeline failed; helps users self-triage. */ + phase: 'download' | 'upload-config' | 'upload-file' | 'upload-cover'; +} + +/** + * Aggregate result of a library-wide sync. Counters are kept granular so the + * UI can render an honest "X uploaded, Y already in sync, Z failed" toast. + */ +export interface SyncLibraryResult { + totalBooks: number; + configsUploaded: number; + configsDownloaded: number; + filesUploaded: number; + filesAlreadyInSync: number; + coversUploaded: number; + booksDownloaded: number; + /** Already-local books whose metadata was refreshed from a newer index copy (#4756). */ + metadataUpdated: number; + /** Distinct books that had any sync activity (pushed, downloaded, or reconciled). */ + booksSynced: number; + failures: number; + /** Per-book failure breakdown for the diagnostic log in the Settings UI. */ + failedBooks: SyncFailureEntry[]; +} + +export interface SyncLibraryOptions { + syncBooks: boolean; + strategy?: SyncStrategy; + /** Stable per-device id; written into every config envelope. */ + deviceId: string; + /** + * When false (default), only books whose local copy differs from the shared + * library.json index are processed — `book.updatedAt` bumps on every + * progress / notes / metadata save, so the index is a reliable per-book + * change marker. When true, every book is re-checked (the original full + * walk), an escape hatch for drift or a first sync to a fresh remote. + */ + fullSync?: boolean; + /** + * Max books processed concurrently per phase (download / reconcile / push). + * Defaults to 4. A bounded pool keeps shared WebDAV servers happy while + * still hiding per-request latency. + */ + concurrency?: number; + /** + * Optional progress callback fired before each book is processed, + * suitable for driving a UI like "Syncing 3 / 42 — Project Hail Mary". + */ + onProgress?: (info: { book: Book; index: number; total: number; action?: string }) => void; +} + +/** + * Reduce an arbitrary error to a short, single-line description for the + * per-book failure breakdown in {@link SyncLibraryResult}. Preserves the + * semantically useful bits (HTTP status, the `code` enum), strips stack + * traces / server XML, and caps at 200 chars. + */ +const formatFailureReason = (e: unknown): string => { + let message: string; + if (e instanceof FileSyncError) { + const parts: string[] = []; + if (e.code) parts.push(e.code); + if (typeof e.status === 'number') parts.push(`HTTP ${e.status}`); + parts.push(e.message || 'Request failed'); + message = parts.join(' · '); + } else if (e instanceof Error) { + message = e.message || e.name || 'Unknown error'; + } else { + message = String(e); + } + message = message.replace(/\s+/g, ' ').trim(); + return message.length > 200 ? `${message.slice(0, 197)}...` : message; +}; + +/** + * Delete the per-book directory `/Readest/books//` — file, + * cover and config.json — in one round-trip. Used by the remote-browser + * cleanup mode to evict orphans. AUTH failures rethrow (a global condition + * the caller surfaces as a single re-auth toast); every other failure is + * folded into `{ ok: false, reason }` so a batch loop can aggregate. + * + * Standalone (not a method) because it needs no {@link LocalStore} — the + * WebDAV-specific browse UI builds a provider and calls it directly. + */ +export const deleteRemoteBookDir = async ( + provider: FileSyncProvider, + bookHash: string, +): Promise => { + const path = buildBookDirPath(provider.rootPath, bookHash); + try { + await provider.deleteDir(path); + return { ok: true }; + } catch (e) { + if (e instanceof FileSyncError && e.code === 'AUTH_FAILED') throw e; + return { ok: false, reason: e instanceof Error ? e.message : String(e) }; + } +}; + +/** + * Run `worker` over `items` with at most `limit` in flight at once. A bounded + * pool: `limit` runner loops each pull the next index off a shared cursor until + * the list drains. JS's single-threaded event loop makes the cursor increment + * and the per-book result mutations race-free between await points. + */ +const runPool = async ( + items: T[], + limit: number, + worker: (item: T, index: number) => Promise, +): Promise => { + if (items.length === 0) return; + let cursor = 0; + const runners = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => { + while (cursor < items.length) { + const index = cursor; + cursor += 1; + await worker(items[index]!, index); + } + }); + await Promise.all(runners); +}; + +/** + * Provider-agnostic file-sync orchestration: progress + booknote merge per + * book, library-wide push/pull with last-writer-wins metadata reconciliation, + * and HEAD-short-circuited binary upload. All remote I/O goes through a + * {@link FileSyncProvider}; all local I/O goes through a {@link LocalStore}. + */ +export class FileSyncEngine { + constructor( + private readonly provider: FileSyncProvider, + private readonly store: LocalStore, + ) {} + + /** + * Pull `/Readest/books//config.json`, merge into the + * provided local config, and return the merged result. The caller writes + * the merged config back (so the engine stays free of store-write side + * effects here). `applied: false` when the remote file is absent/malformed. + */ + async pullBookConfig(book: Book, localConfig: BookConfig): Promise { + const path = buildBookConfigPath(this.provider.rootPath, book.hash); + const remote = parseRemotePayload(await this.provider.readText(path)); + if (!remote) return { applied: false }; + const { config, notes } = mergeBookConfig(localConfig, remote); + return { + applied: true, + mergedConfig: config, + mergedNotes: notes, + remoteDeviceId: remote.writerDeviceId, + }; + } + + /** + * Push the local BookConfig to the remote, creating parent dirs as needed. + * A 409 (parent vanished between MKCOL and PUT) triggers one re-ensure + + * retry. Deciding *whether* to push is the caller's job; this is the dumb + * mechanism. + */ + async pushBookConfig(book: Book, config: BookConfig, deviceId: string): Promise { + const dirPath = buildBookDirPath(this.provider.rootPath, book.hash); + const path = buildBookConfigPath(this.provider.rootPath, book.hash); + const dirs = [...ancestorsOf(`${dirPath}/.placeholder`), dirPath]; + await this.provider.ensureDir(dirs); + const body = JSON.stringify(buildRemotePayload(book, config, deviceId)); + try { + await this.provider.writeText(path, body); + } catch (e) { + if (e instanceof FileSyncError && e.status === 409) { + await this.provider.ensureDir(dirs); + await this.provider.writeText(path, body); + return; + } + throw e; + } + } + + /** + * Upload the book binary to `/Readest/books//.<ext>`. + * HEAD-probe + size compare skips re-uploading an already-mirrored book. + * Streaming (provider.uploadStream, Tauri only) is preferred — constant JS + * heap regardless of book size; web falls back to buffered writeBinary. + */ + async pushBookFile(book: Book): Promise<PushBookFileResult> { + const dirPath = buildBookDirPath(this.provider.rootPath, book.hash); + const path = buildBookFilePath(this.provider.rootPath, book); + const dirs = [...ancestorsOf(`${dirPath}/.placeholder`), dirPath]; + + let remoteHead: FileHead | null = null; + try { + remoteHead = await this.provider.head(path); + } catch (e) { + if (!(e instanceof FileSyncError) || e.code !== 'NETWORK') throw e; + } + + // Streaming path: resolve the on-disk path + size only, then stream the + // bytes straight from disk. The metadata fetch never reads the body, so + // heap stays flat even for gigabyte-scale PDFs. + if (this.provider.uploadStream) { + const src = await this.store.resolveLocalBookPath(book); + if (src) { + if (remoteHead && remoteHead.size === src.size) { + return { uploaded: false, reason: 'remote-matches' }; + } + await this.provider.ensureDir(dirs); + let ok = await this.provider.uploadStream(path, src.path); + if (!ok) { + // Mirror the buffered path's one-shot retry: a parent may have been + // recreated mid-PUT (409). Re-ensure directories and try once more. + await this.provider.ensureDir(dirs); + ok = await this.provider.uploadStream(path, src.path); + if (!ok) throw new FileSyncError('Streaming upload failed', 'NETWORK'); + } + return { uploaded: true }; + } + // src null — book isn't on this device via the streaming resolver; fall + // through to the buffered loader as a last resort. + } + + const local = await this.store.loadBookFile(book); + if (!local) return { uploaded: false, reason: 'no-source' }; + if (remoteHead && remoteHead.size === local.size) { + return { uploaded: false, reason: 'remote-matches' }; + } + await this.provider.ensureDir(dirs); + try { + await this.provider.writeBinary(path, local.bytes); + } catch (e) { + if (e instanceof FileSyncError && e.status === 409) { + await this.provider.ensureDir(dirs); + await this.provider.writeBinary(path, local.bytes); + } else { + throw e; + } + } + return { uploaded: true }; + } + + /** + * Upload the book's cover image to `<rootPath>/Readest/books/<hash>/cover.png`. + * Same HEAD-probe + size-compare idempotency as {@link pushBookFile}. Covers + * are best-effort: a book without a local cover resolves to `no-source`. + */ + async pushBookCover(book: Book): Promise<PushBookFileResult> { + const dirPath = buildBookDirPath(this.provider.rootPath, book.hash); + const path = buildBookCoverPath(this.provider.rootPath, book.hash); + const dirs = [...ancestorsOf(`${dirPath}/.placeholder`), dirPath]; + + let remoteHead: FileHead | null = null; + try { + remoteHead = await this.provider.head(path); + } catch (e) { + if (!(e instanceof FileSyncError) || e.code !== 'NETWORK') throw e; + } + + const local = await this.store.loadBookCover(book); + if (!local) return { uploaded: false, reason: 'no-source' }; + if (remoteHead && remoteHead.size === local.size) { + return { uploaded: false, reason: 'remote-matches' }; + } + await this.provider.ensureDir(dirs); + try { + await this.provider.writeBinary(path, local.bytes, 'image/png'); + } catch (e) { + if (e instanceof FileSyncError && e.status === 409) { + await this.provider.ensureDir(dirs); + await this.provider.writeBinary(path, local.bytes, 'image/png'); + } else { + throw e; + } + } + return { uploaded: true }; + } + + /** GET the remote cover.png bytes for a hash, or null when absent. */ + async pullBookCover(bookHash: string): Promise<ArrayBuffer | null> { + return this.provider.readBinary(buildBookCoverPath(this.provider.rootPath, bookHash)); + } + + /** GET + parse the shared library.json index, or null when absent/malformed. */ + async pullLibraryIndex(): Promise<RemoteLibraryIndex | null> { + const path = buildLibraryPath(this.provider.rootPath); + return parseRemoteLibraryIndex(await this.provider.readText(path)); + } + + /** PUT the shared library.json index, creating its parent dirs. */ + async pushLibraryIndex(index: RemoteLibraryIndex): Promise<void> { + const path = buildLibraryPath(this.provider.rootPath); + await this.provider.ensureDir(ancestorsOf(path)); + await this.provider.writeText(path, JSON.stringify(index)); + } + + /** + * Sync every book in `books` against the remote in sequence (predictable + * progress bar; no parallel PUTs that upset shared servers). Per book: + * pull index → reconcile metadata (LWW) → discover remote-only books and + * download them → pull-merge-push each local config + cover + (optionally) + * file → re-push the merged index. + * + * Strategy gating: 'silent' two-way, 'send' push-only (blind, local + * authoritative), 'receive' pull-only. Single-book failures are caught and + * counted so one bad apple never aborts the rest of the library. + */ + async syncLibrary(books: Book[], options: SyncLibraryOptions): Promise<SyncLibraryResult> { + const result: SyncLibraryResult = { + totalBooks: books.length, + configsUploaded: 0, + configsDownloaded: 0, + filesUploaded: 0, + filesAlreadyInSync: 0, + coversUploaded: 0, + booksDownloaded: 0, + metadataUpdated: 0, + booksSynced: 0, + failures: 0, + failedBooks: [], + }; + + // Distinct books touched in any direction — the single "N book(s) synced" + // number the UI surfaces. Tracked as a set because the per-action counters + // overlap (a Full-Sync re-check both reconciles and re-pushes the same + // book, and one book can push a config + cover + file). + const syncedHashes = new Set<string>(); + + const strategy = options.strategy || 'silent'; + const canPull = strategy !== 'send'; + const canPush = strategy !== 'receive'; + + let remoteIndex: RemoteLibraryIndex | null = null; + if (canPull) { + try { + remoteIndex = await this.pullLibraryIndex(); + } catch (e) { + console.warn('file sync: failed to pull index', e); + } + } + + const allBooksMap = new Map<string, Book>(); + for (const b of books) { + allBooksMap.set(b.hash, b); + } + + const fullSync = options.fullSync ?? false; + const concurrency = Math.max(1, options.concurrency ?? 4); + + // Incremental cursor: a book needs a push only when its local copy is newer + // than (or absent from) the shared library.json index. `book.updatedAt` + // bumps on every progress / notes / metadata save, so the index is a + // reliable per-book change marker. When no index is available (send mode, + // or a failed pull) every local book counts as new and is pushed. + const remoteByHash = new Map<string, Book>(); + if (remoteIndex?.books) { + for (const rb of remoteIndex.books) { + if (!rb.deletedAt) remoteByHash.set(rb.hash, rb); + } + } + const isLocalNewer = (book: Book): boolean => { + const remote = remoteByHash.get(book.hash); + if (!remote) return true; + return (book.updatedAt ?? 0) > (remote.updatedAt ?? 0); + }; + + const remoteBooksToDownload: Book[] = []; + // The remote source of truth for a book's on-disk filename is the per-hash + // directory listing — NOT the book's title (which may be stale). We always + // resolve the path by listing the hash dir. + const explicitRemotePaths = new Map<string, string>(); + + // Metadata reconciliation for books present BOTH locally and in the shared + // library.json (#4756). Last-writer-wins on `book.updatedAt`: when a peer's + // indexed copy is strictly newer, pull its title / author / cover down. + // Updating allBooksMap with the merged copy also stops the final index + // re-push from clobbering the peer's newer metadata with this device's + // stale copy. + if (canPull && remoteIndex && remoteIndex.books) { + const remoteNewer = remoteIndex.books.filter((rb) => { + if (rb.deletedAt) return false; + const local = allBooksMap.get(rb.hash); + return !!local && !local.deletedAt && isRemoteBookMetadataNewer(local, rb); + }); + await runPool(remoteNewer, concurrency, async (rb) => { + const local = allBooksMap.get(rb.hash)!; + const merged = mergeBookMetadata(local, rb); + // Re-pull the cover so a changed cover travels with the metadata. The + // subsequent push-side pushBookCover HEAD/size short-circuit then + // matches (local now equals remote), so we never bounce it back up. + try { + const coverBytes = await this.pullBookCover(rb.hash); + if (coverBytes) await this.store.saveBookCover(merged, coverBytes); + } catch (e) { + console.warn('file sync: metadata cover pull failed', rb.hash, e); + } + // Incremental only: the per-book push loop below skips remote-newer + // books, so pull their config here too — otherwise a peer's progress / + // notes wouldn't propagate without re-walking every book. In full-sync + // mode the push loop pulls each config, so we skip this to avoid a + // duplicate GET. + if (!fullSync) { + try { + const localConfig = (await this.store.loadConfig(merged)) ?? { + updatedAt: 0, + booknotes: [], + }; + const pull = await this.pullBookConfig(merged, localConfig); + if (pull.applied && pull.mergedConfig) { + await this.store.saveBookConfig(merged, pull.mergedConfig); + result.configsDownloaded += 1; + } + } catch (e) { + console.warn('file sync: metadata config pull failed', rb.hash, e); + } + } + try { + await this.store.updateBookMetadata(merged); + allBooksMap.set(rb.hash, merged); + result.metadataUpdated += 1; + syncedHashes.add(rb.hash); + } catch (e) { + console.warn('file sync: metadata update failed', rb.hash, e); + } + }); + } + + if (canPull) { + const candidateHashes = new Set<string>(); + + // 1) Seed with hashes from the remote index (when the file exists). + if (remoteIndex && remoteIndex.books) { + for (const rb of remoteIndex.books) { + if (!allBooksMap.has(rb.hash) && !rb.deletedAt) { + candidateHashes.add(rb.hash); + // Provisionally register the indexed book — fields refreshed below + // once we've inspected the actual hash dir. + allBooksMap.set(rb.hash, rb); + } + } + } + + // 2) Also scan the books/ directory so legacy uploads (no library.json + // entry) and index/disk drift are still picked up. + try { + const booksDirPath = `${buildBasePath(this.provider.rootPath)}/${SYNC_BOOKS_DIR}`; + const dirEntries = await this.provider.list(booksDirPath); + for (const entry of dirEntries) { + if (entry.isDirectory && !allBooksMap.has(entry.name)) { + candidateHashes.add(entry.name); + } + } + } catch (e) { + // 404 is normal if the user has never pushed anything yet. + console.warn('file sync: failed to list books directory', e); + } + + // 3) For every candidate, look inside its hash directory to find the + // actual book file (the only entry that isn't config.json/cover.png). + for (const hash of candidateHashes) { + try { + const hashDirPath = `${buildBasePath(this.provider.rootPath)}/${SYNC_BOOKS_DIR}/${hash}`; + const hashDirEntries = await this.provider.list(hashDirPath); + const fileEntry = hashDirEntries.find( + (e) => + !e.isDirectory && e.name !== SYNC_BOOK_CONFIG_FILE && e.name !== SYNC_BOOK_COVER_FILE, + ); + if (!fileEntry) continue; + + const extMatch = fileEntry.name.match(/\.([^.]+)$/); + const ext = extMatch && extMatch[1] ? extMatch[1].toUpperCase() : 'EPUB'; + const format = ext as Book['format']; + const title = fileEntry.name.replace(/\.[^.]+$/, ''); + + // If the index already gave us a book object, refresh the fields + // that might be wrong/stale from a previous buggy push. + const existing = allBooksMap.get(hash); + const book: Book = existing + ? { + ...existing, + format, + title: + !existing.title || existing.title.toLowerCase().endsWith(`.${ext.toLowerCase()}`) + ? title + : existing.title, + sourceTitle: title, + updatedAt: existing.updatedAt || Date.now(), + createdAt: existing.createdAt || Date.now(), + } + : { + hash, + format, + title, + sourceTitle: title, + author: 'Unknown', + createdAt: Date.now(), + updatedAt: Date.now(), + }; + + explicitRemotePaths.set(hash, fileEntry.path); + remoteBooksToDownload.push(book); + allBooksMap.set(hash, book); + } catch (e) { + console.warn('file sync: failed to inspect hash dir', hash, e); + } + } + } + + // Discovery+download is NOT gated on `syncBooks`. The toggle controls + // whether the *push* side ships binaries to the remote — pulling books + // that exist remotely but not locally is the whole point of a sync. + if (canPull) { + let downloadStarted = 0; + await runPool(remoteBooksToDownload, concurrency, async (rb) => { + options.onProgress?.({ + book: rb, + index: downloadStarted, + total: remoteBooksToDownload.length, + action: 'downloading', + }); + downloadStarted += 1; + try { + const explicitPath = explicitRemotePaths.get(rb.hash); + // Prefer the streaming downloader. On Tauri/Android we MUST take + // this path — moving a 30 MB epub through the WebView<->Rust IPC + // bridge as a single Uint8Array crashes the renderer. + let written = false; + if (this.provider.downloadStream && explicitPath) { + const dst = await this.store.prepareLocalBookPath(rb); + written = await this.provider.downloadStream(explicitPath, dst); + } else { + const remotePath = explicitPath ?? buildBookFilePath(this.provider.rootPath, rb); + const fileBytes = await this.provider.readBinary(remotePath); + if (fileBytes) { + await this.store.saveBookFile(rb, fileBytes); + written = true; + } + } + if (written) { + try { + const coverBytes = await this.pullBookCover(rb.hash); + if (coverBytes) await this.store.saveBookCover(rb, coverBytes); + } catch (e) { + console.warn('file sync: cover download failed', rb.hash, e); + } + // Pull the remote config so progress, bookmarks and annotations + // travel with the book. Best-effort: a missing config is not a + // failure. + try { + const emptyLocal: BookConfig = { updatedAt: 0, booknotes: [] }; + const pullResult = await this.pullBookConfig(rb, emptyLocal); + if (pullResult.applied && pullResult.mergedConfig) { + await this.store.saveBookConfig(rb, pullResult.mergedConfig); + result.configsDownloaded += 1; + } + } catch (e) { + console.warn('file sync: config download failed', rb.hash, e); + } + await this.store.addBookToLibrary(rb); + result.booksDownloaded += 1; + syncedHashes.add(rb.hash); + } else { + // No bytes returned (typically a 404 we couldn't resolve). + result.failures += 1; + result.failedBooks.push({ + hash: rb.hash, + title: rb.title || rb.hash, + phase: 'download', + reason: 'No bytes returned (file may have been moved or deleted on the server)', + }); + console.warn('file sync: book download produced no bytes', rb.hash, explicitPath); + } + } catch (e) { + result.failures += 1; + result.failedBooks.push({ + hash: rb.hash, + title: rb.title || rb.hash, + phase: 'download', + reason: formatFailureReason(e), + }); + console.warn('file sync: book download failed', rb.hash, e); + } + }); + } + + // Books we just downloaded already exist on the remote — don't re-push + // them. Only push books already present in the caller-supplied library. + const downloadedHashes = new Set(remoteBooksToDownload.map((b) => b.hash)); + // Incremental (default): push only books that changed locally since the + // last index push. Full-sync re-checks everything. + const booksToPush = books.filter( + (b) => !b.deletedAt && !downloadedHashes.has(b.hash) && (fullSync || isLocalNewer(b)), + ); + result.totalBooks = booksToPush.length; + + if (canPush && booksToPush.length > 0) { + let pushStarted = 0; + await runPool(booksToPush, concurrency, async (book) => { + options.onProgress?.({ + book, + index: pushStarted, + total: booksToPush.length, + action: 'uploading', + }); + pushStarted += 1; + let phase: SyncFailureEntry['phase'] = 'upload-config'; + try { + const config = await this.store.loadConfig(book); + if (config) { + // Mirror the reader hook's pull-merge-push discipline so a manual + // "Sync now" can't blind-overwrite state this device hasn't pulled + // yet. Only in two-way ('silent') mode — 'send' keeps the blind + // push. A failed pull-merge falls back to the local config. + let configToPush = config; + if (canPull) { + try { + const pull = await this.pullBookConfig(book, config); + if (pull.applied && pull.mergedConfig) { + configToPush = pull.mergedConfig; + // Persist the merged superset locally so this device + // converges too, not just the remote. + await this.store.saveBookConfig(book, pull.mergedConfig); + } + } catch (e) { + console.warn('file sync: config pull-merge failed', book.hash, e); + } + } + await this.pushBookConfig(book, configToPush, options.deviceId); + result.configsUploaded += 1; + syncedHashes.add(book.hash); + } + // Covers ride along with the config-level sync, NOT with syncBooks: + // the receiving device can't regenerate them without the book bytes. + // Failures here are warnings, not hard failures. + try { + const coverResult = await this.pushBookCover(book); + if (coverResult.uploaded) { + result.coversUploaded += 1; + syncedHashes.add(book.hash); + } + } catch (e) { + console.warn('file sync: cover failed', book.hash, e); + } + if (options.syncBooks) { + phase = 'upload-file'; + const fileResult = await this.pushBookFile(book); + if (fileResult.uploaded) { + result.filesUploaded += 1; + syncedHashes.add(book.hash); + } else if (fileResult.reason === 'remote-matches') { + result.filesAlreadyInSync += 1; + } + } + } catch (e) { + result.failures += 1; + result.failedBooks.push({ + hash: book.hash, + title: book.title || book.hash, + phase, + reason: formatFailureReason(e), + }); + console.warn('file sync: book failed', book.hash, e); + } + }); + } + + // Push the merged index whenever we're allowed to write, even if no + // binaries moved this turn (keeps library.json authoritative). Soft-deleted + // books' per-hash dirs are intentionally NOT GC'd here — that's the manual + // cleanup sweep's job (see deleteRemoteBookDir). + if (canPush) { + try { + const newIndex: RemoteLibraryIndex = { + schemaVersion: 1, + books: Array.from(allBooksMap.values()), + updatedAt: Date.now(), + }; + await this.pushLibraryIndex(newIndex); + } catch (e) { + console.warn('file sync: failed to push index', e); + } + } + + result.booksSynced = syncedHashes.size; + return result; + } +} diff --git a/apps/readest-app/src/services/sync/file/index.ts b/apps/readest-app/src/services/sync/file/index.ts new file mode 100644 index 00000000..23f7dde0 --- /dev/null +++ b/apps/readest-app/src/services/sync/file/index.ts @@ -0,0 +1,12 @@ +/** + * Provider-agnostic file-sync core. Consumers import the engine, the provider + * interface, and the local-store bridge from here; a concrete backend lives + * under `src/services/sync/providers/<name>/`. + */ +export * from './provider'; +export * from './localStore'; +export * from './appLocalStore'; +export * from './layout'; +export * from './wire'; +export * from './merge'; +export * from './engine'; diff --git a/apps/readest-app/src/services/webdav/WebDAVPaths.ts b/apps/readest-app/src/services/sync/file/layout.ts similarity index 75% rename from apps/readest-app/src/services/webdav/WebDAVPaths.ts rename to apps/readest-app/src/services/sync/file/layout.ts index ae66a497..3238cbe7 100644 --- a/apps/readest-app/src/services/webdav/WebDAVPaths.ts +++ b/apps/readest-app/src/services/sync/file/layout.ts @@ -3,9 +3,10 @@ import { EXTS } from '@/libs/document'; import { makeSafeFilename } from '@/utils/misc'; /** - * Layout convention for the WebDAV "Readest" subtree under the user's - * configured rootPath. The whole sync feature is scoped to this subtree so - * we never touch unrelated files in the user's WebDAV. + * Layout convention for the "Readest" subtree under the user's configured + * rootPath, shared by every file-based sync provider (WebDAV today; Google + * Drive / Dropbox / FTP / SFTP in future). The whole sync feature is scoped + * to this subtree so we never touch unrelated files in the user's storage. * * Tree: * <rootPath>/ @@ -19,14 +20,18 @@ import { makeSafeFilename } from '@/utils/misc'; * * Why hash directories: avoids title collisions and makes title edits a * pure metadata operation (no remote rename). The friendly file name - * inside the directory keeps the WebDAV client experience readable. + * inside the directory keeps the remote browse experience readable. + * + * These builders are pure functions of `rootPath` — no transport knowledge. + * The directory/file names below are a FROZEN wire layout: changing them + * would orphan every existing remote tree, so they must stay byte-stable. */ -export const WEBDAV_BASE_DIR = 'Readest'; -export const WEBDAV_BOOKS_DIR = 'books'; -export const WEBDAV_LIBRARY_FILE = 'library.json'; -export const WEBDAV_BOOK_CONFIG_FILE = 'config.json'; -export const WEBDAV_BOOK_COVER_FILE = 'cover.png'; +export const SYNC_BASE_DIR = 'Readest'; +export const SYNC_BOOKS_DIR = 'books'; +export const SYNC_LIBRARY_FILE = 'library.json'; +export const SYNC_BOOK_CONFIG_FILE = 'config.json'; +export const SYNC_BOOK_COVER_FILE = 'cover.png'; /** * Normalise the user-entered rootPath so the rest of the code can rely on @@ -48,19 +53,19 @@ const join = (...parts: string[]): string => { /** Absolute path of the Readest base directory (where library.json lives). */ export const buildBasePath = (rootPath: string): string => - join(normalizeRoot(rootPath), WEBDAV_BASE_DIR); + join(normalizeRoot(rootPath), SYNC_BASE_DIR); /** Absolute path of the per-book directory keyed by hash. */ export const buildBookDirPath = (rootPath: string, bookHash: string): string => - join(buildBasePath(rootPath), WEBDAV_BOOKS_DIR, bookHash); + join(buildBasePath(rootPath), SYNC_BOOKS_DIR, bookHash); /** Absolute path of a book's config.json (progress + booknotes). */ export const buildBookConfigPath = (rootPath: string, bookHash: string): string => - join(buildBookDirPath(rootPath, bookHash), WEBDAV_BOOK_CONFIG_FILE); + join(buildBookDirPath(rootPath, bookHash), SYNC_BOOK_CONFIG_FILE); /** Absolute path of the shared library.json index. */ export const buildLibraryPath = (rootPath: string): string => - join(buildBasePath(rootPath), WEBDAV_LIBRARY_FILE); + join(buildBasePath(rootPath), SYNC_LIBRARY_FILE); /** * Friendly book file name "<sanitized title>.<ext>" used inside the @@ -83,7 +88,7 @@ export const buildBookFilePath = (rootPath: string, book: Book): string => /** Absolute path of the book cover image. */ export const buildBookCoverPath = (rootPath: string, bookHash: string): string => - join(buildBookDirPath(rootPath, bookHash), WEBDAV_BOOK_COVER_FILE); + join(buildBookDirPath(rootPath, bookHash), SYNC_BOOK_COVER_FILE); /** * Walk the parents of an absolute path, top-down, so callers can diff --git a/apps/readest-app/src/services/sync/file/localStore.ts b/apps/readest-app/src/services/sync/file/localStore.ts new file mode 100644 index 00000000..80f9165c --- /dev/null +++ b/apps/readest-app/src/services/sync/file/localStore.ts @@ -0,0 +1,53 @@ +import { Book, BookConfig } from '@/types/book'; + +/** A buffered file payload + its byte length (for the HEAD-vs-local probe). */ +export interface BookBytes { + bytes: ArrayBuffer; + size: number; +} + +/** + * App-side local I/O the {@link FileSyncEngine} needs, abstracted away from + * `appService` / Zustand stores so the engine stays testable with a plain + * fake. One implementation — {@link createAppLocalStore} — is built once per + * consumer (the reader hook and the library "Sync now" form) and shared, so + * the buffered + streaming book/cover loaders live in exactly one place + * instead of being copy-pasted across consumers. + * + * Split of responsibilities with {@link FileSyncProvider}: + * - provider owns the REMOTE side (and the actual stream transport); + * - localStore owns the LOCAL side (reading/writing files on this device, + * resolving on-disk paths for streaming, and mutating the local library). + */ +export interface LocalStore { + /** Load a book's local config (progress + booknotes), or null if none. */ + loadConfig(book: Book): Promise<BookConfig | null>; + /** Persist a (merged) config to local disk. */ + saveBookConfig(book: Book, config: BookConfig): Promise<void>; + + /** Buffered upload source: the book file bytes, or null when not on disk. */ + loadBookFile(book: Book): Promise<BookBytes | null>; + /** + * Streaming upload source: the absolute on-disk path + size of the book + * file, or null when not on this device. Size lets the engine run the + * HEAD-vs-local short-circuit without reading any bytes. + */ + resolveLocalBookPath(book: Book): Promise<{ path: string; size: number } | null>; + /** Buffered download sink: write freshly-pulled book bytes to disk. */ + saveBookFile(book: Book, bytes: ArrayBuffer): Promise<void>; + /** + * Streaming download sink: ensure the per-book directory exists and return + * the absolute destination path the provider should stream into. + */ + prepareLocalBookPath(book: Book): Promise<string>; + + /** Buffered cover source, or null when the book has no local cover. */ + loadBookCover(book: Book): Promise<BookBytes | null>; + /** Write a freshly-pulled cover image to disk. */ + saveBookCover(book: Book, bytes: ArrayBuffer): Promise<void>; + + /** Insert a brand-new book row (no-op on an existing hash). */ + addBookToLibrary(book: Book): Promise<void>; + /** Persist refreshed metadata for a book already in the local library. */ + updateBookMetadata(book: Book): Promise<void>; +} diff --git a/apps/readest-app/src/services/sync/file/merge.ts b/apps/readest-app/src/services/sync/file/merge.ts new file mode 100644 index 00000000..7a88cfe2 --- /dev/null +++ b/apps/readest-app/src/services/sync/file/merge.ts @@ -0,0 +1,115 @@ +import { Book, BookConfig, BookNote } from '@/types/book'; +import { RemoteBookConfig } from './wire'; + +/** + * Declarative merge policies for the file-sync engine. Each function is + * pure (no I/O) and independently unit-tested with algebraic laws so the + * convergence guarantees are explicit rather than implied by the + * orchestration code: + * - notes → element-set CRDT (union by id, per-note updatedAt, + * deletedAt tombstones). + * - config → last-writer-wins on `config.updatedAt` for scalars, + * notes merged via the CRDT regardless of scalar winner. + * - book meta → last-writer-wins on `book.updatedAt` over a fixed field + * subset; device-local / on-disk fields always preserved. + * + * State-based CRDT semantics make this safe over a lossy single-file + * transport: every replica holds full state and re-merges, so a blind PUT + * of a merged superset converges even when intermediate writes are lost. + */ + +/** + * Per-note merge: pick the locally-stored copy or the remote copy of each + * note based on `updatedAt` / `deletedAt`. Mirrors `processNewNote` in + * `useNotesSync.ts` so users get the same semantics regardless of which + * sync backend produced the row. + * + * A note is keyed by `id`. When the same id exists on both sides we keep + * whichever side has the larger updatedAt; ties go to the side whose + * `deletedAt` is more recent (which usually means the deletion came after + * the creation/edit). + */ +export const mergeNotes = (local: BookNote[], remote: BookNote[]): BookNote[] => { + const byId = new Map<string, BookNote>(); + for (const n of local) byId.set(n.id, n); + for (const r of remote) { + const l = byId.get(r.id); + if (!l) { + byId.set(r.id, r); + continue; + } + const lUpdated = l.updatedAt ?? 0; + const rUpdated = r.updatedAt ?? 0; + const lDeleted = l.deletedAt ?? 0; + const rDeleted = r.deletedAt ?? 0; + if (rUpdated > lUpdated || rDeleted > lDeleted) { + byId.set(r.id, { ...l, ...r }); + } else { + byId.set(r.id, { ...r, ...l }); + } + } + return Array.from(byId.values()); +}; + +/** + * Merge a remote config envelope into the local BookConfig. + * + * Scalars use a per-config `updatedAt` LWW (same as the native cloud sync + * in `useProgressSync.applyRemoteProgress`); booknotes always merge via the + * element-set CRDT regardless of which side won the scalar race. Null / + * undefined remote fields are dropped before the spread so a server can + * never inject keys the wire envelope isn't supposed to carry (viewSettings, + * searchConfig, RSVP) — those never appear in `remote.config` because + * `buildRemotePayload` strips them on push. + * + * Returns both the merged config (with `booknotes` populated) and the merged + * notes separately so callers can drive a live view off the note set. + */ +export const mergeBookConfig = ( + local: BookConfig, + remote: RemoteBookConfig, +): { config: BookConfig; notes: BookNote[] } => { + const remoteConfigUpdated = remote.config.updatedAt ?? remote.updatedAt; + const localConfigUpdated = local.updatedAt ?? 0; + const filteredRemote = Object.fromEntries( + Object.entries(remote.config).filter(([, v]) => v !== null && v !== undefined), + ) as Partial<BookConfig>; + const merged: BookConfig = + remoteConfigUpdated >= localConfigUpdated + ? ({ ...local, ...filteredRemote } as BookConfig) + : ({ ...filteredRemote, ...local } as BookConfig); + const notes = mergeNotes(local.booknotes ?? [], remote.booknotes ?? []); + merged.booknotes = notes; + return { config: merged, notes }; +}; + +/** + * Overlay the user-facing metadata of `remote` onto `local`, preserving every + * device-local / file-system field: `filePath`, `sourceTitle` (which names the + * on-disk file), `coverImageUrl` (a device-local blob URL the caller + * regenerates), reading progress, reading status, group membership, `hash`, + * `format`, `createdAt`, etc. + * + * Only the fields a metadata edit actually changes travel — this list mirrors + * `getBookWithUpdatedMetadata` in `utils/book.ts`, which is the local side of + * the same operation. The cover image is replicated separately as cover.png + * bytes (see the reconciliation pass in the engine), so it is intentionally + * absent here. + */ +export const mergeBookMetadata = (local: Book, remote: Book): Book => ({ + ...local, + title: remote.title, + author: remote.author, + metadata: remote.metadata ?? local.metadata, + primaryLanguage: remote.primaryLanguage ?? local.primaryLanguage, + updatedAt: remote.updatedAt, +}); + +/** + * LWW predicate for the library-index metadata reconciliation: true when the + * remote indexed copy is strictly newer than the local one and neither side + * is tombstoned. A strict `>` keeps the pass a no-op when timestamps match so + * we never re-apply identical metadata or bounce updates between devices. + */ +export const isRemoteBookMetadataNewer = (local: Book, remote: Book): boolean => + !remote.deletedAt && !local.deletedAt && (remote.updatedAt ?? 0) > (local.updatedAt ?? 0); diff --git a/apps/readest-app/src/services/sync/file/provider.ts b/apps/readest-app/src/services/sync/file/provider.ts new file mode 100644 index 00000000..b1d72247 --- /dev/null +++ b/apps/readest-app/src/services/sync/file/provider.ts @@ -0,0 +1,87 @@ +/** + * Transport abstraction for a file-based sync backend. + * + * A `FileSyncProvider` exposes the minimal set of remote file operations the + * {@link FileSyncEngine} needs; everything above this line (layout, wire + * envelopes, merge policy, orchestration) is provider-agnostic. Implementing + * a new backend (Google Drive / Dropbox / FTP / SFTP) means writing one of + * these — ideally validated against the shared provider-conformance test + * suite — and nothing else. + * + * Error contract: read/head operations return `null` for a missing path + * (HTTP 404 and equivalents); every other failure throws a + * {@link FileSyncError} carrying a normalised `code` so the engine can branch + * on auth / not-found / network / conflict without knowing the backend. + */ + +export type FileSyncErrorCode = 'AUTH_FAILED' | 'NOT_FOUND' | 'NETWORK' | 'CONFLICT' | 'UNKNOWN'; + +export class FileSyncError extends Error { + code: FileSyncErrorCode; + /** HTTP status when the request reached the server, if applicable. */ + status?: number; + + constructor(message: string, code: FileSyncErrorCode = 'UNKNOWN', status?: number) { + super(message); + this.name = 'FileSyncError'; + this.code = code; + this.status = status; + } +} + +/** A single directory entry returned by {@link FileSyncProvider.list}. */ +export interface FileEntry { + /** File or directory name (single decoded path segment). */ + name: string; + /** Absolute path on the backend, leading slash, decoded. */ + path: string; + isDirectory: boolean; + /** Content length in bytes when the backend reports it (files only). */ + size?: number; + /** Backend-provided modification timestamp, if any. */ + lastModified?: string; +} + +/** Metadata returned by a HEAD-style probe. */ +export interface FileHead { + size?: number; + etag?: string; +} + +export interface FileSyncProvider { + /** Normalised root path: leading '/', no trailing slash (root === '/'). */ + readonly rootPath: string; + + /** GET text. Resolves `null` when the path doesn't exist (404). */ + readText(path: string): Promise<string | null>; + /** GET binary. Resolves `null` when the path doesn't exist (404). */ + readBinary(path: string): Promise<ArrayBuffer | null>; + /** HEAD probe. Resolves `null` when the path doesn't exist (404). */ + head(path: string): Promise<FileHead | null>; + /** List immediate children of a directory. Throws on non-2xx. */ + list(path: string): Promise<FileEntry[]>; + + /** PUT text. Parent directories must already exist (see {@link ensureDir}). */ + writeText(path: string, body: string, contentType?: string): Promise<void>; + /** PUT binary. Parent directories must already exist. */ + writeBinary(path: string, body: ArrayBuffer, contentType?: string): Promise<void>; + + /** Create each directory in `paths` (top-down). Idempotent. */ + ensureDir(paths: string[]): Promise<void>; + /** Recursively delete a directory subtree. A missing dir is success. */ + deleteDir(path: string): Promise<void>; + + /** + * Optional streaming upload: PUT `localPath`'s bytes to `remotePath` + * without materialising them in the JS heap. The provider owns the + * remote URL + auth. Resolves `true` on success, `false` on a swallowed + * failure. Absent on backends without a streaming primitive (e.g. web); + * the engine then falls back to buffered {@link writeBinary}. + */ + uploadStream?(remotePath: string, localPath: string): Promise<boolean>; + /** + * Optional streaming download: GET `remotePath` straight to `localPath`. + * Same ownership + fallback rules as {@link uploadStream}. + */ + downloadStream?(remotePath: string, localPath: string): Promise<boolean>; +} diff --git a/apps/readest-app/src/services/sync/file/wire.ts b/apps/readest-app/src/services/sync/file/wire.ts new file mode 100644 index 00000000..c44d3f7e --- /dev/null +++ b/apps/readest-app/src/services/sync/file/wire.ts @@ -0,0 +1,110 @@ +import { Book, BookConfig, BookNote } from '@/types/book'; + +/** + * Per-book remote payload stored at + * <rootPath>/Readest/books/<hash>/config.json + * + * The wire format is a thin envelope around the existing local + * `BookConfig` so the merge logic can stay identical to readest's other + * sync providers (per-field updatedAt LWW for top-level keys, per-note + * updatedAt + deletedAt for booknotes). The envelope adds the bare + * minimum "who/when last wrote this" metadata so clients can detect + * cross-device clobbers and surface them in diagnostics. + * + * FROZEN: `schemaVersion`, `writerVersion`, and the trimmed-config field + * set are part of the on-wire contract. A refactored client and an old + * client must interoperate byte-for-byte, so do not rename or re-shape + * these fields. `writerVersion` keeps its historical `'readest-webdav-1'` + * value even though the engine is now provider-agnostic — it is an opaque + * tag, never branched on, and changing it buys nothing. + */ +export interface RemoteBookConfig { + schemaVersion: 1; + bookHash: string; + metaHash?: string; + /** Trimmed BookConfig — only the keys we care about syncing. */ + config: Partial<BookConfig>; + /** Booknotes carry their own per-note updatedAt/deletedAt for merging. */ + booknotes: BookNote[]; + writerDeviceId: string; + writerVersion: 'readest-webdav-1'; + /** When the writer last touched the row (client wall clock, millis). */ + updatedAt: number; +} + +/** + * Convert the live local BookConfig into the wire envelope. We deliberately + * drop transient view state (search config, RSVP position, viewSettings, + * etc.) — those are device-local UI preferences, not progress. + * + * Why viewSettings stays local even though it lives in BookConfig: + * - Different devices have different screen sizes / DPI / typography + * preferences. Pushing a phone's 14pt setting onto a desktop would + * surprise users in a bad way. + * - readest's own cloud sync similarly carves out viewSettings from + * cross-device replication. Duplicating that policy here keeps the + * backends behaviourally aligned. + * - The trim list below is the SOURCE OF TRUTH for what travels — + * when adding a new BookConfig field, decide here whether it's a + * "reading state" (include) or a "device preference" (skip). + * + * Anything not in `trimmed` therefore never reaches the server, and + * conversely the config merge (see `mergeBookConfig` in merge.ts) only + * ever merges fields the server actually carries — so a malicious or + * buggy server can't somehow inject viewSettings into a local config. + */ +export const buildRemotePayload = ( + book: Book, + config: BookConfig, + deviceId: string, +): RemoteBookConfig => { + const trimmed: Partial<BookConfig> = { + progress: config.progress, + location: config.location, + xpointer: config.xpointer, + updatedAt: config.updatedAt, + }; + return { + schemaVersion: 1, + bookHash: book.hash, + metaHash: book.metaHash, + config: trimmed, + booknotes: config.booknotes ?? [], + writerDeviceId: deviceId, + writerVersion: 'readest-webdav-1', + updatedAt: Date.now(), + }; +}; + +export const parseRemotePayload = (raw: string | null): RemoteBookConfig | null => { + if (!raw) return null; + try { + const parsed = JSON.parse(raw) as RemoteBookConfig; + if (!parsed || parsed.schemaVersion !== 1) return null; + return parsed; + } catch { + return null; + } +}; + +/** + * The shared `<rootPath>/Readest/library.json` index. Membership is a + * union-by-hash CRDT (with `deletedAt` tombstones); per-book metadata is + * last-writer-wins on `book.updatedAt`. See merge.ts for the policies. + */ +export interface RemoteLibraryIndex { + schemaVersion: 1; + books: Book[]; + updatedAt: number; +} + +export const parseRemoteLibraryIndex = (raw: string | null): RemoteLibraryIndex | null => { + if (!raw) return null; + try { + const parsed = JSON.parse(raw) as RemoteLibraryIndex; + if (parsed && parsed.schemaVersion === 1) return parsed; + } catch { + // Ignore parse errors — a malformed index is treated as "no index". + } + return null; +}; diff --git a/apps/readest-app/src/services/sync/providers/webdav/WebDAVProvider.ts b/apps/readest-app/src/services/sync/providers/webdav/WebDAVProvider.ts new file mode 100644 index 00000000..4405c6a2 --- /dev/null +++ b/apps/readest-app/src/services/sync/providers/webdav/WebDAVProvider.ts @@ -0,0 +1,123 @@ +import { WebDAVSettings } from '@/types/settings'; +import { isTauriAppPlatform } from '@/services/environment'; +import { tauriDownload, tauriUpload } from '@/utils/transfer'; +import { + FileEntry, + FileHead, + FileSyncError, + FileSyncErrorCode, + FileSyncProvider, +} from '@/services/sync/file/provider'; +import { + WebDAVConfig, + WebDAVRequestError, + buildBasicAuthHeader, + buildRequestUrl, + deleteDirectory, + ensureDirectory, + getFile, + getFileBinary, + headFile, + listDirectory, + normalizeRootPath, + putFile, + putFileBinary, +} from './client'; + +/** + * WebDAV implementation of {@link FileSyncProvider} — the first concrete + * backend for the provider-agnostic file-sync engine. + * + * Responsibilities unique to this layer: + * - own the WebDAV transport config (serverUrl + Basic-auth credentials); + * - translate the transport client's {@link WebDAVRequestError} into the + * engine's neutral {@link FileSyncError} so nothing above this file + * knows the backend is WebDAV; + * - own streaming upload/download (URL + auth + the Tauri-side + * `tauriUpload`/`tauriDownload`), keeping gigabyte-scale book payloads out + * of the JS heap. Streaming is exposed only on Tauri; on web the engine + * falls back to buffered {@link FileSyncProvider.writeBinary}/`readBinary`. + */ + +const mapError = (e: unknown): FileSyncError => { + if (e instanceof FileSyncError) return e; + if (e instanceof WebDAVRequestError) { + const code: FileSyncErrorCode = + e.code === 'AUTH_FAILED' + ? 'AUTH_FAILED' + : e.code === 'NOT_FOUND' + ? 'NOT_FOUND' + : e.code === 'NETWORK' + ? 'NETWORK' + : e.status === 409 + ? 'CONFLICT' + : 'UNKNOWN'; + return new FileSyncError(e.message, code, e.status); + } + return new FileSyncError(e instanceof Error ? e.message : String(e), 'UNKNOWN'); +}; + +const wrap = async <T>(fn: () => Promise<T>): Promise<T> => { + try { + return await fn(); + } catch (e) { + throw mapError(e); + } +}; + +export const createWebDAVProvider = (settings: WebDAVSettings): FileSyncProvider => { + const config: WebDAVConfig = { + serverUrl: settings.serverUrl, + username: settings.username, + password: settings.password, + }; + + const provider: FileSyncProvider = { + rootPath: normalizeRootPath(settings.rootPath), + readText: (path) => wrap(() => getFile(config, path)), + readBinary: (path) => wrap(() => getFileBinary(config, path)), + head: (path): Promise<FileHead | null> => wrap(() => headFile(config, path)), + list: (path): Promise<FileEntry[]> => wrap(() => listDirectory(config, path)), + writeText: (path, body, contentType) => wrap(() => putFile(config, path, body, contentType)), + writeBinary: (path, body, contentType) => + wrap(() => putFileBinary(config, path, body, contentType)), + ensureDir: (paths) => wrap(() => ensureDirectory(config, paths)), + deleteDir: (path) => wrap(() => deleteDirectory(config, path)), + }; + + if (isTauriAppPlatform()) { + const authHeaders = (): Record<string, string> => ({ + Authorization: buildBasicAuthHeader(settings.username, settings.password), + }); + provider.uploadStream = async (remotePath, localPath) => { + const url = buildRequestUrl(settings.serverUrl, remotePath); + try { + // tauriUpload's TS type says Map, but the Rust command accepts a JSON + // object → HashMap<String, String>; pass the headers object directly. + await tauriUpload( + url, + localPath, + 'PUT', + undefined, + authHeaders() as unknown as Map<string, string>, + ); + return true; + } catch (e) { + console.warn('WebDAVProvider.uploadStream failed', remotePath, e); + return false; + } + }; + provider.downloadStream = async (remotePath, localPath) => { + const url = buildRequestUrl(settings.serverUrl, remotePath); + try { + await tauriDownload(url, localPath, undefined, authHeaders()); + return true; + } catch (e) { + console.warn('WebDAVProvider.downloadStream failed', remotePath, e); + return false; + } + }; + } + + return provider; +}; diff --git a/apps/readest-app/src/services/webdav/WebDAVClient.ts b/apps/readest-app/src/services/sync/providers/webdav/client.ts similarity index 95% rename from apps/readest-app/src/services/webdav/WebDAVClient.ts rename to apps/readest-app/src/services/sync/providers/webdav/client.ts index 69797948..c00cd248 100644 --- a/apps/readest-app/src/services/webdav/WebDAVClient.ts +++ b/apps/readest-app/src/services/sync/providers/webdav/client.ts @@ -1,5 +1,5 @@ import { fetch as tauriFetch } from '@tauri-apps/plugin-http'; -import { isTauriAppPlatform } from '../environment'; +import { isTauriAppPlatform } from '@/services/environment'; /** * Minimal WebDAV client used by the Integrations panel. @@ -299,17 +299,31 @@ export const listDirectory = async ( const root = normalizeRootPath(rootPath); const url = buildUrl(config.serverUrl, root); const fetchFn = getFetch(); - const response = await fetchFn(url, { - method: 'PROPFIND', - headers: { - Authorization: buildAuthHeader(config.username, config.password), - Depth: '1', - 'Content-Type': 'application/xml; charset=utf-8', - }, - body: PROPFIND_BODY, - }); + // Throw the same WebDAVRequestError taxonomy as the file-level helpers so the + // provider layer can map list() failures to FileSyncError codes (auth / + // not-found / network) instead of flattening every failure to UNKNOWN. + let response: Response; + try { + response = await fetchFn(url, { + method: 'PROPFIND', + headers: { + Authorization: buildAuthHeader(config.username, config.password), + Depth: '1', + 'Content-Type': 'application/xml; charset=utf-8', + }, + body: PROPFIND_BODY, + }); + } catch (e) { + throw new WebDAVRequestError((e as Error).message || 'Network error', undefined, 'NETWORK'); + } + if (response.status === 401 || response.status === 403) { + throw new WebDAVRequestError('Authentication failed', response.status, 'AUTH_FAILED'); + } + if (response.status === 404) { + throw new WebDAVRequestError('Directory not found', response.status, 'NOT_FOUND'); + } if (response.status !== 207 && response.status !== 200) { - throw new Error(`PROPFIND failed with status ${response.status}`); + throw new WebDAVRequestError(`PROPFIND failed with status ${response.status}`, response.status); } const xml = await response.text(); let serverOrigin = ''; diff --git a/apps/readest-app/src/services/webdav/webdavConnectSettings.ts b/apps/readest-app/src/services/sync/providers/webdav/connectSettings.ts similarity index 100% rename from apps/readest-app/src/services/webdav/webdavConnectSettings.ts rename to apps/readest-app/src/services/sync/providers/webdav/connectSettings.ts diff --git a/apps/readest-app/src/services/webdav/WebDAVSync.ts b/apps/readest-app/src/services/webdav/WebDAVSync.ts deleted file mode 100644 index d64c9f4d..00000000 --- a/apps/readest-app/src/services/webdav/WebDAVSync.ts +++ /dev/null @@ -1,1114 +0,0 @@ -import { Book, BookConfig, BookNote } from '@/types/book'; -import { WebDAVSettings } from '@/types/settings'; -import { - WebDAVConfig, - buildBasicAuthHeader, - buildRequestUrl, - deleteDirectory, - ensureDirectory, - getFile, - getFileBinary, - headFile, - listDirectory, - putFile, - putFileBinary, - WebDAVRequestError, -} from './WebDAVClient'; -import { - ancestorsOf, - buildBasePath, - buildBookConfigPath, - buildBookCoverPath, - buildBookDirPath, - buildBookFilePath, - buildLibraryPath, - WEBDAV_BOOKS_DIR, -} from './WebDAVPaths'; - -/** - * Per-book remote payload stored at - * <rootPath>/Readest/books/<hash>/config.json - * - * The wire format is a thin envelope around the existing local - * `BookConfig` so the merge logic can stay identical to readest's other - * sync providers (per-field updatedAt LWW for top-level keys, per-note - * updatedAt + deletedAt for booknotes). The envelope adds the bare - * minimum "who/when last wrote this" metadata so clients can detect - * cross-device clobbers and surface them in diagnostics. - */ -export interface RemoteBookConfig { - schemaVersion: 1; - bookHash: string; - metaHash?: string; - /** Trimmed BookConfig — only the keys we care about syncing. */ - config: Partial<BookConfig>; - /** Booknotes carry their own per-note updatedAt/deletedAt for merging. */ - booknotes: BookNote[]; - writerDeviceId: string; - writerVersion: 'readest-webdav-1'; - /** When the writer last touched the row (client wall clock, millis). */ - updatedAt: number; -} - -/** - * Convert the live local BookConfig into the wire envelope. We deliberately - * drop transient view state (search config, RSVP position, viewSettings, - * etc.) — those are device-local UI preferences, not progress. - * - * Why viewSettings stays local even though it lives in BookConfig: - * - Different devices have different screen sizes / DPI / typography - * preferences. Pushing a phone's 14pt setting onto a desktop would - * surprise users in a bad way. - * - readest's own cloud sync similarly carves out viewSettings from - * cross-device replication. Duplicating that policy here keeps the - * two backends behaviourally aligned. - * - The trim list below is the SOURCE OF TRUTH for what travels — - * when adding a new BookConfig field, decide here whether it's a - * "reading state" (include) or a "device preference" (skip). - * - * Anything not in `trimmed` therefore never reaches the server, and - * conversely `pullBookConfig` only ever merges fields the server - * actually carries (see filteredRemote spread there) — so a malicious - * or buggy server can't somehow inject viewSettings into a local config. - */ -const buildRemotePayload = (book: Book, config: BookConfig, deviceId: string): RemoteBookConfig => { - const trimmed: Partial<BookConfig> = { - progress: config.progress, - location: config.location, - xpointer: config.xpointer, - updatedAt: config.updatedAt, - }; - return { - schemaVersion: 1, - bookHash: book.hash, - metaHash: book.metaHash, - config: trimmed, - booknotes: config.booknotes ?? [], - writerDeviceId: deviceId, - writerVersion: 'readest-webdav-1', - updatedAt: Date.now(), - }; -}; - -const parseRemotePayload = (raw: string | null): RemoteBookConfig | null => { - if (!raw) return null; - try { - const parsed = JSON.parse(raw) as RemoteBookConfig; - if (!parsed || parsed.schemaVersion !== 1) return null; - return parsed; - } catch { - return null; - } -}; - -const toClientConfig = (settings: WebDAVSettings): WebDAVConfig => ({ - serverUrl: settings.serverUrl, - username: settings.username, - password: settings.password, -}); - -/** - * Per-note merge: pick the locally-stored copy or the remote copy of each - * note based on `updatedAt` / `deletedAt`. Mirrors `processNewNote` in - * `useNotesSync.ts` so users get the same semantics regardless of which - * sync backend produced the row. - * - * Implementation detail: a note is keyed by `id`. When the same id exists - * on both sides we keep whichever side has the larger updatedAt; ties go - * to the side whose `deletedAt` is more recent (which usually means the - * deletion came after the creation/edit). - */ -const mergeNotes = (local: BookNote[], remote: BookNote[]): BookNote[] => { - const byId = new Map<string, BookNote>(); - for (const n of local) byId.set(n.id, n); - for (const r of remote) { - const l = byId.get(r.id); - if (!l) { - byId.set(r.id, r); - continue; - } - const lUpdated = l.updatedAt ?? 0; - const rUpdated = r.updatedAt ?? 0; - const lDeleted = l.deletedAt ?? 0; - const rDeleted = r.deletedAt ?? 0; - if (rUpdated > lUpdated || rDeleted > lDeleted) { - byId.set(r.id, { ...l, ...r }); - } else { - byId.set(r.id, { ...r, ...l }); - } - } - return Array.from(byId.values()); -}; - -/** - * Overlay the user-facing metadata of `remote` onto `local`, preserving every - * device-local / file-system field: `filePath`, `sourceTitle` (which names the - * on-disk file), `coverImageUrl` (a device-local blob URL the caller - * regenerates), reading progress, reading status, group membership, `hash`, - * `format`, `createdAt`, etc. - * - * Only the fields a metadata edit actually changes travel — this list mirrors - * `getBookWithUpdatedMetadata` in `utils/book.ts`, which is the local side of - * the same operation. The cover image is replicated separately as cover.png - * bytes (see the reconciliation pass in `syncLibrary`), so it is intentionally - * absent here. - */ -const mergeRemoteBookMetadata = (local: Book, remote: Book): Book => ({ - ...local, - title: remote.title, - author: remote.author, - metadata: remote.metadata ?? local.metadata, - primaryLanguage: remote.primaryLanguage ?? local.primaryLanguage, - updatedAt: remote.updatedAt, -}); - -export interface PullResult { - /** True when the remote had a config and we merged something into local. */ - applied: boolean; - /** The merged config to be written back into the local store. */ - mergedConfig?: BookConfig; - /** When non-empty, these are the notes after merge — use them to update the live view. */ - mergedNotes?: BookNote[]; - /** The remote's writerDeviceId, useful for diagnostics. */ - remoteDeviceId?: string; -} - -/** - * Pull `<rootPath>/Readest/books/<hash>/config.json`, merge into the - * provided local config, and return the merged result. The caller is - * responsible for writing the merged config back via the bookData store - * (so this module stays free of React/store dependencies). - * - * Returns `applied: false` when the remote file doesn't exist yet or its - * envelope is malformed. Auth failures and other server errors propagate - * as `WebDAVRequestError` so the caller can show a toast. - */ -export const pullBookConfig = async ( - settings: WebDAVSettings, - book: Book, - localConfig: BookConfig, -): Promise<PullResult> => { - const path = buildBookConfigPath(settings.rootPath, book.hash); - const raw = await getFile(toClientConfig(settings), path); - const remote = parseRemotePayload(raw); - if (!remote) { - return { applied: false }; - } - // Top-level field merge: same per-config updatedAt LWW that the native - // cloud sync uses in useProgressSync.applyRemoteProgress. - const remoteConfigUpdated = remote.config.updatedAt ?? remote.updatedAt; - const localConfigUpdated = localConfig.updatedAt ?? 0; - // Drop null/undefined fields the server might have left in (e.g. an - // older client that didn't write `xpointer`). Crucially, this also - // means the spread below can NEVER introduce server-driven values for - // keys the server isn't supposed to care about (viewSettings, - // searchConfig, RSVP) — those keys never appear in `remote.config` to - // begin with because `buildRemotePayload` strips them on push. The - // invariant "wire envelope only carries reading state" therefore - // protects pull as well as push. - const filteredRemote = Object.fromEntries( - Object.entries(remote.config).filter(([, v]) => v !== null && v !== undefined), - ) as Partial<BookConfig>; - const mergedConfig: BookConfig = - remoteConfigUpdated >= localConfigUpdated - ? ({ ...localConfig, ...filteredRemote } as BookConfig) - : ({ ...filteredRemote, ...localConfig } as BookConfig); - const mergedNotes = mergeNotes(localConfig.booknotes ?? [], remote.booknotes ?? []); - mergedConfig.booknotes = mergedNotes; - return { - applied: true, - mergedConfig, - mergedNotes, - remoteDeviceId: remote.writerDeviceId, - }; -}; - -/** - * Push the local BookConfig to the remote. Creates parent directories as - * needed (idempotent — MKCOL 405 is treated as success). The caller - * provides a stable deviceId so we can record which device last wrote - * the file. - * - * If the remote already has a strictly newer payload, we still overwrite — - * deciding whether to push is the caller's responsibility (it has the - * strategy / preview-mode context). This function is the dumb mechanism. - */ -export const pushBookConfig = async ( - settings: WebDAVSettings, - book: Book, - config: BookConfig, - deviceId: string, -): Promise<void> => { - const client = toClientConfig(settings); - const dirPath = buildBookDirPath(settings.rootPath, book.hash); - const path = buildBookConfigPath(settings.rootPath, book.hash); - // Ensure every ancestor up to and including the per-book directory. - const dirs = [...ancestorsOf(`${dirPath}/.placeholder`), dirPath]; - await ensureDirectory(client, dirs); - const payload = buildRemotePayload(book, config, deviceId); - try { - await putFile(client, path, JSON.stringify(payload)); - } catch (e) { - if (e instanceof WebDAVRequestError && e.status === 409) { - // 409 from PUT means a parent disappeared between our MKCOL and the - // PUT. Re-create the chain and retry once. - await ensureDirectory(client, dirs); - await putFile(client, path, JSON.stringify(payload)); - return; - } - throw e; - } -}; - -export interface BookFileSource { - /** Bytes to upload. The caller owns reading them off disk. */ - bytes: ArrayBuffer; - /** Total byte length, used for the HEAD-vs-local size short-circuit. */ - size: number; -} - -export type BookFileLoader = () => Promise<BookFileSource | null>; - -/** - * Streaming alternative to {@link BookFileLoader}: hands the syncer a - * cheap-to-resolve metadata bundle (just `size` is needed for the - * HEAD-vs-local short-circuit) plus an `upload` callback that streams - * the bytes directly from disk to the WebDAV server, never letting the - * full file land in the JS heap. - * - * Why this exists separately from BookFileLoader: - * - `loader` materialises the entire file as an ArrayBuffer up-front, - * which is fine for small books (a few MB) but a hard kill switch - * for a library of multi-hundred-megabyte PDFs: each book opens - * its own ArrayBuffer in the renderer's V8 heap, and even with - * sequential `pushBookFile` calls the GC can't reliably free them - * before the next `arrayBuffer()` allocates. After 2–3 large - * books the renderer hits the heap ceiling and the WebView - * crashes — symptom: the entire UI goes blank mid-sync. - * - The streaming loader hands the file path to a Tauri command - * (`upload_file` via `tauriUpload`) which reads + uploads off the - * Rust side in chunks. JS heap stays flat regardless of book size. - * - * Callers should provide `streamingLoader` when running on Tauri (where - * `tauriUpload` is available) and fall back to `loader` on web targets - * that don't have a streaming HTTP primitive. - */ -export interface BookFileStreamingSource { - /** File size in bytes, for the HEAD-vs-local short-circuit. */ - size: number; - /** - * Stream the bytes to `remoteUrl` via PUT. Authentication headers - * are pre-baked by the caller (typically via {@link buildBasicAuthHeader}) - * because the streaming primitive can't see the WebDAV settings. - * Returns `true` on success, `false` when the upload was skipped or - * failed in a way the caller wants to swallow. - */ - upload: (remoteUrl: string, headers: Record<string, string>) => Promise<boolean>; -} - -export type BookFileStreamingLoader = () => Promise<BookFileStreamingSource | null>; - -export interface PushBookFileResult { - /** True when bytes were uploaded; false when the upload was skipped. */ - uploaded: boolean; - /** Reason for the skip, when applicable — surfaced for diagnostics. */ - reason?: 'remote-matches' | 'no-source' | 'disabled'; -} - -/** - * Upload the book file binary to - * <rootPath>/Readest/books/<hash>/<safe-title>.<ext> - * - * Idempotency story: the path lives under the per-hash directory, so the - * remote location is uniquely determined by `book.hash` (the local hash - * of the file's content). A HEAD probe + size compare lets us skip a - * re-upload whenever the remote already has a copy of the matching size, - * which in practice means we only PUT bytes the very first time a book - * is seen on a device. Renaming a book locally never re-uploads — we - * MOVE the friendly file name in a future patch (Step 3). - * - * Two upload modes are supported and mutually exclusive: - * - {@link BookFileStreamingLoader} (preferred when available): hands - * the file path off to a Tauri-side streamer. Constant JS heap - * regardless of book size — required for libraries with multi- - * hundred-megabyte PDFs, where buffering was crashing the renderer. - * - {@link BookFileLoader} (fallback): materialises the file as an - * ArrayBuffer in JS, then PUTs it via `putFileBinary`. Fine for - * small files; OOMs the WebView for large libraries. - * - * Pass `streamingLoader` when running on Tauri; pass `loader` otherwise. - * Passing both makes streaming win — the HEAD short-circuit is shared - * either way so steady-state syncs cost a single round-trip per book. - */ -export const pushBookFile = async ( - settings: WebDAVSettings, - book: Book, - loader: BookFileLoader, - streamingLoader?: BookFileStreamingLoader, -): Promise<PushBookFileResult> => { - const client = toClientConfig(settings); - const dirPath = buildBookDirPath(settings.rootPath, book.hash); - const path = buildBookFilePath(settings.rootPath, book); - - // Cheap probe first — if the remote already has a same-sized blob we - // know it's the same content (hash-keyed directory, single file inside). - let remoteHead: { size?: number } | null = null; - try { - remoteHead = await headFile(client, path); - } catch (e) { - // HEAD failures other than 404 propagate; the caller decides whether - // to surface them as a toast. - if (!(e instanceof WebDAVRequestError) || e.code !== 'NETWORK') throw e; - } - - // Streaming path: resolve metadata only, then stream bytes off disk. - // The metadata fetch (file.size) doesn't read the body, so heap - // stays flat even for gigabyte-scale PDFs. - if (streamingLoader) { - const meta = await streamingLoader(); - if (!meta) { - // Loader returned null — most often "file isn't on this device"; - // check the buffered loader as a last resort so callers that - // wired both keep working when one path is unavailable. - if (!loader) return { uploaded: false, reason: 'no-source' }; - } else { - if (remoteHead && remoteHead.size === meta.size) { - return { uploaded: false, reason: 'remote-matches' }; - } - const dirs = [...ancestorsOf(`${dirPath}/.placeholder`), dirPath]; - await ensureDirectory(client, dirs); - const remoteUrl = buildRequestUrl(settings.serverUrl, path); - const headers: Record<string, string> = { - Authorization: buildBasicAuthHeader(settings.username, settings.password), - }; - const ok = await meta.upload(remoteUrl, headers); - if (!ok) { - // Some upstream servers return 409 if a parent is recreated - // mid-PUT. Mirror the buffered path's one-shot retry: re- - // ensure directories and try again. The caller's `upload` - // implementation owns the actual error mapping; we just give - // it one more chance. - await ensureDirectory(client, dirs); - const retried = await meta.upload(remoteUrl, headers); - if (!retried) { - throw new WebDAVRequestError('Streaming upload failed', undefined, 'NETWORK'); - } - } - return { uploaded: true }; - } - } - - const local = await loader(); - if (!local) { - return { uploaded: false, reason: 'no-source' }; - } - - if (remoteHead && remoteHead.size === local.size) { - return { uploaded: false, reason: 'remote-matches' }; - } - - const dirs = [...ancestorsOf(`${dirPath}/.placeholder`), dirPath]; - await ensureDirectory(client, dirs); - try { - await putFileBinary(client, path, local.bytes); - } catch (e) { - if (e instanceof WebDAVRequestError && e.status === 409) { - await ensureDirectory(client, dirs); - await putFileBinary(client, path, local.bytes); - } else { - throw e; - } - } - return { uploaded: true }; -}; - -/** - * Upload the book's cover image to - * <rootPath>/Readest/books/<hash>/cover.png - * - * Same idempotency model as `pushBookFile`: HEAD-probe + size compare, - * then PUT only when missing or sized differently. Covers are tiny - * (typically 50–200 KB), but multiplied across a 100-book library they - * still add up — the HEAD short-circuit keeps the steady state cheap. - * - * Why a separate function (rather than rolling into `pushBookFile`): - * - Most readers don't enable `syncBooks` in v1; covers travel with - * books rather than configs, but should be a stand-alone primitive - * so a future "syncCovers" toggle can reuse it. - * - readest fetches custom covers via metadata services that produce - * better art than what's embedded in the EPUB. Those custom covers - * can't be regenerated on the receiving device, so syncing them is - * the only way to preserve user choice across devices. - */ -export const pushBookCover = async ( - settings: WebDAVSettings, - bookHash: string, - loader: BookFileLoader, -): Promise<PushBookFileResult> => { - const client = toClientConfig(settings); - const dirPath = buildBookDirPath(settings.rootPath, bookHash); - const path = buildBookCoverPath(settings.rootPath, bookHash); - - let remoteHead: { size?: number } | null = null; - try { - remoteHead = await headFile(client, path); - } catch (e) { - if (!(e instanceof WebDAVRequestError) || e.code !== 'NETWORK') throw e; - } - - const local = await loader(); - // Covers are best-effort — books without a local cover are a normal - // state (TXT/MD without metadata), so a missing source is not a - // failure and the caller shouldn't toast about it. - if (!local) return { uploaded: false, reason: 'no-source' }; - - if (remoteHead && remoteHead.size === local.size) { - return { uploaded: false, reason: 'remote-matches' }; - } - - const dirs = [...ancestorsOf(`${dirPath}/.placeholder`), dirPath]; - await ensureDirectory(client, dirs); - try { - await putFileBinary(client, path, local.bytes, 'image/png'); - } catch (e) { - if (e instanceof WebDAVRequestError && e.status === 409) { - await ensureDirectory(client, dirs); - await putFileBinary(client, path, local.bytes, 'image/png'); - } else { - throw e; - } - } - return { uploaded: true }; -}; - -export const pullBookFile = async ( - settings: WebDAVSettings, - book: Book, - explicitPath?: string, -): Promise<ArrayBuffer | null> => { - const client = toClientConfig(settings); - const path = explicitPath ?? buildBookFilePath(settings.rootPath, book); - return getFileBinary(client, path); -}; - -export const pullBookCover = async ( - settings: WebDAVSettings, - bookHash: string, -): Promise<ArrayBuffer | null> => { - const client = toClientConfig(settings); - const path = buildBookCoverPath(settings.rootPath, bookHash); - return getFileBinary(client, path); -}; - -/** - * Delete the per-book directory `<rootPath>/Readest/books/<hash>/` - * — the file, the cover and the config.json — from the WebDAV - * server in one round-trip. - * - * Used by the WebDAV browser's cleanup mode to evict orphans (remote - * dirs whose local Book carries `deletedAt`). We deliberately do - * *not* touch the local library here: the local row's `deletedAt` - * tombstone is the signal that propagates the deletion to other - * sync clients, and clearing it would resurrect the book on next - * push from a sibling device. See the cleanup-pane comment in - * `WebDAVBrowsePane.tsx` for the broader rationale. - * - * The result shape mirrors {@link PushBookFileResult} so callers - * batching a list of hashes can aggregate without exception - * boilerplate. AUTH failures still throw — they're a global - * condition, not a per-book problem, and the caller surfaces a - * single re-auth toast. - */ -export interface DeleteRemoteBookDirResult { - /** True when the server confirmed deletion (or the dir was already gone). */ - ok: boolean; - /** Compact reason string when `ok === false`, for the failure toast. */ - reason?: string; -} - -export const deleteRemoteBookDir = async ( - settings: WebDAVSettings, - bookHash: string, -): Promise<DeleteRemoteBookDirResult> => { - const client = toClientConfig(settings); - const path = buildBookDirPath(settings.rootPath, bookHash); - try { - await deleteDirectory(client, path); - return { ok: true }; - } catch (e) { - // Auth failures aren't a "this hash failed" condition — every - // subsequent hash would fail the same way. Re-throw so the - // batch loop can short-circuit and the caller can surface a - // single re-auth prompt. - if (e instanceof WebDAVRequestError && e.code === 'AUTH_FAILED') throw e; - return { - ok: false, - reason: e instanceof Error ? e.message : String(e), - }; - } -}; - -export interface RemoteLibraryIndex { - schemaVersion: 1; - books: Book[]; - updatedAt: number; -} - -export const pullLibraryIndex = async ( - settings: WebDAVSettings, -): Promise<RemoteLibraryIndex | null> => { - const client = toClientConfig(settings); - const path = buildLibraryPath(settings.rootPath); - const raw = await getFile(client, path); - if (!raw) return null; - try { - const parsed = JSON.parse(raw) as RemoteLibraryIndex; - if (parsed && parsed.schemaVersion === 1) return parsed; - } catch { - // Ignore parse errors - } - return null; -}; - -export const pushLibraryIndex = async ( - settings: WebDAVSettings, - index: RemoteLibraryIndex, -): Promise<void> => { - const client = toClientConfig(settings); - const path = buildLibraryPath(settings.rootPath); - const dirs = ancestorsOf(path); - await ensureDirectory(client, dirs); - await putFile(client, path, JSON.stringify(index)); -}; - -/** - * Aggregate result of a library-wide push. Counters are kept granular - * so the UI can render an honest "X uploaded, Y already in sync, Z - * failed" toast at the end. - */ -export interface SyncLibraryResult { - totalBooks: number; - configsUploaded: number; - configsDownloaded: number; - filesUploaded: number; - filesAlreadyInSync: number; - coversUploaded: number; - booksDownloaded: number; - /** - * Number of already-local books whose metadata (title / author / cover) - * was refreshed from a newer copy in the shared library.json (#4756). - */ - metadataUpdated: number; - failures: number; - /** - * Per-book failure breakdown for the diagnostic log surfaced in the - * Settings UI. Populated alongside `failures` so the user-facing log - * can show which books failed and why without needing to re-run with - * verbose console output. `reason` is a short single-line string — - * the caller is responsible for truncating server XML / stacks - * before persisting. - */ - failedBooks: SyncFailureEntry[]; -} - -export interface SyncFailureEntry { - hash: string; - title: string; - reason: string; - /** Which phase of the per-book pipeline failed; helps users self-triage. */ - phase: 'download' | 'upload-config' | 'upload-file' | 'upload-cover'; -} - -export interface SyncLibraryOptions { - syncBooks: boolean; - strategy?: 'silent' | 'send' | 'receive'; - loadConfig: (book: Book) => Promise<BookConfig | null>; - /** - * Provider that returns the bytes of a book's local file. Resolve to - * `null` when the book hasn't been downloaded locally — those books - * are silently skipped (they'll be picked up on the device that - * actually has the binary). - */ - loadBookFile: (book: Book) => Promise<BookFileSource | null>; - /** - * Streaming alternative to {@link SyncLibraryOptions.loadBookFile}. - * When supplied, the per-book upload path uses this in preference to - * `loadBookFile`, handing the file off to a transport that streams - * the bytes off disk directly to the WebDAV server. Required for - * libraries with multi-hundred-megabyte books — see - * {@link BookFileStreamingSource} for the heap-pressure rationale. - * - * Tauri callers should provide this; web callers (where streaming - * PUTs aren't available) leave it undefined and fall back to the - * buffered `loadBookFile` path. - */ - loadBookFileStreaming?: (book: Book) => Promise<BookFileStreamingSource | null>; - /** - * Provider that returns the bytes of a book's local cover image. - * Books without a cover (e.g. plaintext imports) resolve to `null` - * and are silently skipped — covers are best-effort, not load-bearing. - */ - loadBookCover?: (book: Book) => Promise<BookFileSource | null>; - saveBookFile?: (book: Book, bytes: ArrayBuffer) => Promise<void>; - /** - * Streaming alternative to (pullBookFile + saveBookFile) — when - * provided, the syncer hands the caller the *remote WebDAV path* and - * expects the caller to download + persist the bytes itself. This is - * how the Tauri app keeps gigabyte-scale book payloads out of the - * WebView heap (which crashes Android with a Binder OOM otherwise). - * Return `true` when the book was successfully written; `false` when - * the server returned 404 / nothing to download. - */ - downloadBookFile?: (book: Book, remotePath: string) => Promise<boolean>; - saveBookCover?: (book: Book, bytes: ArrayBuffer) => Promise<void>; - /** - * Persist a freshly-downloaded BookConfig to local disk so progress, - * bookmarks, and annotations are available the next time the user - * opens the book. Called once per book whose remote config existed. - */ - saveBookConfig?: (book: Book, config: BookConfig) => Promise<void>; - addBookToLibrary?: (book: Book) => Promise<void>; - /** - * Persist refreshed metadata for a book that already exists in the local - * library. Called once per book whose copy in the shared library.json is - * strictly newer than the local one (last-writer-wins on `book.updatedAt`), - * so a peer's title / author / cover edit propagates to devices that - * already hold the book. Distinct from `addBookToLibrary`, which only - * inserts brand-new rows and no-ops on an existing hash. - */ - updateBookMetadata?: (book: Book) => Promise<void>; - /** Stable per-device id; written into every config envelope. */ - deviceId: string; - /** - * Optional progress callback fired before each book is processed, - * suitable for driving a UI like "Syncing 3 / 42 — Project Hail Mary". - */ - onProgress?: (info: { book: Book; index: number; total: number; action?: string }) => void; -} - -/** - * Push every book in `books` to the WebDAV remote in sequence. Designed - * for the user-facing "Sync now" flow, where we trade parallelism for - * a predictable progress bar and for not hammering shared servers. - * - * Per-book steps: - * 1. Load the local config from disk (skip the book if it has none — - * brand-new entries the user has never opened don't need to sync - * anything yet). - * 2. `pushBookConfig` — creates `Readest/books/<hash>/config.json`. - * 3. `pushBookCover` (whenever a cover loader was provided) — - * HEAD-then-PUT, independent of `syncBooks`. Covers are part of - * the book's metadata: they're tiny (~30–60 KB after the import - * downscale), they can't be regenerated on a fresh device that - * doesn't hold the book bytes, and users who only opt into - * progress / notes sync still expect their bookshelf art to - * appear on the receiving device. Failures are treated as - * warnings, not hard failures. - * 4. `pushBookFile` (only when `syncBooks` is on) — HEAD-probes the - * friendly file name path, uploads if missing or size-mismatched. - * - * Failures on a single book are caught and counted; we keep going so a - * single bad apple doesn't abort the rest of the library. The aggregate - * counters returned to the caller drive the final toast. - */ -/** - * Reduce an arbitrary error to a short, single-line description for the - * per-book failure breakdown in the returned {@link SyncLibraryResult}. - * - * Goals: - * - Strip stack traces and any embedded server XML so each reason stays - * a compact one-liner. - * - Preserve the semantically useful bits — HTTP status, our own - * `code` enum (`AUTH_FAILED`, `NOT_FOUND`, `NETWORK`) — because that - * is what tells a user whether they should re-tap or fix their - * credentials. - * - Cap at 200 chars so a runaway server response doesn't make a - * single failure entry dominate the result. - */ -const formatFailureReason = (e: unknown): string => { - let message: string; - if (e instanceof WebDAVRequestError) { - const parts: string[] = []; - if (e.code) parts.push(e.code); - if (typeof e.status === 'number') parts.push(`HTTP ${e.status}`); - parts.push(e.message || 'Request failed'); - message = parts.join(' · '); - } else if (e instanceof Error) { - message = e.message || e.name || 'Unknown error'; - } else { - message = String(e); - } - // Collapse whitespace (newlines, tabs from server XML) to single spaces - // so the entry stays a true one-liner in the UI. - message = message.replace(/\s+/g, ' ').trim(); - return message.length > 200 ? `${message.slice(0, 197)}...` : message; -}; - -export const syncLibrary = async ( - settings: WebDAVSettings, - books: Book[], - options: SyncLibraryOptions, -): Promise<SyncLibraryResult> => { - const result: SyncLibraryResult = { - totalBooks: books.length, - configsUploaded: 0, - configsDownloaded: 0, - filesUploaded: 0, - filesAlreadyInSync: 0, - coversUploaded: 0, - booksDownloaded: 0, - metadataUpdated: 0, - failures: 0, - failedBooks: [], - }; - - const strategy = options.strategy || 'silent'; - const canPull = strategy !== 'send'; - const canPush = strategy !== 'receive'; - - let remoteIndex: RemoteLibraryIndex | null = null; - if (canPull) { - try { - remoteIndex = await pullLibraryIndex(settings); - } catch (e) { - console.warn('WD library sync: failed to pull index', e); - } - } - - const allBooksMap = new Map<string, Book>(); - for (const b of books) { - allBooksMap.set(b.hash, b); - } - - const remoteBooksToDownload: Book[] = []; - // The remote source of truth for "what filename does this book actually - // have on disk" is the per-hash directory listing — NOT the book's title - // (which may have been written into library.json before makeSafeFilename - // existed, or by an older buggy build). We always resolve the path by - // listing the hash dir. - const explicitRemotePaths = new Map<string, string>(); - - // Metadata reconciliation for books present BOTH locally and in the shared - // library.json (#4756). Last-writer-wins on `book.updatedAt`: when a peer's - // indexed copy is strictly newer, pull its title / author / cover down to - // this device. Without this, a device that already holds the book never - // learns about a peer's metadata edit, AND the index re-push at the end of - // syncLibrary would clobber the peer's newer metadata with this device's - // stale copy (allBooksMap still pointed at the local book). Updating - // allBooksMap with the merged copy fixes both directions at once. - if (canPull && remoteIndex && remoteIndex.books && options.updateBookMetadata) { - for (const rb of remoteIndex.books) { - if (rb.deletedAt) continue; - const local = allBooksMap.get(rb.hash); - if (!local || local.deletedAt) continue; - if ((rb.updatedAt ?? 0) <= (local.updatedAt ?? 0)) continue; - const merged = mergeRemoteBookMetadata(local, rb); - // Re-pull the cover so a changed cover travels with the metadata. The - // cover is best-effort: a missing remote cover.png simply leaves the - // existing local cover in place. The subsequent push-side pushBookCover - // HEAD/size short-circuit then matches (local now equals remote), so we - // never bounce the freshly-pulled cover back up. - if (options.saveBookCover) { - try { - const coverBytes = await pullBookCover(settings, rb.hash); - if (coverBytes) await options.saveBookCover(merged, coverBytes); - } catch (e) { - console.warn('WD library sync: metadata cover pull failed', rb.hash, e); - } - } - try { - await options.updateBookMetadata(merged); - // Keep the merged metadata authoritative for the index re-push below. - allBooksMap.set(rb.hash, merged); - result.metadataUpdated += 1; - } catch (e) { - console.warn('WD library sync: metadata update failed', rb.hash, e); - } - } - } - - if (canPull) { - const client = toClientConfig(settings); - const candidateHashes = new Set<string>(); - - // 1) Seed with hashes from the remote index (when the file exists). - if (remoteIndex && remoteIndex.books) { - for (const rb of remoteIndex.books) { - if (!allBooksMap.has(rb.hash) && !rb.deletedAt) { - candidateHashes.add(rb.hash); - // Provisionally register the indexed book — fields will be - // refreshed below once we've inspected the actual hash dir. - allBooksMap.set(rb.hash, rb); - } - } - } - - // 2) Also scan the books/ directory so legacy uploads (no library.json - // entry) and any drift between index and disk are still picked up. - try { - const booksDirPath = `${buildBasePath(settings.rootPath)}/${WEBDAV_BOOKS_DIR}`; - const dirEntries = await listDirectory(client, booksDirPath); - for (const entry of dirEntries) { - if (entry.isDirectory && !allBooksMap.has(entry.name)) { - candidateHashes.add(entry.name); - } - } - } catch (e) { - // 404 is normal if the user has never pushed anything yet. - console.warn('WD library sync: failed to list books directory', e); - } - - // 3) For every candidate, look inside its hash directory to find the - // actual book file (the only entry that isn't config.json/cover.png). - // We use that file's real path for the GET and derive title/format - // from its real name — independent of whatever is in library.json. - for (const hash of candidateHashes) { - try { - const hashDirPath = `${buildBasePath(settings.rootPath)}/${WEBDAV_BOOKS_DIR}/${hash}`; - const hashDirEntries = await listDirectory(client, hashDirPath); - const fileEntry = hashDirEntries.find( - (e) => !e.isDirectory && e.name !== 'config.json' && e.name !== 'cover.png', - ); - if (!fileEntry) continue; - - const extMatch = fileEntry.name.match(/\.([^.]+)$/); - const ext = extMatch && extMatch[1] ? extMatch[1].toUpperCase() : 'EPUB'; - const format = ext as Book['format']; - const title = fileEntry.name.replace(/\.[^.]+$/, ''); - - // If the index already gave us a book object, refresh the fields - // that might be wrong/stale from a previous buggy push. - const existing = allBooksMap.get(hash); - const book: Book = existing - ? { - ...existing, - format, - // Only override title/sourceTitle when the existing values - // look broken (no value, or contain the file extension). - title: - !existing.title || existing.title.toLowerCase().endsWith(`.${ext.toLowerCase()}`) - ? title - : existing.title, - sourceTitle: title, - updatedAt: existing.updatedAt || Date.now(), - createdAt: existing.createdAt || Date.now(), - } - : { - hash, - format, - title, - sourceTitle: title, - author: 'Unknown', - createdAt: Date.now(), - updatedAt: Date.now(), - }; - - explicitRemotePaths.set(hash, fileEntry.path); - remoteBooksToDownload.push(book); - allBooksMap.set(hash, book); - } catch (e) { - console.warn('WD library sync: failed to inspect hash dir', hash, e); - } - } - } - - // Discovery+download is *not* gated on `syncBooks`. The toggle controls - // whether the *push* side ships binaries to the remote — pulling books - // that exist remotely but not locally is the whole point of a sync, and - // the user already paid the storage cost on the remote side. Without - // this, a fresh device with `syncBooks: false` (the default) would - // never see any of the books in the WebDAV root, which is the exact - // "I only see 1 / 5 books" footgun users have hit. - if (canPull && (options.saveBookFile || options.downloadBookFile) && options.addBookToLibrary) { - for (let i = 0; i < remoteBooksToDownload.length; i++) { - const rb = remoteBooksToDownload[i]!; - options.onProgress?.({ - book: rb, - index: i, - total: remoteBooksToDownload.length, - action: 'downloading', - }); - try { - const explicitPath = explicitRemotePaths.get(rb.hash); - // Prefer the streaming downloader when the caller provides one. - // On Tauri/Android we MUST take this path — moving a 30 MB epub - // through the WebView <-> Rust IPC bridge as a single Uint8Array - // crashes the renderer. - let written = false; - if (options.downloadBookFile && explicitPath) { - written = await options.downloadBookFile(rb, explicitPath); - } else if (options.saveBookFile) { - const fileBytes = await pullBookFile(settings, rb, explicitPath); - if (fileBytes) { - await options.saveBookFile(rb, fileBytes); - written = true; - } - } - if (written) { - if (options.saveBookCover) { - try { - const coverBytes = await pullBookCover(settings, rb.hash); - if (coverBytes) await options.saveBookCover(rb, coverBytes); - } catch (e) { - console.warn('WD library sync: cover download failed', rb.hash, e); - } - } - // Pull the remote config so progress, bookmarks and annotations - // travel with the book. This is best-effort: a missing config - // simply means "no remote progress yet" and is not a failure. - if (options.saveBookConfig) { - try { - const emptyLocal: BookConfig = { updatedAt: 0, booknotes: [] }; - const pullResult = await pullBookConfig(settings, rb, emptyLocal); - if (pullResult.applied && pullResult.mergedConfig) { - await options.saveBookConfig(rb, pullResult.mergedConfig); - result.configsDownloaded += 1; - } - } catch (e) { - console.warn('WD library sync: config download failed', rb.hash, e); - } - } - await options.addBookToLibrary(rb); - result.booksDownloaded += 1; - } else { - // No bytes returned (typically a 404 we couldn't resolve) — - // count as a failure so the user sees something happened. - result.failures += 1; - result.failedBooks.push({ - hash: rb.hash, - title: rb.title || rb.hash, - phase: 'download', - reason: 'No bytes returned (file may have been moved or deleted on the server)', - }); - console.warn('WD library sync: book download produced no bytes', rb.hash, explicitPath); - } - } catch (e) { - result.failures += 1; - result.failedBooks.push({ - hash: rb.hash, - title: rb.title || rb.hash, - phase: 'download', - reason: formatFailureReason(e), - }); - console.warn('WD library sync: book download failed', rb.hash, e); - } - } - } - - // Books we just downloaded already exist on the remote — don't waste - // bandwidth/time HEAD-probing and re-pushing them. Only push books that - // were already present in the caller-supplied local library. - const downloadedHashes = new Set(remoteBooksToDownload.map((b) => b.hash)); - const booksToPush = books.filter((b) => !b.deletedAt && !downloadedHashes.has(b.hash)); - result.totalBooks = booksToPush.length; - - if (canPush && booksToPush.length > 0) { - for (let i = 0; i < booksToPush.length; i += 1) { - const book = booksToPush[i]!; - options.onProgress?.({ book, index: i, total: booksToPush.length, action: 'uploading' }); - // Track which step we were in when an exception escapes the inner - // try, so the user-facing log can pinpoint whether config / file / - // cover upload tripped the wire. Cover failures are caught locally - // (covers are best-effort) and don't update this. - let phase: SyncFailureEntry['phase'] = 'upload-config'; - try { - const config = await options.loadConfig(book); - if (config) { - // Mirror the reader hook's pull-merge-push discipline so a manual - // "Sync now" can't blind-overwrite state this device hasn't pulled - // yet: a peer's booknotes (element-set CRDT) or newer progress - // (per-config LWW). Only in two-way ('silent') mode — 'send' - // deliberately treats the local copy as authoritative and keeps the - // blind push. A failed pull-merge falls back to the local config so - // a flaky GET never blocks the book's sync. - let configToPush = config; - if (canPull) { - try { - const pull = await pullBookConfig(settings, book, config); - if (pull.applied && pull.mergedConfig) { - configToPush = pull.mergedConfig; - // Persist the merged superset locally so this device converges - // too, not just the remote. - if (options.saveBookConfig) { - await options.saveBookConfig(book, pull.mergedConfig); - } - } - } catch (e) { - console.warn('WD library sync: config pull-merge failed', book.hash, e); - } - } - await pushBookConfig(settings, book, configToPush, options.deviceId); - result.configsUploaded += 1; - } - // Covers ride along with the config-level sync, NOT with - // syncBooks. They are conceptually part of the book's metadata - // (the receiving device can't regenerate them without the book - // bytes, which the user may have chosen not to sync), and at - // ~30–60 KB after the import-time downscale the bandwidth cost - // is negligible compared to the user benefit of "my shelf art - // shows up on every device". Failures here are warnings, not - // hard failures, so a flaky cover upload doesn't abort the - // rest of the per-book pipeline. - if (options.loadBookCover) { - try { - const coverResult = await pushBookCover(settings, book.hash, () => - options.loadBookCover!(book), - ); - if (coverResult.uploaded) result.coversUploaded += 1; - } catch (e) { - console.warn('WD library sync: cover failed', book.hash, e); - } - } - if (options.syncBooks) { - phase = 'upload-file'; - const fileResult = await pushBookFile( - settings, - book, - () => options.loadBookFile(book), - options.loadBookFileStreaming ? () => options.loadBookFileStreaming!(book) : undefined, - ); - if (fileResult.uploaded) { - result.filesUploaded += 1; - } else if (fileResult.reason === 'remote-matches') { - result.filesAlreadyInSync += 1; - } - } - } catch (e) { - result.failures += 1; - result.failedBooks.push({ - hash: book.hash, - title: book.title || book.hash, - phase, - reason: formatFailureReason(e), - }); - console.warn('WD library sync: book failed', book.hash, e); - } - } - } - - // Push the merged index whenever we're allowed to write to the remote, - // even if we didn't upload any binaries this turn (e.g. all books were - // freshly pulled from the remote). Keeps library.json authoritative. - // - // Per-hash directories of soft-deleted books are intentionally NOT - // GC'd from this sync path: a peer that hasn't pulled this push yet - // would see the `deletedAt` tombstone arrive together with the bytes - // already gone, surfacing as a phantom-deleted shelf row. The orphan - // sweep is instead exposed manually via WebDAVBrowsePane's cleanup - // mode, where it's the user (per device) who decides when the - // deletion has settled. See `deleteRemoteBookDir` above. - if (canPush) { - try { - const newIndex: RemoteLibraryIndex = { - schemaVersion: 1, - books: Array.from(allBooksMap.values()), - updatedAt: Date.now(), - }; - await pushLibraryIndex(settings, newIndex); - } catch (e) { - console.warn('WD library sync: failed to push index', e); - } - } - - return result; -}; diff --git a/apps/readest-app/src/types/settings.ts b/apps/readest-app/src/types/settings.ts index 729ec5cf..41412953 100644 --- a/apps/readest-app/src/types/settings.ts +++ b/apps/readest-app/src/types/settings.ts @@ -117,6 +117,10 @@ export interface WebDAVSettings { syncProgress?: boolean; syncNotes?: boolean; syncBooks?: boolean; + // When true, "Sync now" re-checks every book instead of only those whose + // local copy differs from the shared library.json index (the default + // incremental walk). An escape hatch for drift or a first full sync. + fullSync?: boolean; // Conflict policy — same vocabulary as KOSync so users only learn one. strategy?: KOSyncStrategy; // Stable per-device id (uuidv4); written into library.json so we can tell