From 3b86229c8f37acb898a5b57bff8a3af5cdae87bb Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Mon, 27 Oct 2025 20:37:04 +0800 Subject: [PATCH] perf(sync): batched updating notes and books when syncing (#2337) --- .../src/app/library/hooks/useBooksSync.ts | 56 ++++++- .../src/app/reader/hooks/useNotesSync.ts | 35 ++-- apps/readest-app/src/pages/api/sync.ts | 157 ++++++++++++------ 3 files changed, 171 insertions(+), 77 deletions(-) diff --git a/apps/readest-app/src/app/library/hooks/useBooksSync.ts b/apps/readest-app/src/app/library/hooks/useBooksSync.ts index ee742fda..0fe918b5 100644 --- a/apps/readest-app/src/app/library/hooks/useBooksSync.ts +++ b/apps/readest-app/src/app/library/hooks/useBooksSync.ts @@ -1,34 +1,74 @@ -import { useCallback, useEffect } from 'react'; -import { useAuth } from '@/context/AuthContext'; -import { useEnv } from '@/context/EnvContext'; -import { useSync } from '@/hooks/useSync'; +import { useCallback, useEffect, useRef } from 'react'; import { Book } from '@/types/book'; +import { useSync } from '@/hooks/useSync'; +import { useEnv } from '@/context/EnvContext'; +import { useAuth } from '@/context/AuthContext'; import { useLibraryStore } from '@/store/libraryStore'; +import { SYNC_BOOKS_INTERVAL_SEC } from '@/services/constants'; +import { throttle } from '@/utils/throttle'; export const useBooksSync = () => { const { user } = useAuth(); const { appService } = useEnv(); const { library, isSyncing, setLibrary, setIsSyncing, setSyncProgress } = useLibraryStore(); const { syncedBooks, syncBooks, lastSyncedAtBooks } = useSync(); + const isPullingRef = useRef(false); const getNewBooks = useCallback(() => { - if (!user) return []; + if (!user) return {}; const library = useLibraryStore.getState().library; const newBooks = library.filter( (book) => lastSyncedAtBooks < book.updatedAt || lastSyncedAtBooks < (book.deletedAt ?? 0), ); - return newBooks; + return { + books: newBooks, + lastSyncedAt: lastSyncedAtBooks, + }; }, [user, lastSyncedAtBooks]); const pullLibrary = useCallback(async () => { if (!user) return; - await syncBooks([], 'pull'); + if (isPullingRef.current) { + console.log('Pull already in progress, skipping...'); + return; + } + try { + isPullingRef.current = true; + await syncBooks([], 'pull'); + } finally { + isPullingRef.current = false; + } }, [user, syncBooks]); + // eslint-disable-next-line react-hooks/exhaustive-deps + const handleAutoSync = useCallback( + throttle( + () => { + const newBooks = getNewBooks(); + if (newBooks.lastSyncedAt) { + syncBooks(newBooks.books, 'both'); + } + }, + SYNC_BOOKS_INTERVAL_SEC * 1000, + { emitLast: false }, + ), + [syncBooks], + ); + + useEffect(() => { + if (!user) return; + if (isPullingRef.current) { + console.log('Pull already in progress, skipping...'); + return; + } + handleAutoSync(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [library, handleAutoSync]); + const pushLibrary = useCallback(async () => { if (!user) return; const newBooks = getNewBooks(); - await syncBooks(newBooks, 'push'); + await syncBooks(newBooks?.books, 'push'); }, [user, syncBooks, getNewBooks]); useEffect(() => { diff --git a/apps/readest-app/src/app/reader/hooks/useNotesSync.ts b/apps/readest-app/src/app/reader/hooks/useNotesSync.ts index e54fd770..12c6725f 100644 --- a/apps/readest-app/src/app/reader/hooks/useNotesSync.ts +++ b/apps/readest-app/src/app/reader/hooks/useNotesSync.ts @@ -4,7 +4,7 @@ import { useSync } from '@/hooks/useSync'; import { BookNote } from '@/types/book'; import { useBookDataStore } from '@/store/bookDataStore'; import { SYNC_NOTES_INTERVAL_SEC } from '@/services/constants'; -import { debounce } from '@/utils/debounce'; +import { throttle } from '@/utils/throttle'; export const useNotesSync = (bookKey: string) => { const { user } = useAuth(); @@ -12,32 +12,37 @@ export const useNotesSync = (bookKey: string) => { const { getConfig, setConfig, getBookData } = useBookDataStore(); const config = getConfig(bookKey); - const bookHash = bookKey.split('-')[0]!; - const getNewNotes = () => { + const getNewNotes = useCallback(() => { const config = getConfig(bookKey); const book = getBookData(bookKey)?.book; - if (!config?.location || !book || !user) return []; + if (!config?.location || !book || !user) return {}; - const metaHash = book.metaHash; const bookNotes = config.booknotes ?? []; const newNotes = bookNotes.filter( (note) => lastSyncedAtNotes < note.updatedAt || lastSyncedAtNotes < (note.deletedAt ?? 0), ); newNotes.forEach((note) => { - note.bookHash = bookHash; - note.metaHash = metaHash; + note.bookHash = book.hash; + note.metaHash = book.metaHash; }); - return newNotes; - }; + return { + notes: newNotes, + lastSyncedAt: lastSyncedAtNotes, + }; + }, [user, bookKey, lastSyncedAtNotes, getConfig, getBookData]); // eslint-disable-next-line react-hooks/exhaustive-deps const handleAutoSync = useCallback( - debounce(() => { - const book = getBookData(bookKey)?.book; - const newNotes = getNewNotes(); - syncNotes(newNotes, bookHash, book?.metaHash, 'both'); - }, SYNC_NOTES_INTERVAL_SEC * 1000), + throttle( + () => { + const book = getBookData(bookKey)?.book; + const newNotes = getNewNotes(); + syncNotes(newNotes.notes, book?.hash, book?.metaHash, 'both'); + }, + SYNC_NOTES_INTERVAL_SEC * 1000, + { emitLast: false }, + ), [syncNotes], ); @@ -67,7 +72,7 @@ export const useNotesSync = (bookKey: string) => { if (syncedNotes?.length && config) { const book = getBookData(bookKey)?.book; const newNotes = syncedNotes.filter( - (note) => note.bookHash === bookHash || note.metaHash === book?.metaHash, + (note) => note.bookHash === book?.hash || note.metaHash === book?.metaHash, ); if (!newNotes.length) return; const oldNotes = config.booknotes ?? []; diff --git a/apps/readest-app/src/pages/api/sync.ts b/apps/readest-app/src/pages/api/sync.ts index a350298e..eeb0164f 100644 --- a/apps/readest-app/src/pages/api/sync.ts +++ b/apps/readest-app/src/pages/api/sync.ts @@ -9,6 +9,7 @@ import { transformBookToDB } from '@/utils/transform'; import { runMiddleware, corsAllMethods } from '@/utils/cors'; import { SyncData, SyncResult, SyncType } from '@/libs/sync'; import { validateUserAndToken } from '@/utils/access'; +import { DBBook, DBBookConfig } from '@/types/records'; const transformsToDB = { books: transformBookToDB, @@ -129,82 +130,130 @@ export async function POST(req: NextRequest) { return NextResponse.json({ error: 'Not authenticated' }, { status: 403 }); } const supabase = createSupabaseClient(token); - const body = await req.json(); const { books = [], configs = [], notes = [] } = body as SyncData; + const BATCH_SIZE = 100; const upsertRecords = async ( table: TableName, primaryKeys: (keyof BookDataRecord)[], records: BookDataRecord[], ) => { - const authoritativeRecords: BookDataRecord[] = []; + if (records.length === 0) return { data: [] }; - for (const rec of records) { - const dbRec = transformsToDB[table](rec, user.id); - rec.user_id = user.id; - rec.book_hash = dbRec.book_hash; - const matchConditions: Record = { user_id: user.id }; - for (const pk of primaryKeys) { - matchConditions[pk] = rec[pk]!; - } + const allAuthoritativeRecords: BookDataRecord[] = []; - const { data: serverData, error: fetchError } = await supabase + // Process in batches + for (let i = 0; i < records.length; i += BATCH_SIZE) { + const batch = records.slice(i, i + BATCH_SIZE); + + // Transform all records to DB format + const dbRecords = batch.map((rec) => { + const dbRec = transformsToDB[table](rec, user.id); + rec.user_id = user.id; + rec.book_hash = dbRec.book_hash; + return { original: rec, db: dbRec }; + }); + + // Build match conditions for batch + const matchConditions = dbRecords.map(({ original }) => { + const conditions: Record = { user_id: user.id }; + for (const pk of primaryKeys) { + conditions[pk] = original[pk]!; + } + return conditions; + }); + + // Fetch existing records for this batch + const orConditions = matchConditions + .map((cond) => { + const parts = Object.entries(cond).map(([key, val]) => `and(${key}.eq.${val})`); + return parts.join(','); + }) + .join(','); + + const { data: serverRecords, error: fetchError } = await supabase .from(table) .select() - .match(matchConditions) - .single(); + .or(orConditions); - if (fetchError && fetchError.code !== 'PGRST116') { + if (fetchError) { return { error: fetchError.message }; } - if (!serverData) { - // use server updated_at for new records - dbRec.updated_at = new Date().toISOString(); - const { data: inserted, error: insertError } = await supabase - .from(table) - .insert(dbRec) - .select() - .single(); - if (insertError) { - console.log(`Failed to insert ${table} record:`, JSON.stringify(dbRec)); - return { - error: insertError.message, - }; - } - authoritativeRecords.push(inserted); - } else { - const clientUpdatedAt = dbRec.updated_at ? new Date(dbRec.updated_at).getTime() : 0; - const serverUpdatedAt = serverData.updated_at - ? new Date(serverData.updated_at).getTime() - : 0; - const clientDeletedAt = dbRec.deleted_at ? new Date(dbRec.deleted_at).getTime() : 0; - const serverDeletedAt = serverData.deleted_at - ? new Date(serverData.deleted_at).getTime() - : 0; - const clientIsNewer = - clientDeletedAt > serverDeletedAt || clientUpdatedAt > serverUpdatedAt; + // Create lookup map + const serverRecordsMap = new Map(); + (serverRecords || []).forEach((record) => { + const key = primaryKeys.map((pk) => record[pk]).join('|'); + serverRecordsMap.set(key, record); + }); - if (clientIsNewer) { - const { data: updated, error: updateError } = await supabase - .from(table) - .update(dbRec) - .match(matchConditions) - .select() - .single(); - if (updateError) { - console.log(`Failed to update ${table} record:`, JSON.stringify(dbRec)); - return { error: updateError.message }; - } - authoritativeRecords.push(updated); + // Separate into inserts and updates + const toInsert: (DBBook | DBBookConfig | DBBookConfig)[] = []; + const toUpdate: (DBBook | DBBookConfig | DBBookConfig)[] = []; + const batchAuthoritativeRecords: BookDataRecord[] = []; + + for (const { original, db: dbRec } of dbRecords) { + const key = primaryKeys.map((pk) => original[pk]).join('|'); + const serverData = serverRecordsMap.get(key); + + if (!serverData) { + dbRec.updated_at = new Date().toISOString(); + toInsert.push(dbRec); } else { - authoritativeRecords.push(serverData); + const clientUpdatedAt = dbRec.updated_at ? new Date(dbRec.updated_at).getTime() : 0; + const serverUpdatedAt = serverData.updated_at + ? new Date(serverData.updated_at).getTime() + : 0; + const clientDeletedAt = dbRec.deleted_at ? new Date(dbRec.deleted_at).getTime() : 0; + const serverDeletedAt = serverData.deleted_at + ? new Date(serverData.deleted_at).getTime() + : 0; + const clientIsNewer = + clientDeletedAt > serverDeletedAt || clientUpdatedAt > serverUpdatedAt; + + if (clientIsNewer) { + toUpdate.push(dbRec); + } else { + batchAuthoritativeRecords.push(serverData); + } } } + + // Batch insert + if (toInsert.length > 0) { + const { data: inserted, error: insertError } = await supabase + .from(table) + .insert(toInsert) + .select(); + + if (insertError) { + console.log(`Failed to insert ${table} records:`, JSON.stringify(toInsert)); + return { error: insertError.message }; + } + batchAuthoritativeRecords.push(...(inserted || [])); + } + + // Batch upsert + if (toUpdate.length > 0) { + const { data: updated, error: updateError } = await supabase + .from(table) + .upsert(toUpdate, { + onConflict: ['user_id', ...primaryKeys].join(','), + }) + .select(); + + if (updateError) { + console.log(`Failed to update ${table} records:`, JSON.stringify(toUpdate)); + return { error: updateError.message }; + } + batchAuthoritativeRecords.push(...(updated || [])); + } + + allAuthoritativeRecords.push(...batchAuthoritativeRecords); } - return { data: authoritativeRecords }; + return { data: allAuthoritativeRecords }; }; try {