perf: much faster sync of the whole library (#2336)

2025-10-27 12:33:12 +08:00
parent 9bc46914e2
commit c4d498d183
7 changed files with 326 additions and 155 deletions
@@ -2,73 +2,56 @@ import { useCallback, useEffect } from 'react';
 import { useAuth } from '@/context/AuthContext';
 import { useEnv } from '@/context/EnvContext';
 import { useSync } from '@/hooks/useSync';
-import { useLibraryStore } from '@/store/libraryStore';
 import { Book } from '@/types/book';
-import { debounce } from '@/utils/debounce';
-import { SYNC_BOOKS_INTERVAL_SEC } from '@/services/constants';
+import { useLibraryStore } from '@/store/libraryStore';

-export interface UseBooksSyncProps {
-  onSyncStart?: () => void;
-  onSyncEnd?: () => void;
-}
-
-export const useBooksSync = ({ onSyncStart, onSyncEnd }: UseBooksSyncProps) => {
+export const useBooksSync = () => {
  const { user } = useAuth();
  const { appService } = useEnv();
-  const { library, setLibrary } = useLibraryStore();
+  const { library, isSyncing, setLibrary, setIsSyncing, setSyncProgress } = useLibraryStore();
  const { syncedBooks, syncBooks, lastSyncedAtBooks } = useSync();

-  const pullLibrary = useCallback(async () => {
-    if (!user) return;
-    await syncBooks([], 'pull');
-  }, [user, syncBooks]);
-
-  const pushLibrary = async () => {
-    if (!user) return;
-    const newBooks = getNewBooks();
-    syncBooks(newBooks, 'push');
-  };
-
-  useEffect(() => {
-    if (!user) return;
-    pullLibrary();
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [user]);
-
-  const getNewBooks = () => {
+  const getNewBooks = useCallback(() => {
    if (!user) return [];
    const library = useLibraryStore.getState().library;
    const newBooks = library.filter(
      (book) => lastSyncedAtBooks < book.updatedAt || lastSyncedAtBooks < (book.deletedAt ?? 0),
    );
    return newBooks;
-  };
+  }, [user, lastSyncedAtBooks]);

-  // eslint-disable-next-line react-hooks/exhaustive-deps
-  const handleAutoSync = useCallback(
-    debounce(() => {
-      const newBooks = getNewBooks();
-      syncBooks(newBooks, 'both');
-    }, SYNC_BOOKS_INTERVAL_SEC * 1000),
-    [syncBooks],
-  );
+  const pullLibrary = useCallback(async () => {
+    if (!user) return;
+    await syncBooks([], 'pull');
+  }, [user, syncBooks]);
+
+  const pushLibrary = useCallback(async () => {
+    if (!user) return;
+    const newBooks = getNewBooks();
+    await syncBooks(newBooks, 'push');
+  }, [user, syncBooks, getNewBooks]);

  useEffect(() => {
    if (!user) return;
-    handleAutoSync();
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [library, handleAutoSync]);
+    pullLibrary();
+  }, [user, pullLibrary]);

  const updateLibrary = async () => {
+    if (isSyncing) return;
    if (!syncedBooks?.length) return;
+
    // Process old books first so that when we update the library the order is preserved
    syncedBooks.sort((a, b) => a.updatedAt - b.updatedAt);
+    const bookHashesInSynced = new Set(syncedBooks.map((book) => book.hash));
+    const oldBooks = library.filter((book) => bookHashesInSynced.has(book.hash));
+    const oldBooksNeedsDownload = oldBooks.filter((book) => {
+      return !book.deletedAt && book.uploadedAt && !book.coverDownloadedAt;
+    });

    const processOldBook = async (oldBook: Book) => {
      const matchingBook = syncedBooks.find((newBook) => newBook.hash === oldBook.hash);
      if (matchingBook) {
        if (!matchingBook.deletedAt && matchingBook.uploadedAt && !oldBook.coverDownloadedAt) {
-          await appService?.downloadBook(oldBook, true);
          oldBook.coverImageUrl = await appService?.generateCoverImageUrl(oldBook);
        }
        const mergedBook =
@@ -80,31 +63,44 @@ export const useBooksSync = ({ onSyncStart, onSyncEnd }: UseBooksSyncProps) => {
      return oldBook;
    };

-    const updatedLibrary = await Promise.all(library.map(processOldBook));
-    const processNewBook = async (newBook: Book) => {
-      if (!updatedLibrary.some((oldBook) => oldBook.hash === newBook.hash)) {
-        if (newBook.uploadedAt && !newBook.deletedAt) {
-          try {
-            await appService?.downloadBook(newBook, true);
-          } catch {
-            console.error('Failed to download book:', newBook);
-          } finally {
-            newBook.coverImageUrl = await appService?.generateCoverImageUrl(newBook);
-            updatedLibrary.push(newBook);
-            setLibrary(updatedLibrary);
-          }
-        }
-      }
-    };
-    onSyncStart?.();
-    const batchSize = 3;
-    for (let i = 0; i < syncedBooks.length; i += batchSize) {
-      const batch = syncedBooks.slice(i, i + batchSize);
-      await Promise.all(batch.map(processNewBook));
+    const oldBooksBatchSize = 20;
+    for (let i = 0; i < oldBooksNeedsDownload.length; i += oldBooksBatchSize) {
+      const batch = oldBooksNeedsDownload.slice(i, i + oldBooksBatchSize);
+      await appService?.downloadBookCovers(batch);
    }
-    onSyncEnd?.();
+
+    const updatedLibrary = await Promise.all(library.map(processOldBook));
+    const bookHashesInLibrary = new Set(updatedLibrary.map((book) => book.hash));
+    const newBooks = syncedBooks.filter(
+      (newBook) =>
+        !bookHashesInLibrary.has(newBook.hash) && newBook.uploadedAt && !newBook.deletedAt,
+    );
+
+    const processNewBook = async (newBook: Book) => {
+      newBook.coverImageUrl = await appService?.generateCoverImageUrl(newBook);
+      updatedLibrary.push(newBook);
+    };
+
+    if (newBooks.length > 0) {
+      setIsSyncing(true);
+    }
+
+    const batchSize = 10;
+    for (let i = 0; i < newBooks.length; i += batchSize) {
+      const batch = newBooks.slice(i, i + batchSize);
+      await appService?.downloadBookCovers(batch);
+      await Promise.all(batch.map(processNewBook));
+      const progress = Math.min((i + batchSize) / newBooks.length, 1);
+      setLibrary([...updatedLibrary]);
+      setSyncProgress(progress);
+    }
+
    setLibrary(updatedLibrary);
    appService?.saveLibraryBooks(updatedLibrary);
+
+    if (newBooks.length > 0) {
+      setIsSyncing(false);
+    }
  };

  useEffect(() => {
@@ -68,6 +68,8 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP
  const { token, user } = useAuth();
  const {
    library: libraryBooks,
+    isSyncing,
+    syncProgress,
    updateBook,
    setLibrary,
    checkOpenWithBooks,
@@ -103,10 +105,7 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP

  useOpenWithBooks();

-  const { pullLibrary, pushLibrary } = useBooksSync({
-    onSyncStart: () => setLoading(true),
-    onSyncEnd: () => setLoading(false),
-  });
+  const { pullLibrary, pushLibrary } = useBooksSync();

  usePullToRefresh(containerRef, pullLibrary);
  useScreenWakeLock(settings.screenWakeLock);
@@ -692,7 +691,7 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP
          onDeselectAll={handleDeselectAll}
        />
      </div>
-      {loading && (
+      {(loading || isSyncing) && (
        <div className='fixed inset-0 z-50 flex items-center justify-center'>
          <Spinner loading />
        </div>
@@ -723,6 +722,14 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP
                paddingLeft: `${insets.left}px`,
              }}
            >
+              <progress
+                className={clsx(
+                  'progress progress-success absolute left-0 right-0 top-[2px] z-30 h-1 transition-opacity duration-200',
+                  isSyncing ? 'opacity-100' : 'opacity-0',
+                )}
+                value={syncProgress * 100}
+                max='100'
+              ></progress>
              <DropIndicator />
              <Bookshelf
                libraryBooks={libraryBooks}
@@ -1,4 +1,5 @@
 import { getAPIBaseUrl, isWebAppPlatform } from '@/services/environment';
+import { AppService } from '@/types/system';
 import { getUserID } from '@/utils/access';
 import { fetchWithAuth } from '@/utils/fetch';
 import {
@@ -69,35 +70,84 @@ export const uploadFile = async (
  }
 };

-export const downloadFile = async (
-  filePath: string,
-  fileFullPath: string,
-  onProgress?: ProgressHandler,
-) => {
+export const batchGetDownloadUrls = async (files: { lfp: string; cfp: string }[]) => {
+  try {
+    const userId = await getUserID();
+    if (!userId) {
+      throw new Error('Not authenticated');
+    }
+    const filePaths = files.map((file) => file.cfp);
+    const fileKeys = filePaths.map((path) => `${userId}/${path}`);
+    const response = await fetchWithAuth(`${API_ENDPOINTS.download}`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ fileKeys }),
+    });
+
+    const { downloadUrls } = await response.json();
+    return files.map((file) => {
+      const fileKey = `${userId}/${file.cfp}`;
+      return {
+        lfp: file.lfp,
+        cfp: file.cfp,
+        downloadUrl: downloadUrls[fileKey],
+      };
+    });
+  } catch (error) {
+    console.error('Batch get download URLs failed:', error);
+    throw new Error('Batch get download URLs failed');
+  }
+};
+
+type DownloadFileParams = {
+  appService: AppService;
+  dst: string;
+  cfp: string;
+  url?: string;
+  onProgress?: ProgressHandler;
+};
+
+export const downloadFile = async ({
+  appService,
+  dst,
+  cfp,
+  url,
+  onProgress,
+}: DownloadFileParams) => {
  try {
    const userId = await getUserID();
    if (!userId) {
      throw new Error('Not authenticated');
    }

-    const fileKey = `${userId}/${filePath}`;
-    const response = await fetchWithAuth(
-      `${API_ENDPOINTS.download}?fileKey=${encodeURIComponent(fileKey)}`,
-      {
-        method: 'GET',
-      },
-    );
+    let downloadUrl = url;
+    if (!downloadUrl) {
+      const fileKey = `${userId}/${cfp}`;
+      const response = await fetchWithAuth(
+        `${API_ENDPOINTS.download}?fileKey=${encodeURIComponent(fileKey)}`,
+        {
+          method: 'GET',
+        },
+      );

-    const { downloadUrl } = await response.json();
+      const { downloadUrl: url } = await response.json();
+      downloadUrl = url;
+    }
+
+    if (!downloadUrl) {
+      throw new Error('No download URL available');
+    }

    if (isWebAppPlatform()) {
-      return await webDownload(downloadUrl, onProgress);
+      const file = await webDownload(downloadUrl, onProgress);
+      await appService.writeFile(dst, 'None', await file.arrayBuffer());
    } else {
-      await tauriDownload(downloadUrl, fileFullPath, onProgress);
-      return;
+      await tauriDownload(downloadUrl, dst, onProgress);
    }
  } catch (error) {
-    console.error(`File '${filePath}' download failed:`, error);
+    console.error(`File '${dst}' download failed:`, error);
    throw new Error('File download failed');
  }
 };
@@ -7,7 +7,7 @@ import { validateUserAndToken } from '@/utils/access';
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
  await runMiddleware(req, res, corsAllMethods);

-  if (req.method !== 'GET') {
+  if (req.method !== 'GET' && req.method !== 'POST') {
    return res.status(405).json({ error: 'Method not allowed' });
  }

@@ -17,70 +17,141 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
      return res.status(403).json({ error: 'Not authenticated' });
    }

-    const { fileKey } = req.query;
-
-    if (!fileKey || typeof fileKey !== 'string') {
-      return res.status(400).json({ error: 'Missing or invalid fileKey' });
-    }
-
-    // Verify the file belongs to the user
-    const supabase = createSupabaseAdminClient();
-    const result = await supabase
-      .from('files')
-      .select('user_id, file_key')
-      .eq('user_id', user.id)
-      .eq('file_key', fileKey) // index idx_files_file_key_deleted_at on public.files
-      .is('deleted_at', null)
-      .limit(1)
-      .single();
-
-    const { error: fileError } = result;
-    let { data: fileRecord } = result;
-
-    if (fileError || !fileRecord) {
-      // Fallback for corrupted file names, using book hash and file extension to match fileKey
-      if (fileKey.includes('Readest/Book')) {
-        const parts = fileKey.split('/');
-        if (parts.length === 5) {
-          const bookHash = parts[3]!;
-          const filename = parts[4]!;
-          const fileExtension = filename.split('.').pop() || '';
-
-          const { data: fileRecords, error: fileError } = await supabase
-            .from('files')
-            .select('user_id, file_key')
-            .eq('user_id', user.id)
-            .eq('book_hash', bookHash)
-            .is('deleted_at', null);
-
-          if (!fileError && fileRecords && fileRecords.length > 0) {
-            const matchedFile = fileRecords.find((f) => f.file_key.endsWith(`.${fileExtension}`));
-            if (matchedFile) {
-              fileRecord = matchedFile;
-            }
-          } else {
-            return res.status(404).json({ error: 'File not found' });
-          }
-        }
+    if (req.method === 'GET') {
+      const { fileKey } = req.query;
+      if (!fileKey || typeof fileKey !== 'string') {
+        return res.status(400).json({ error: 'Missing or invalid fileKey' });
      }
+
+      const downloadUrlsMap = await processFileKeys([fileKey], user.id);
+      const downloadUrl = downloadUrlsMap[fileKey];
+
+      if (!downloadUrl) {
+        return res.status(404).json({ error: 'File not found' });
+      }
+
+      return res.status(200).json({ downloadUrl });
    }

-    if (fileRecord?.user_id !== user.id) {
-      return res.status(403).json({ error: 'Unauthorized access to the file' });
-    }
+    if (req.method === 'POST') {
+      const { fileKeys } = req.body;

-    try {
-      const downloadUrl = await getDownloadSignedUrl(fileRecord.file_key, 1800);
+      if (!fileKeys || !Array.isArray(fileKeys)) {
+        return res.status(400).json({ error: 'Missing or invalid fileKeys array' });
+      }

-      res.status(200).json({
-        downloadUrl,
-      });
-    } catch (error) {
-      console.error('Error creating signed URL for download:', error);
-      res.status(500).json({ error: 'Could not create signed URL for download' });
+      if (fileKeys.length === 0) {
+        return res.status(400).json({ error: 'fileKeys array cannot be empty' });
+      }
+
+      if (!fileKeys.every((key) => typeof key === 'string')) {
+        return res.status(400).json({ error: 'All fileKeys must be strings' });
+      }
+
+      const downloadUrls = await processFileKeys(fileKeys, user.id);
+
+      return res.status(200).json({ downloadUrls });
    }
  } catch (error) {
    console.error(error);
    return res.status(500).json({ error: 'Something went wrong' });
  }
 }
+
+async function processFileKeys(
+  fileKeys: string[],
+  userId: string,
+): Promise<Record<string, string | undefined>> {
+  const supabase = createSupabaseAdminClient();
+
+  const { data: fileRecords, error: fileError } = await supabase
+    .from('files')
+    .select('user_id, file_key, book_hash')
+    .eq('user_id', userId)
+    .in('file_key', fileKeys)
+    .is('deleted_at', null);
+
+  if (fileError) {
+    console.error('Error querying files:', fileError);
+    return Object.fromEntries(fileKeys.map((key) => [key, undefined]));
+  }
+
+  const fileRecordMap = new Map((fileRecords || []).map((record) => [record.file_key, record]));
+
+  const missingFileKeys = fileKeys.filter((key) => !fileRecordMap.has(key));
+
+  if (missingFileKeys.length > 0) {
+    const fallbackCandidates = missingFileKeys
+      .filter((key) => key.includes('Readest/Book'))
+      .map((key) => {
+        const parts = key.split('/');
+        if (parts.length === 5) {
+          const bookHash = parts[3]!;
+          const filename = parts[4]!;
+          const fileExtension = filename.split('.').pop() || '';
+          return { originalKey: key, bookHash, fileExtension };
+        }
+        return null;
+      })
+      .filter(Boolean) as Array<{ originalKey: string; bookHash: string; fileExtension: string }>;
+
+    if (fallbackCandidates.length > 0) {
+      const bookHashes = [...new Set(fallbackCandidates.map((c) => c.bookHash))];
+
+      const { data: fallbackRecords, error: fallbackError } = await supabase
+        .from('files')
+        .select('user_id, file_key, book_hash')
+        .eq('user_id', userId)
+        .in('book_hash', bookHashes)
+        .is('deleted_at', null);
+
+      if (!fallbackError && fallbackRecords) {
+        for (const candidate of fallbackCandidates) {
+          const matchedFile = fallbackRecords.find(
+            (f) =>
+              f.book_hash === candidate.bookHash &&
+              f.file_key.endsWith(`.${candidate.fileExtension}`),
+          );
+          if (matchedFile) {
+            fileRecordMap.set(candidate.originalKey, matchedFile);
+          }
+        }
+      }
+    }
+  }
+
+  const results = await Promise.allSettled(
+    fileKeys.map(async (fileKey) => {
+      const fileRecord = fileRecordMap.get(fileKey);
+
+      if (!fileRecord) {
+        return { fileKey, downloadUrl: undefined };
+      }
+
+      if (fileRecord.user_id !== userId) {
+        return { fileKey, downloadUrl: undefined };
+      }
+
+      try {
+        const downloadUrl = await getDownloadSignedUrl(fileRecord.file_key, 1800);
+        return { fileKey, downloadUrl };
+      } catch (error) {
+        console.error(`Error creating signed URL for ${fileKey}:`, error);
+        return { fileKey, downloadUrl: undefined };
+      }
+    }),
+  );
+
+  const downloadUrls: Record<string, string | undefined> = {};
+
+  results.forEach((result, index) => {
+    const fileKey = fileKeys[index]!;
+    if (result.status === 'fulfilled') {
+      downloadUrls[fileKey] = result.value.downloadUrl;
+    } else {
+      downloadUrls[fileKey] = undefined;
+    }
+  });
+
+  return downloadUrls;
+}
@@ -56,7 +56,13 @@ import {
 import { fetch as tauriFetch } from '@tauri-apps/plugin-http';
 import { getOSPlatform, getTargetLang, isCJKEnv, isContentURI, isValidURL } from '@/utils/misc';
 import { deserializeConfig, serializeConfig } from '@/utils/serializer';
-import { downloadFile, uploadFile, deleteFile, createProgressHandler } from '@/libs/storage';
+import {
+  downloadFile,
+  uploadFile,
+  deleteFile,
+  createProgressHandler,
+  batchGetDownloadUrls,
+} from '@/libs/storage';
 import { ClosableFile } from '@/utils/file';
 import { ProgressHandler } from '@/utils/transfer';
 import { TxtToEpubConverter } from '@/utils/txt';
@@ -112,6 +118,10 @@ export abstract class BaseAppService implements AppService {
    return await this.fs.copyFile(srcPath, dstPath, base);
  }

+  async writeFile(path: string, base: BaseDir, content: string | ArrayBuffer | File) {
+    return await this.fs.writeFile(path, base, content);
+  }
+
  async createDir(path: string, base: BaseDir, recursive: boolean = true): Promise<void> {
    return await this.fs.createDir(path, base, recursive);
  }
@@ -498,19 +508,46 @@ export abstract class BaseAppService implements AppService {
    }
  }

-  async downloadCloudFile(lfp: string, cfp: string, handleProgress: ProgressHandler) {
+  async downloadCloudFile(lfp: string, cfp: string, onProgress: ProgressHandler) {
    console.log('Downloading file:', cfp, 'to', lfp);
-    const localFullpath = `${this.localBooksDir}/${lfp}`;
-    const result = await downloadFile(cfp, localFullpath, handleProgress);
-    try {
-      if (this.appPlatform === 'web') {
-        const fileobj = result as Blob;
-        await this.fs.writeFile(lfp, 'Books', await fileobj.arrayBuffer());
-      }
-    } catch {
-      console.log('Failed to download file:', cfp);
-      throw new Error('Failed to download file');
-    }
+    const dstPath = `${this.localBooksDir}/${lfp}`;
+    await downloadFile({ appService: this, cfp, dst: dstPath, onProgress });
+  }
+
+  async downloadBookCovers(books: Book[]): Promise<void> {
+    const booksLfps = new Map(
+      books.map((book) => {
+        const lfp = getCoverFilename(book);
+        return [lfp, book];
+      }),
+    );
+    const filePaths = books.map((book) => ({
+      lfp: getCoverFilename(book),
+      cfp: `${CLOUD_BOOKS_SUBDIR}/${getCoverFilename(book)}`,
+    }));
+    const downloadUrls = await batchGetDownloadUrls(filePaths);
+    await Promise.all(
+      books.map(async (book) => {
+        if (!(await this.fs.exists(getDir(book), 'Books'))) {
+          await this.fs.createDir(getDir(book), 'Books');
+        }
+      }),
+    );
+    await Promise.all(
+      downloadUrls.map(async (file) => {
+        try {
+          const dst = `${this.localBooksDir}/${file.lfp}`;
+          if (!file.downloadUrl) return;
+          await downloadFile({ appService: this, dst, cfp: file.cfp, url: file.downloadUrl });
+          const book = booksLfps.get(file.lfp);
+          if (book && !book.coverDownloadedAt) {
+            book.coverDownloadedAt = Date.now();
+          }
+        } catch (error) {
+          console.log(`Failed to download cover file for book: '${file.lfp}'`, error);
+        }
+      }),
+    );
  }

  async downloadBook(
@@ -4,10 +4,14 @@ import { EnvConfigType, isTauriAppPlatform } from '@/services/environment';

 interface LibraryState {
  library: Book[]; // might contain deleted books
+  isSyncing: boolean;
+  syncProgress: number;
  checkOpenWithBooks: boolean;
  checkLastOpenBooks: boolean;
  currentBookshelf: (Book | BooksGroup)[];
  selectedBooks: Set<string>; // hashes for books, ids for groups
+  setIsSyncing: (syncing: boolean) => void;
+  setSyncProgress: (progress: number) => void;
  setSelectedBooks: (ids: string[]) => void;
  getSelectedBooks: () => string[];
  toggleSelectedBook: (id: string) => void;
@@ -21,10 +25,14 @@ interface LibraryState {

 export const useLibraryStore = create<LibraryState>((set, get) => ({
  library: [],
+  isSyncing: false,
+  syncProgress: 0,
  currentBookshelf: [],
  selectedBooks: new Set(),
  checkOpenWithBooks: isTauriAppPlatform(),
  checkLastOpenBooks: isTauriAppPlatform(),
+  setIsSyncing: (syncing: boolean) => set({ isSyncing: syncing }),
+  setSyncProgress: (progress: number) => set({ syncProgress: progress }),
  getVisibleLibrary: () => get().library.filter((book) => !book.deletedAt),
  setCurrentBookshelf: (bookshelf: (Book | BooksGroup)[]) => {
    set({ currentBookshelf: bookshelf });
@@ -70,6 +70,7 @@ export interface AppService {
  init(): Promise<void>;
  openFile(path: string, base: BaseDir): Promise<File>;
  copyFile(srcPath: string, dstPath: string, base: BaseDir): Promise<void>;
+  writeFile(path: string, base: BaseDir, content: string | ArrayBuffer | File): Promise<void>;
  createDir(path: string, base: BaseDir, recursive?: boolean): Promise<void>;
  deleteFile(path: string, base: BaseDir): Promise<void>;
  deleteDir(path: string, base: BaseDir, recursive?: boolean): Promise<void>;
@@ -104,6 +105,7 @@ export interface AppService {
    redownload?: boolean,
    onProgress?: ProgressHandler,
  ): Promise<void>;
+  downloadBookCovers(books: Book[], redownload?: boolean): Promise<void>;
  isBookAvailable(book: Book): Promise<boolean>;
  getBookFileSize(book: Book): Promise<number | null>;
  loadBookConfig(book: Book, settings: SystemSettings): Promise<BookConfig>;