diff --git a/apps/readest-app/src/app/library/hooks/useBooksSync.ts b/apps/readest-app/src/app/library/hooks/useBooksSync.ts index a29efd97..ee742fda 100644 --- a/apps/readest-app/src/app/library/hooks/useBooksSync.ts +++ b/apps/readest-app/src/app/library/hooks/useBooksSync.ts @@ -2,73 +2,56 @@ import { useCallback, useEffect } from 'react'; import { useAuth } from '@/context/AuthContext'; import { useEnv } from '@/context/EnvContext'; import { useSync } from '@/hooks/useSync'; -import { useLibraryStore } from '@/store/libraryStore'; import { Book } from '@/types/book'; -import { debounce } from '@/utils/debounce'; -import { SYNC_BOOKS_INTERVAL_SEC } from '@/services/constants'; +import { useLibraryStore } from '@/store/libraryStore'; -export interface UseBooksSyncProps { - onSyncStart?: () => void; - onSyncEnd?: () => void; -} - -export const useBooksSync = ({ onSyncStart, onSyncEnd }: UseBooksSyncProps) => { +export const useBooksSync = () => { const { user } = useAuth(); const { appService } = useEnv(); - const { library, setLibrary } = useLibraryStore(); + const { library, isSyncing, setLibrary, setIsSyncing, setSyncProgress } = useLibraryStore(); const { syncedBooks, syncBooks, lastSyncedAtBooks } = useSync(); - const pullLibrary = useCallback(async () => { - if (!user) return; - await syncBooks([], 'pull'); - }, [user, syncBooks]); - - const pushLibrary = async () => { - if (!user) return; - const newBooks = getNewBooks(); - syncBooks(newBooks, 'push'); - }; - - useEffect(() => { - if (!user) return; - pullLibrary(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [user]); - - const getNewBooks = () => { + const getNewBooks = useCallback(() => { if (!user) return []; const library = useLibraryStore.getState().library; const newBooks = library.filter( (book) => lastSyncedAtBooks < book.updatedAt || lastSyncedAtBooks < (book.deletedAt ?? 0), ); return newBooks; - }; + }, [user, lastSyncedAtBooks]); - // eslint-disable-next-line react-hooks/exhaustive-deps - const handleAutoSync = useCallback( - debounce(() => { - const newBooks = getNewBooks(); - syncBooks(newBooks, 'both'); - }, SYNC_BOOKS_INTERVAL_SEC * 1000), - [syncBooks], - ); + const pullLibrary = useCallback(async () => { + if (!user) return; + await syncBooks([], 'pull'); + }, [user, syncBooks]); + + const pushLibrary = useCallback(async () => { + if (!user) return; + const newBooks = getNewBooks(); + await syncBooks(newBooks, 'push'); + }, [user, syncBooks, getNewBooks]); useEffect(() => { if (!user) return; - handleAutoSync(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [library, handleAutoSync]); + pullLibrary(); + }, [user, pullLibrary]); const updateLibrary = async () => { + if (isSyncing) return; if (!syncedBooks?.length) return; + // Process old books first so that when we update the library the order is preserved syncedBooks.sort((a, b) => a.updatedAt - b.updatedAt); + const bookHashesInSynced = new Set(syncedBooks.map((book) => book.hash)); + const oldBooks = library.filter((book) => bookHashesInSynced.has(book.hash)); + const oldBooksNeedsDownload = oldBooks.filter((book) => { + return !book.deletedAt && book.uploadedAt && !book.coverDownloadedAt; + }); const processOldBook = async (oldBook: Book) => { const matchingBook = syncedBooks.find((newBook) => newBook.hash === oldBook.hash); if (matchingBook) { if (!matchingBook.deletedAt && matchingBook.uploadedAt && !oldBook.coverDownloadedAt) { - await appService?.downloadBook(oldBook, true); oldBook.coverImageUrl = await appService?.generateCoverImageUrl(oldBook); } const mergedBook = @@ -80,31 +63,44 @@ export const useBooksSync = ({ onSyncStart, onSyncEnd }: UseBooksSyncProps) => { return oldBook; }; - const updatedLibrary = await Promise.all(library.map(processOldBook)); - const processNewBook = async (newBook: Book) => { - if (!updatedLibrary.some((oldBook) => oldBook.hash === newBook.hash)) { - if (newBook.uploadedAt && !newBook.deletedAt) { - try { - await appService?.downloadBook(newBook, true); - } catch { - console.error('Failed to download book:', newBook); - } finally { - newBook.coverImageUrl = await appService?.generateCoverImageUrl(newBook); - updatedLibrary.push(newBook); - setLibrary(updatedLibrary); - } - } - } - }; - onSyncStart?.(); - const batchSize = 3; - for (let i = 0; i < syncedBooks.length; i += batchSize) { - const batch = syncedBooks.slice(i, i + batchSize); - await Promise.all(batch.map(processNewBook)); + const oldBooksBatchSize = 20; + for (let i = 0; i < oldBooksNeedsDownload.length; i += oldBooksBatchSize) { + const batch = oldBooksNeedsDownload.slice(i, i + oldBooksBatchSize); + await appService?.downloadBookCovers(batch); } - onSyncEnd?.(); + + const updatedLibrary = await Promise.all(library.map(processOldBook)); + const bookHashesInLibrary = new Set(updatedLibrary.map((book) => book.hash)); + const newBooks = syncedBooks.filter( + (newBook) => + !bookHashesInLibrary.has(newBook.hash) && newBook.uploadedAt && !newBook.deletedAt, + ); + + const processNewBook = async (newBook: Book) => { + newBook.coverImageUrl = await appService?.generateCoverImageUrl(newBook); + updatedLibrary.push(newBook); + }; + + if (newBooks.length > 0) { + setIsSyncing(true); + } + + const batchSize = 10; + for (let i = 0; i < newBooks.length; i += batchSize) { + const batch = newBooks.slice(i, i + batchSize); + await appService?.downloadBookCovers(batch); + await Promise.all(batch.map(processNewBook)); + const progress = Math.min((i + batchSize) / newBooks.length, 1); + setLibrary([...updatedLibrary]); + setSyncProgress(progress); + } + setLibrary(updatedLibrary); appService?.saveLibraryBooks(updatedLibrary); + + if (newBooks.length > 0) { + setIsSyncing(false); + } }; useEffect(() => { diff --git a/apps/readest-app/src/app/library/page.tsx b/apps/readest-app/src/app/library/page.tsx index 8c614ddb..cb160f7d 100644 --- a/apps/readest-app/src/app/library/page.tsx +++ b/apps/readest-app/src/app/library/page.tsx @@ -68,6 +68,8 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP const { token, user } = useAuth(); const { library: libraryBooks, + isSyncing, + syncProgress, updateBook, setLibrary, checkOpenWithBooks, @@ -103,10 +105,7 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP useOpenWithBooks(); - const { pullLibrary, pushLibrary } = useBooksSync({ - onSyncStart: () => setLoading(true), - onSyncEnd: () => setLoading(false), - }); + const { pullLibrary, pushLibrary } = useBooksSync(); usePullToRefresh(containerRef, pullLibrary); useScreenWakeLock(settings.screenWakeLock); @@ -692,7 +691,7 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP onDeselectAll={handleDeselectAll} /> - {loading && ( + {(loading || isSyncing) && (
@@ -723,6 +722,14 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP paddingLeft: `${insets.left}px`, }} > + { +export const batchGetDownloadUrls = async (files: { lfp: string; cfp: string }[]) => { + try { + const userId = await getUserID(); + if (!userId) { + throw new Error('Not authenticated'); + } + const filePaths = files.map((file) => file.cfp); + const fileKeys = filePaths.map((path) => `${userId}/${path}`); + const response = await fetchWithAuth(`${API_ENDPOINTS.download}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ fileKeys }), + }); + + const { downloadUrls } = await response.json(); + return files.map((file) => { + const fileKey = `${userId}/${file.cfp}`; + return { + lfp: file.lfp, + cfp: file.cfp, + downloadUrl: downloadUrls[fileKey], + }; + }); + } catch (error) { + console.error('Batch get download URLs failed:', error); + throw new Error('Batch get download URLs failed'); + } +}; + +type DownloadFileParams = { + appService: AppService; + dst: string; + cfp: string; + url?: string; + onProgress?: ProgressHandler; +}; + +export const downloadFile = async ({ + appService, + dst, + cfp, + url, + onProgress, +}: DownloadFileParams) => { try { const userId = await getUserID(); if (!userId) { throw new Error('Not authenticated'); } - const fileKey = `${userId}/${filePath}`; - const response = await fetchWithAuth( - `${API_ENDPOINTS.download}?fileKey=${encodeURIComponent(fileKey)}`, - { - method: 'GET', - }, - ); + let downloadUrl = url; + if (!downloadUrl) { + const fileKey = `${userId}/${cfp}`; + const response = await fetchWithAuth( + `${API_ENDPOINTS.download}?fileKey=${encodeURIComponent(fileKey)}`, + { + method: 'GET', + }, + ); - const { downloadUrl } = await response.json(); + const { downloadUrl: url } = await response.json(); + downloadUrl = url; + } + + if (!downloadUrl) { + throw new Error('No download URL available'); + } if (isWebAppPlatform()) { - return await webDownload(downloadUrl, onProgress); + const file = await webDownload(downloadUrl, onProgress); + await appService.writeFile(dst, 'None', await file.arrayBuffer()); } else { - await tauriDownload(downloadUrl, fileFullPath, onProgress); - return; + await tauriDownload(downloadUrl, dst, onProgress); } } catch (error) { - console.error(`File '${filePath}' download failed:`, error); + console.error(`File '${dst}' download failed:`, error); throw new Error('File download failed'); } }; diff --git a/apps/readest-app/src/pages/api/storage/download.ts b/apps/readest-app/src/pages/api/storage/download.ts index 00d04ee2..e0ecbc7f 100644 --- a/apps/readest-app/src/pages/api/storage/download.ts +++ b/apps/readest-app/src/pages/api/storage/download.ts @@ -7,7 +7,7 @@ import { validateUserAndToken } from '@/utils/access'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { await runMiddleware(req, res, corsAllMethods); - if (req.method !== 'GET') { + if (req.method !== 'GET' && req.method !== 'POST') { return res.status(405).json({ error: 'Method not allowed' }); } @@ -17,70 +17,141 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) return res.status(403).json({ error: 'Not authenticated' }); } - const { fileKey } = req.query; - - if (!fileKey || typeof fileKey !== 'string') { - return res.status(400).json({ error: 'Missing or invalid fileKey' }); - } - - // Verify the file belongs to the user - const supabase = createSupabaseAdminClient(); - const result = await supabase - .from('files') - .select('user_id, file_key') - .eq('user_id', user.id) - .eq('file_key', fileKey) // index idx_files_file_key_deleted_at on public.files - .is('deleted_at', null) - .limit(1) - .single(); - - const { error: fileError } = result; - let { data: fileRecord } = result; - - if (fileError || !fileRecord) { - // Fallback for corrupted file names, using book hash and file extension to match fileKey - if (fileKey.includes('Readest/Book')) { - const parts = fileKey.split('/'); - if (parts.length === 5) { - const bookHash = parts[3]!; - const filename = parts[4]!; - const fileExtension = filename.split('.').pop() || ''; - - const { data: fileRecords, error: fileError } = await supabase - .from('files') - .select('user_id, file_key') - .eq('user_id', user.id) - .eq('book_hash', bookHash) - .is('deleted_at', null); - - if (!fileError && fileRecords && fileRecords.length > 0) { - const matchedFile = fileRecords.find((f) => f.file_key.endsWith(`.${fileExtension}`)); - if (matchedFile) { - fileRecord = matchedFile; - } - } else { - return res.status(404).json({ error: 'File not found' }); - } - } + if (req.method === 'GET') { + const { fileKey } = req.query; + if (!fileKey || typeof fileKey !== 'string') { + return res.status(400).json({ error: 'Missing or invalid fileKey' }); } + + const downloadUrlsMap = await processFileKeys([fileKey], user.id); + const downloadUrl = downloadUrlsMap[fileKey]; + + if (!downloadUrl) { + return res.status(404).json({ error: 'File not found' }); + } + + return res.status(200).json({ downloadUrl }); } - if (fileRecord?.user_id !== user.id) { - return res.status(403).json({ error: 'Unauthorized access to the file' }); - } + if (req.method === 'POST') { + const { fileKeys } = req.body; - try { - const downloadUrl = await getDownloadSignedUrl(fileRecord.file_key, 1800); + if (!fileKeys || !Array.isArray(fileKeys)) { + return res.status(400).json({ error: 'Missing or invalid fileKeys array' }); + } - res.status(200).json({ - downloadUrl, - }); - } catch (error) { - console.error('Error creating signed URL for download:', error); - res.status(500).json({ error: 'Could not create signed URL for download' }); + if (fileKeys.length === 0) { + return res.status(400).json({ error: 'fileKeys array cannot be empty' }); + } + + if (!fileKeys.every((key) => typeof key === 'string')) { + return res.status(400).json({ error: 'All fileKeys must be strings' }); + } + + const downloadUrls = await processFileKeys(fileKeys, user.id); + + return res.status(200).json({ downloadUrls }); } } catch (error) { console.error(error); return res.status(500).json({ error: 'Something went wrong' }); } } + +async function processFileKeys( + fileKeys: string[], + userId: string, +): Promise> { + const supabase = createSupabaseAdminClient(); + + const { data: fileRecords, error: fileError } = await supabase + .from('files') + .select('user_id, file_key, book_hash') + .eq('user_id', userId) + .in('file_key', fileKeys) + .is('deleted_at', null); + + if (fileError) { + console.error('Error querying files:', fileError); + return Object.fromEntries(fileKeys.map((key) => [key, undefined])); + } + + const fileRecordMap = new Map((fileRecords || []).map((record) => [record.file_key, record])); + + const missingFileKeys = fileKeys.filter((key) => !fileRecordMap.has(key)); + + if (missingFileKeys.length > 0) { + const fallbackCandidates = missingFileKeys + .filter((key) => key.includes('Readest/Book')) + .map((key) => { + const parts = key.split('/'); + if (parts.length === 5) { + const bookHash = parts[3]!; + const filename = parts[4]!; + const fileExtension = filename.split('.').pop() || ''; + return { originalKey: key, bookHash, fileExtension }; + } + return null; + }) + .filter(Boolean) as Array<{ originalKey: string; bookHash: string; fileExtension: string }>; + + if (fallbackCandidates.length > 0) { + const bookHashes = [...new Set(fallbackCandidates.map((c) => c.bookHash))]; + + const { data: fallbackRecords, error: fallbackError } = await supabase + .from('files') + .select('user_id, file_key, book_hash') + .eq('user_id', userId) + .in('book_hash', bookHashes) + .is('deleted_at', null); + + if (!fallbackError && fallbackRecords) { + for (const candidate of fallbackCandidates) { + const matchedFile = fallbackRecords.find( + (f) => + f.book_hash === candidate.bookHash && + f.file_key.endsWith(`.${candidate.fileExtension}`), + ); + if (matchedFile) { + fileRecordMap.set(candidate.originalKey, matchedFile); + } + } + } + } + } + + const results = await Promise.allSettled( + fileKeys.map(async (fileKey) => { + const fileRecord = fileRecordMap.get(fileKey); + + if (!fileRecord) { + return { fileKey, downloadUrl: undefined }; + } + + if (fileRecord.user_id !== userId) { + return { fileKey, downloadUrl: undefined }; + } + + try { + const downloadUrl = await getDownloadSignedUrl(fileRecord.file_key, 1800); + return { fileKey, downloadUrl }; + } catch (error) { + console.error(`Error creating signed URL for ${fileKey}:`, error); + return { fileKey, downloadUrl: undefined }; + } + }), + ); + + const downloadUrls: Record = {}; + + results.forEach((result, index) => { + const fileKey = fileKeys[index]!; + if (result.status === 'fulfilled') { + downloadUrls[fileKey] = result.value.downloadUrl; + } else { + downloadUrls[fileKey] = undefined; + } + }); + + return downloadUrls; +} diff --git a/apps/readest-app/src/services/appService.ts b/apps/readest-app/src/services/appService.ts index c829fdbf..014df3c0 100644 --- a/apps/readest-app/src/services/appService.ts +++ b/apps/readest-app/src/services/appService.ts @@ -56,7 +56,13 @@ import { import { fetch as tauriFetch } from '@tauri-apps/plugin-http'; import { getOSPlatform, getTargetLang, isCJKEnv, isContentURI, isValidURL } from '@/utils/misc'; import { deserializeConfig, serializeConfig } from '@/utils/serializer'; -import { downloadFile, uploadFile, deleteFile, createProgressHandler } from '@/libs/storage'; +import { + downloadFile, + uploadFile, + deleteFile, + createProgressHandler, + batchGetDownloadUrls, +} from '@/libs/storage'; import { ClosableFile } from '@/utils/file'; import { ProgressHandler } from '@/utils/transfer'; import { TxtToEpubConverter } from '@/utils/txt'; @@ -112,6 +118,10 @@ export abstract class BaseAppService implements AppService { return await this.fs.copyFile(srcPath, dstPath, base); } + async writeFile(path: string, base: BaseDir, content: string | ArrayBuffer | File) { + return await this.fs.writeFile(path, base, content); + } + async createDir(path: string, base: BaseDir, recursive: boolean = true): Promise { return await this.fs.createDir(path, base, recursive); } @@ -498,19 +508,46 @@ export abstract class BaseAppService implements AppService { } } - async downloadCloudFile(lfp: string, cfp: string, handleProgress: ProgressHandler) { + async downloadCloudFile(lfp: string, cfp: string, onProgress: ProgressHandler) { console.log('Downloading file:', cfp, 'to', lfp); - const localFullpath = `${this.localBooksDir}/${lfp}`; - const result = await downloadFile(cfp, localFullpath, handleProgress); - try { - if (this.appPlatform === 'web') { - const fileobj = result as Blob; - await this.fs.writeFile(lfp, 'Books', await fileobj.arrayBuffer()); - } - } catch { - console.log('Failed to download file:', cfp); - throw new Error('Failed to download file'); - } + const dstPath = `${this.localBooksDir}/${lfp}`; + await downloadFile({ appService: this, cfp, dst: dstPath, onProgress }); + } + + async downloadBookCovers(books: Book[]): Promise { + const booksLfps = new Map( + books.map((book) => { + const lfp = getCoverFilename(book); + return [lfp, book]; + }), + ); + const filePaths = books.map((book) => ({ + lfp: getCoverFilename(book), + cfp: `${CLOUD_BOOKS_SUBDIR}/${getCoverFilename(book)}`, + })); + const downloadUrls = await batchGetDownloadUrls(filePaths); + await Promise.all( + books.map(async (book) => { + if (!(await this.fs.exists(getDir(book), 'Books'))) { + await this.fs.createDir(getDir(book), 'Books'); + } + }), + ); + await Promise.all( + downloadUrls.map(async (file) => { + try { + const dst = `${this.localBooksDir}/${file.lfp}`; + if (!file.downloadUrl) return; + await downloadFile({ appService: this, dst, cfp: file.cfp, url: file.downloadUrl }); + const book = booksLfps.get(file.lfp); + if (book && !book.coverDownloadedAt) { + book.coverDownloadedAt = Date.now(); + } + } catch (error) { + console.log(`Failed to download cover file for book: '${file.lfp}'`, error); + } + }), + ); } async downloadBook( diff --git a/apps/readest-app/src/store/libraryStore.ts b/apps/readest-app/src/store/libraryStore.ts index 3fbf7263..4612f263 100644 --- a/apps/readest-app/src/store/libraryStore.ts +++ b/apps/readest-app/src/store/libraryStore.ts @@ -4,10 +4,14 @@ import { EnvConfigType, isTauriAppPlatform } from '@/services/environment'; interface LibraryState { library: Book[]; // might contain deleted books + isSyncing: boolean; + syncProgress: number; checkOpenWithBooks: boolean; checkLastOpenBooks: boolean; currentBookshelf: (Book | BooksGroup)[]; selectedBooks: Set; // hashes for books, ids for groups + setIsSyncing: (syncing: boolean) => void; + setSyncProgress: (progress: number) => void; setSelectedBooks: (ids: string[]) => void; getSelectedBooks: () => string[]; toggleSelectedBook: (id: string) => void; @@ -21,10 +25,14 @@ interface LibraryState { export const useLibraryStore = create((set, get) => ({ library: [], + isSyncing: false, + syncProgress: 0, currentBookshelf: [], selectedBooks: new Set(), checkOpenWithBooks: isTauriAppPlatform(), checkLastOpenBooks: isTauriAppPlatform(), + setIsSyncing: (syncing: boolean) => set({ isSyncing: syncing }), + setSyncProgress: (progress: number) => set({ syncProgress: progress }), getVisibleLibrary: () => get().library.filter((book) => !book.deletedAt), setCurrentBookshelf: (bookshelf: (Book | BooksGroup)[]) => { set({ currentBookshelf: bookshelf }); diff --git a/apps/readest-app/src/types/system.ts b/apps/readest-app/src/types/system.ts index e6b34336..fa8dff1a 100644 --- a/apps/readest-app/src/types/system.ts +++ b/apps/readest-app/src/types/system.ts @@ -70,6 +70,7 @@ export interface AppService { init(): Promise; openFile(path: string, base: BaseDir): Promise; copyFile(srcPath: string, dstPath: string, base: BaseDir): Promise; + writeFile(path: string, base: BaseDir, content: string | ArrayBuffer | File): Promise; createDir(path: string, base: BaseDir, recursive?: boolean): Promise; deleteFile(path: string, base: BaseDir): Promise; deleteDir(path: string, base: BaseDir, recursive?: boolean): Promise; @@ -104,6 +105,7 @@ export interface AppService { redownload?: boolean, onProgress?: ProgressHandler, ): Promise; + downloadBookCovers(books: Book[], redownload?: boolean): Promise; isBookAvailable(book: Book): Promise; getBookFileSize(book: Book): Promise; loadBookConfig(book: Book, settings: SystemSettings): Promise;