From e7f0b53bdf739cd8c0f800cf2404e67e7e8b1e2f Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Mon, 6 Jul 2026 02:31:57 +0900 Subject: [PATCH] fix(opds): crawl subdirectories when auto-downloading directory-style catalogs (#4948) Copyparty and other file servers expose each folder as an OPDS feed where subfolders are rel="subsection" navigation entries. Auto-download only followed the catalog's "by newest" feed or the subscribed feed itself, so books in subfolders were never discovered, and a folder containing only subfolders was skipped entirely. When a catalog has no "by newest" feed, treat it as a directory-style listing and crawl its subsection navigation entries breadth-first, bounded by MAX_CRAWL_DEPTH levels, MAX_FEEDS_PER_CRAWL fetches, and the visited set. Library catalogs with a "by newest" feed keep the previous behavior and are never crawled. Facet and structural rels (self, up, start, top, search) are excluded so the crawl cannot escape the subscribed folder. Fixes #4272 Co-authored-by: Claude Fable 5 --- .../services/opds-feed-crawl.test.ts | 377 ++++++++++++++++++ .../src/services/opds/feedChecker.ts | 192 ++++++--- apps/readest-app/src/services/opds/types.ts | 5 + 3 files changed, 510 insertions(+), 64 deletions(-) create mode 100644 apps/readest-app/src/__tests__/services/opds-feed-crawl.test.ts diff --git a/apps/readest-app/src/__tests__/services/opds-feed-crawl.test.ts b/apps/readest-app/src/__tests__/services/opds-feed-crawl.test.ts new file mode 100644 index 00000000..e5056e16 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/opds-feed-crawl.test.ts @@ -0,0 +1,377 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type { OPDSCatalog, OPDSFeed } from '@/types/opds'; +import type { OPDSSubscriptionState } from '@/services/opds/types'; +import { MAX_CRAWL_DEPTH, MAX_FEEDS_PER_CRAWL } from '@/services/opds/types'; +import { checkFeedForNewItems, getSubsectionURLs } from '@/services/opds/feedChecker'; +import { fetchWithAuth } from '@/app/opds/utils/opdsReq'; + +vi.mock('@/services/environment', () => ({ + isWebAppPlatform: vi.fn(() => false), + isTauriAppPlatform: vi.fn(() => true), + getAPIBaseUrl: () => '/api', + getNodeAPIBaseUrl: () => '/node-api', +})); + +vi.mock('@tauri-apps/plugin-http', () => ({ + fetch: vi.fn(), +})); + +vi.mock('@/app/opds/utils/opdsReq', () => ({ + fetchWithAuth: vi.fn(), + needsProxy: vi.fn(() => false), + getProxiedURL: vi.fn((url: string) => url), +})); + +// --- Copyparty-style feed fixtures ----------------------------------------- +// Reproduces https://github.com/readest/readest/issues/4272: copyparty +// exposes each directory as an OPDS feed where subdirectories are +// rel="subsection" navigation entries and files are acquisition entries. +// There is no "by newest" feed and no pagination. + +const dirEntry = (name: string, href: string) => ` + + urn:uuid:dir-${name} + ${name}/ + + 2026-03-28T05:37:03Z + `; + +const bookEntry = (id: string, title: string, href: string) => ` + + ${id} + ${title} + 2025-11-02T17:50:21Z + + `; + +const feedXML = ( + title: string, + body: string, + extraLinks = '', +) => ` + + urn:uuid:feed-${title} + ${title} + ${extraLinks} + ${body} +`; + +const BASE = 'https://files.example.com/books/Kids/?opds'; +const RAMONA_URL = 'https://files.example.com/books/Kids/Ramona/?opds'; + +// URL → XML served by the mocked fetchWithAuth; set per test. +let feeds: Record; + +const makeCatalog = (): OPDSCatalog => ({ id: 'cat-1', name: 'Kids', url: BASE }); + +const emptyState = (): OPDSSubscriptionState => ({ + catalogId: 'cat-1', + lastCheckedAt: 0, + knownEntryIds: [], + failedEntries: [], +}); + +beforeEach(() => { + vi.clearAllMocks(); + feeds = {}; + vi.mocked(fetchWithAuth).mockImplementation(async (url: string) => { + const xml = feeds[url]; + if (!xml) { + return { + ok: false, + url, + status: 404, + statusText: 'Not Found', + text: async () => '', + } as unknown as Response; + } + return { + ok: true, + url, + status: 200, + statusText: 'OK', + text: async () => xml, + } as unknown as Response; + }); +}); + +describe('checkFeedForNewItems directory crawl (#4272)', () => { + it('collects books from subdirectories of a directory-style catalog', async () => { + feeds[BASE] = feedXML( + 'Kids', + dirEntry('Ramona', '/books/Kids/Ramona/?opds') + + bookEntry('urn:uuid:rosie', 'Rosie Revere.epub', '/books/Kids/Rosie%20Revere.epub?dl'), + ); + feeds[RAMONA_URL] = feedXML( + 'Ramona', + bookEntry( + 'urn:uuid:ramona-1', + 'Complete Ramona Collection.epub', + '/books/Kids/Ramona/Complete%20Ramona%20Collection.epub?dl', + ), + ); + + const items = await checkFeedForNewItems(makeCatalog(), emptyState()); + + const ids = items.map((i) => i.entryId); + expect(ids).toContain('urn:uuid:rosie'); + expect(ids).toContain('urn:uuid:ramona-1'); + // The subdirectory book must resolve its href against the sub-feed URL. + const ramonaItem = items.find((i) => i.entryId === 'urn:uuid:ramona-1')!; + expect(ramonaItem.baseURL).toBe(RAMONA_URL); + }); + + it('crawls a folder that contains only subfolders (no top-level books)', async () => { + feeds[BASE] = feedXML( + 'Kids', + dirEntry('Ramona', '/books/Kids/Ramona/?opds') + dirEntry('Beaty', '/books/Kids/Beaty/?opds'), + ); + feeds[RAMONA_URL] = feedXML( + 'Ramona', + bookEntry('urn:uuid:ramona-1', 'Ramona.epub', '/books/Kids/Ramona/Ramona.epub?dl'), + ); + feeds['https://files.example.com/books/Kids/Beaty/?opds'] = feedXML( + 'Beaty', + bookEntry('urn:uuid:beaty-1', 'Beaty.epub', '/books/Kids/Beaty/Beaty.epub?dl'), + ); + + const items = await checkFeedForNewItems(makeCatalog(), emptyState()); + expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:beaty-1', 'urn:uuid:ramona-1']); + }); + + it('collects nested subdirectories (depth 2)', async () => { + const SUB2 = 'https://files.example.com/books/Kids/Ramona/Extras/?opds'; + feeds[BASE] = feedXML('Kids', dirEntry('Ramona', '/books/Kids/Ramona/?opds')); + feeds[RAMONA_URL] = feedXML( + 'Ramona', + dirEntry('Extras', '/books/Kids/Ramona/Extras/?opds') + + bookEntry('urn:uuid:ramona-1', 'Ramona.epub', '/books/Kids/Ramona/Ramona.epub?dl'), + ); + feeds[SUB2] = feedXML( + 'Extras', + bookEntry('urn:uuid:extra-1', 'Extra.epub', '/books/Kids/Ramona/Extras/Extra.epub?dl'), + ); + + const items = await checkFeedForNewItems(makeCatalog(), emptyState()); + expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:extra-1', 'urn:uuid:ramona-1']); + }); + + it('does not crawl navigation when the catalog has a "by newest" feed', async () => { + const NEWEST_URL = 'https://library.example.com/opds/new'; + const BY_AUTHOR_URL = 'https://library.example.com/opds/author'; + const ROOT = 'https://library.example.com/opds'; + + feeds[ROOT] = feedXML( + 'Library', + dirEntry('author', '/opds/author'), + '', + ); + feeds[NEWEST_URL] = feedXML( + 'Newest', + bookEntry('urn:uuid:new-1', 'New Book.epub', '/dl/new-1.epub'), + ); + feeds[BY_AUTHOR_URL] = feedXML( + 'By Author', + bookEntry('urn:uuid:author-1', 'Author Book.epub', '/dl/author-1.epub'), + ); + + const catalog: OPDSCatalog = { id: 'cat-1', name: 'Library', url: ROOT }; + const items = await checkFeedForNewItems(catalog, emptyState()); + + expect(items.map((i) => i.entryId)).toEqual(['urn:uuid:new-1']); + const fetchedURLs = vi.mocked(fetchWithAuth).mock.calls.map((c) => c[0]); + expect(fetchedURLs).not.toContain(BY_AUTHOR_URL); + }); + + it('falls back to root publications without crawling when the newest feed is broken', async () => { + const ROOT = 'https://library.example.com/opds'; + const BY_AUTHOR_URL = 'https://library.example.com/opds/author'; + + // sort/new link points at a 404; root itself has publications + navigation. + feeds[ROOT] = feedXML( + 'Library', + dirEntry('author', '/opds/author') + + bookEntry('urn:uuid:root-1', 'Root Book.epub', '/dl/root-1.epub'), + '', + ); + feeds[BY_AUTHOR_URL] = feedXML( + 'By Author', + bookEntry('urn:uuid:author-1', 'Author Book.epub', '/dl/author-1.epub'), + ); + + const catalog: OPDSCatalog = { id: 'cat-1', name: 'Library', url: ROOT }; + const items = await checkFeedForNewItems(catalog, emptyState()); + + expect(items.map((i) => i.entryId)).toEqual(['urn:uuid:root-1']); + const fetchedURLs = vi.mocked(fetchWithAuth).mock.calls.map((c) => c[0]); + expect(fetchedURLs).not.toContain(BY_AUTHOR_URL); + }); + + it('stops descending at MAX_CRAWL_DEPTH', async () => { + // Chain: root -> d1 -> d2 -> ... each level has one book and one subdir. + const dirURL = (i: number) => `https://files.example.com/books/Kids/${'d/'.repeat(i)}?opds`; + const chainLen = MAX_CRAWL_DEPTH + 2; + feeds[BASE] = feedXML('Kids', dirEntry('d1', dirURL(1))); + for (let i = 1; i <= chainLen; i++) { + feeds[dirURL(i)] = feedXML( + `d${i}`, + bookEntry(`urn:uuid:book-${i}`, `Book ${i}.epub`, `/dl/book-${i}.epub`) + + dirEntry(`d${i + 1}`, dirURL(i + 1)), + ); + } + + const items = await checkFeedForNewItems(makeCatalog(), emptyState()); + const ids = items.map((i) => i.entryId); + expect(ids).toContain(`urn:uuid:book-${MAX_CRAWL_DEPTH}`); + expect(ids).not.toContain(`urn:uuid:book-${MAX_CRAWL_DEPTH + 1}`); + }); + + it('stops fetching after MAX_FEEDS_PER_CRAWL feeds', async () => { + const subCount = MAX_FEEDS_PER_CRAWL + 10; + let rootBody = ''; + for (let i = 0; i < subCount; i++) { + const url = `https://files.example.com/books/Kids/sub${i}/?opds`; + rootBody += dirEntry(`sub${i}`, url); + feeds[url] = feedXML( + `sub${i}`, + bookEntry(`urn:uuid:sub-${i}`, `Sub ${i}.epub`, `/dl/sub-${i}.epub`), + ); + } + feeds[BASE] = feedXML('Kids', rootBody); + + const items = await checkFeedForNewItems(makeCatalog(), emptyState()); + expect(vi.mocked(fetchWithAuth).mock.calls.length).toBeLessThanOrEqual(MAX_FEEDS_PER_CRAWL); + // Root fetch consumes one slot from the budget. + expect(items).toHaveLength(MAX_FEEDS_PER_CRAWL - 1); + }); + + it('does not loop on subdirectories that link back to the root', async () => { + feeds[BASE] = feedXML( + 'Kids', + dirEntry('Ramona', '/books/Kids/Ramona/?opds') + + bookEntry('urn:uuid:rosie', 'Rosie.epub', '/dl/rosie.epub'), + ); + feeds[RAMONA_URL] = feedXML( + 'Ramona', + dirEntry('Kids', '/books/Kids/?opds') + + bookEntry('urn:uuid:ramona-1', 'Ramona.epub', '/dl/ramona.epub'), + ); + + const items = await checkFeedForNewItems(makeCatalog(), emptyState()); + expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:ramona-1', 'urn:uuid:rosie']); + // Root fetched exactly once. + const rootFetches = vi.mocked(fetchWithAuth).mock.calls.filter((c) => c[0] === BASE); + expect(rootFetches).toHaveLength(1); + }); + + it('returns a subdirectory book only once when listed in two feeds', async () => { + feeds[BASE] = feedXML( + 'Kids', + dirEntry('Ramona', '/books/Kids/Ramona/?opds') + + bookEntry('urn:uuid:dup', 'Dup.epub', '/dl/dup.epub'), + ); + feeds[RAMONA_URL] = feedXML('Ramona', bookEntry('urn:uuid:dup', 'Dup.epub', '/dl/dup.epub')); + + const items = await checkFeedForNewItems(makeCatalog(), emptyState()); + expect(items.filter((i) => i.entryId === 'urn:uuid:dup')).toHaveLength(1); + }); + + it('skips already-known entries found in subdirectories', async () => { + feeds[BASE] = feedXML('Kids', dirEntry('Ramona', '/books/Kids/Ramona/?opds')); + feeds[RAMONA_URL] = feedXML( + 'Ramona', + bookEntry('urn:uuid:known', 'Known.epub', '/dl/known.epub') + + bookEntry('urn:uuid:fresh', 'Fresh.epub', '/dl/fresh.epub'), + ); + + const state = { ...emptyState(), knownEntryIds: ['urn:uuid:known'] }; + const items = await checkFeedForNewItems(makeCatalog(), state); + expect(items.map((i) => i.entryId)).toEqual(['urn:uuid:fresh']); + }); + + it('follows rel=next pagination inside crawled subdirectories', async () => { + const PAGE2 = 'https://files.example.com/books/Kids/Ramona/?opds&page=2'; + feeds[BASE] = feedXML('Kids', dirEntry('Ramona', '/books/Kids/Ramona/?opds')); + feeds[RAMONA_URL] = feedXML( + 'Ramona', + bookEntry('urn:uuid:p1', 'Page1.epub', '/dl/p1.epub'), + ``, + ); + feeds[PAGE2] = feedXML('Ramona p2', bookEntry('urn:uuid:p2', 'Page2.epub', '/dl/p2.epub')); + + const items = await checkFeedForNewItems(makeCatalog(), emptyState()); + expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:p1', 'urn:uuid:p2']); + }); +}); + +describe('getSubsectionURLs', () => { + const baseURL = 'https://files.example.com/books/Kids/?opds'; + + it('returns resolved URLs for subsection navigation entries', () => { + const feed: OPDSFeed = { + metadata: { title: 'Kids' }, + links: [], + navigation: [ + { + title: 'Ramona/', + href: '/books/Kids/Ramona/?opds', + rel: 'subsection', + type: 'application/atom+xml;profile=opds-catalog', + properties: {}, + }, + ], + }; + expect(getSubsectionURLs(feed, baseURL)).toEqual([RAMONA_URL]); + }); + + it('accepts navigation entries without a type', () => { + const feed: OPDSFeed = { + metadata: { title: 'Kids' }, + links: [], + navigation: [{ title: 'Sub', href: 'Sub/?opds', properties: {} }], + }; + expect(getSubsectionURLs(feed, baseURL)).toEqual([ + 'https://files.example.com/books/Kids/Sub/?opds', + ]); + }); + + it('skips entries with non-catalog types', () => { + const feed: OPDSFeed = { + metadata: { title: 'Kids' }, + links: [], + navigation: [{ title: 'Readme', href: '/readme.html', type: 'text/html', properties: {} }], + }; + expect(getSubsectionURLs(feed, baseURL)).toEqual([]); + }); + + it('skips facet, self, up, search and start rels', () => { + const catalogType = 'application/atom+xml;profile=opds-catalog'; + const feed: OPDSFeed = { + metadata: { title: 'Kids' }, + links: [], + navigation: [ + { + title: 'F', + href: '/f', + rel: 'http://opds-spec.org/facet', + type: catalogType, + properties: {}, + }, + { title: 'S', href: '/s', rel: 'self', type: catalogType, properties: {} }, + { title: 'U', href: '/u', rel: 'up', type: catalogType, properties: {} }, + { title: 'Q', href: '/q', rel: 'search', type: catalogType, properties: {} }, + { title: 'T', href: '/t', rel: 'start', type: catalogType, properties: {} }, + ], + }; + expect(getSubsectionURLs(feed, baseURL)).toEqual([]); + }); + + it('returns empty for feeds without navigation', () => { + const feed: OPDSFeed = { metadata: { title: 'Kids' }, links: [] }; + expect(getSubsectionURLs(feed, baseURL)).toEqual([]); + }); +}); diff --git a/apps/readest-app/src/services/opds/feedChecker.ts b/apps/readest-app/src/services/opds/feedChecker.ts index 516f683c..0f8c326b 100644 --- a/apps/readest-app/src/services/opds/feedChecker.ts +++ b/apps/readest-app/src/services/opds/feedChecker.ts @@ -21,7 +21,7 @@ import { } from '@/app/opds/utils/opdsUtils'; import { normalizeOPDSCustomHeaders } from '@/app/opds/utils/customHeaders'; import type { OPDSSubscriptionState, PendingItem } from './types'; -import { MAX_PAGES_PER_FEED } from './types'; +import { MAX_CRAWL_DEPTH, MAX_FEEDS_PER_CRAWL, MAX_PAGES_PER_FEED } from './types'; const SORT_NEW_REL = 'http://opds-spec.org/sort/new'; @@ -268,9 +268,10 @@ function hasRel(item: LinkLike, target: string): boolean { /** * Find the catalog's "by newest" feed URL — the one that lists publications - * in reverse-chronological order. Auto-download follows only this feed (plus - * its rel=next pages); we deliberately don't crawl the rest of the navigation - * tree because subscribing to a whole catalog is rarely what the user wants. + * in reverse-chronological order. For library catalogs auto-download follows + * only this feed (plus its rel=next pages); we deliberately don't crawl the + * rest of the navigation tree because subscribing to a whole library catalog + * is rarely what the user wants. * * Detection order: * 1. Authoritative: any link or navigation entry with @@ -280,8 +281,8 @@ function hasRel(item: LinkLike, target: string): boolean { * 3. Href heuristics: ?sort_order=release_date (Project Gutenberg), * /new-releases, /recently-added, ?sort=new, etc. * - * Returns undefined when no candidate matches — the caller should treat the - * catalog as not auto-download-capable rather than fall back to a deep crawl. + * Returns undefined when no candidate matches — the caller then treats the + * catalog as a directory-style feed and crawls its subsections instead. */ export function findNewestFeedURL(feed: OPDSFeed, baseURL: string): string | undefined { const candidates: LinkLike[] = [...(feed.links ?? []), ...(feed.navigation ?? [])]; @@ -312,43 +313,110 @@ function feedHasContent(feed: OPDSFeed): boolean { return false; } +// Rels that must not be treated as subdirectories when crawling a +// directory-style catalog: facets and structural links (self/up/start/…) +// would re-list the same feed under another sort order or escape the +// subscribed folder into its parent. +const CRAWL_SKIP_RELS = [REL.FACET, 'self', 'start', 'up', 'top', 'search']; + /** - * Resolve a catalog URL down to its acquisition feed: - * - If the root already contains publications, use it as-is. - * - Otherwise look for a "by newest" link and follow it. - * - * Returns null when no acquisition feed can be found — auto-download will - * skip the catalog rather than crawl through every navigation branch. + * Collect sub-catalog URLs from a feed's navigation entries — the + * subdirectories of a directory-style catalog (copyparty and other file + * servers expose folders as rel="subsection" entries). Entries with a + * non-catalog media type or a facet/structural rel are skipped; a missing + * type is accepted since many servers omit it on navigation entries. */ -async function resolveAcquisitionFeed( - url: string, - username: string, - password: string, - customHeaders: Record, - visited: Set, -): Promise<{ feed: OPDSFeed; baseURL: string } | null> { - visited.add(url); - const root = await fetchFeed(url, username, password, customHeaders); - if (!root) return null; - - const newestURL = findNewestFeedURL(root.feed, root.baseURL); - if (newestURL && !visited.has(newestURL)) { - visited.add(newestURL); - const newest = await fetchFeed(newestURL, username, password, customHeaders); - if (newest && feedHasContent(newest.feed)) return newest; +export function getSubsectionURLs(feed: OPDSFeed, baseURL: string): string[] { + const urls: string[] = []; + for (const item of feed.navigation ?? []) { + if (!item.href) continue; + const rels = Array.isArray(item.rel) ? item.rel : [item.rel ?? '']; + if (rels.some((rel) => CRAWL_SKIP_RELS.includes(rel))) continue; + if (item.type && !isOPDSCatalog(item.type)) continue; + urls.push(resolveURL(item.href, baseURL)); } + return urls; +} - if (feedHasContent(root.feed)) return root; - return null; +interface CrawlContext { + catalog: OPDSCatalog; + knownIds: Set; + username: string; + password: string; + customHeaders: Record; + visited: Set; + /** Follow subsection navigation entries (directory-style catalogs). */ + crawlNav: boolean; } /** - * Check a catalog's "by newest" feed for new items. + * Walk feeds breadth-first from an already-fetched start feed, collecting + * new PendingItems. Every feed's rel=next chain is followed up to + * MAX_PAGES_PER_FEED pages. When ctx.crawlNav is set, subsection navigation + * entries are followed too, at most MAX_CRAWL_DEPTH levels below the start + * feed and MAX_FEEDS_PER_CRAWL fetches in total. + */ +async function crawlFeeds( + start: { feed: OPDSFeed; baseURL: string }, + ctx: CrawlContext, +): Promise { + const items: PendingItem[] = []; + const queue: Array<{ url: string; depth: number; page: number }> = []; + + const processFeed = (feed: OPDSFeed, baseURL: string, depth: number, page: number) => { + const newItems = collectNewEntries(feed, ctx.knownIds, baseURL); + // Mark as known so a book listed by several crawled feeds is only + // collected once. + for (const item of newItems) ctx.knownIds.add(item.entryId); + items.push(...newItems); + + const nextHref = getNextPageUrl(feed); + if (nextHref && page < MAX_PAGES_PER_FEED) { + const nextURL = resolveURL(nextHref, baseURL); + if (!ctx.visited.has(nextURL)) { + ctx.visited.add(nextURL); + queue.push({ url: nextURL, depth, page: page + 1 }); + } + } + + if (ctx.crawlNav && depth < MAX_CRAWL_DEPTH) { + for (const subURL of getSubsectionURLs(feed, baseURL)) { + if (ctx.visited.has(subURL)) continue; + ctx.visited.add(subURL); + queue.push({ url: subURL, depth: depth + 1, page: 1 }); + } + } + }; + + processFeed(start.feed, start.baseURL, 0, 1); + + let fetches = 1; // the start feed was already fetched by the caller + while (queue.length > 0 && fetches < MAX_FEEDS_PER_CRAWL) { + const node = queue.shift()!; + const next = await fetchFeed(node.url, ctx.username, ctx.password, ctx.customHeaders); + fetches++; + if (!next) continue; + processFeed(next.feed, next.baseURL, node.depth, node.page); + } + if (queue.length > 0) { + console.warn( + `OPDS sync: catalog "${ctx.catalog.name}" crawl budget exhausted; ${queue.length} sub-feed(s) skipped`, + ); + } + + return items; +} + +/** + * Check a catalog for new items. Pure discovery — no downloads, no state + * mutations. * - * Pure discovery — no downloads, no state mutations. Resolves the catalog - * URL to its newest-acquisition feed (see resolveAcquisitionFeed) and walks - * up to MAX_PAGES_PER_FEED of rel=next pagination, collecting entries that - * aren't already in knownEntryIds. + * Library catalogs (those exposing a "by newest" feed, see + * findNewestFeedURL) are checked through that feed and its rel=next pages + * only. Catalogs without one are directory-style listings (e.g. copyparty): + * the subscribed URL itself is the acquisition feed and its subsection + * navigation entries are subdirectories, which are crawled breadth-first so + * books in subfolders are downloaded too (#4272). */ export async function checkFeedForNewItems( catalog: OPDSCatalog, @@ -358,42 +426,38 @@ export async function checkFeedForNewItems( const customHeaders = normalizeOPDSCustomHeaders(catalog.customHeaders); const username = catalog.username ?? ''; const password = catalog.password ?? ''; - const visited = new Set(); + const visited = new Set([catalog.url]); - const acquisition = await resolveAcquisitionFeed( - catalog.url, + const root = await fetchFeed(catalog.url, username, password, customHeaders); + if (!root) return []; + + const ctx: CrawlContext = { + catalog, + knownIds, username, password, customHeaders, visited, - ); - if (!acquisition) { + crawlNav: false, + }; + + const newestURL = findNewestFeedURL(root.feed, root.baseURL); + if (newestURL) { + if (!visited.has(newestURL)) { + visited.add(newestURL); + const newest = await fetchFeed(newestURL, username, password, customHeaders); + if (newest && feedHasContent(newest.feed)) return crawlFeeds(newest, ctx); + } + // Broken or empty "by newest" feed: fall back to the root feed's own + // publications, still without crawling navigation. + return crawlFeeds(root, ctx); + } + + if (!feedHasContent(root.feed) && getSubsectionURLs(root.feed, root.baseURL).length === 0) { console.warn( - `OPDS sync: catalog "${catalog.name}" has no recognizable "by newest" feed; skipping`, + `OPDS sync: catalog "${catalog.name}" has no publications or subdirectories; skipping`, ); return []; } - - const items: PendingItem[] = []; - let { feed, baseURL } = acquisition; - items.push(...collectNewEntries(feed, knownIds, baseURL)); - - let pageCount = 1; - while (pageCount < MAX_PAGES_PER_FEED) { - const nextHref = getNextPageUrl(feed); - if (!nextHref) break; - const nextUrl = resolveURL(nextHref, baseURL); - if (visited.has(nextUrl)) break; - visited.add(nextUrl); - - const next = await fetchFeed(nextUrl, username, password, customHeaders); - if (!next) break; - - items.push(...collectNewEntries(next.feed, knownIds, next.baseURL)); - feed = next.feed; - baseURL = next.baseURL; - pageCount++; - } - - return items; + return crawlFeeds(root, { ...ctx, crawlNav: true }); } diff --git a/apps/readest-app/src/services/opds/types.ts b/apps/readest-app/src/services/opds/types.ts index ca97261b..3905131a 100644 --- a/apps/readest-app/src/services/opds/types.ts +++ b/apps/readest-app/src/services/opds/types.ts @@ -3,6 +3,11 @@ import type { Book } from '@/types/book'; // --- Constants --- export const MAX_PAGES_PER_FEED = 5; +// Directory-style catalogs (e.g. copyparty file listings) expose subfolders +// as rel="subsection" navigation entries. When a catalog has no "by newest" +// feed those subsections are crawled breadth-first, bounded by these caps. +export const MAX_CRAWL_DEPTH = 5; +export const MAX_FEEDS_PER_CRAWL = 50; export const MAX_KNOWN_ENTRIES = 2000; export const MAX_RETRY_ATTEMPTS = 3; export const RETRY_BACKOFF_MS = 60_000;