diff --git a/apps/readest-app/src/__tests__/services/opds-feed-crawl.test.ts b/apps/readest-app/src/__tests__/services/opds-feed-crawl.test.ts
new file mode 100644
index 00000000..e5056e16
--- /dev/null
+++ b/apps/readest-app/src/__tests__/services/opds-feed-crawl.test.ts
@@ -0,0 +1,377 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import type { OPDSCatalog, OPDSFeed } from '@/types/opds';
+import type { OPDSSubscriptionState } from '@/services/opds/types';
+import { MAX_CRAWL_DEPTH, MAX_FEEDS_PER_CRAWL } from '@/services/opds/types';
+import { checkFeedForNewItems, getSubsectionURLs } from '@/services/opds/feedChecker';
+import { fetchWithAuth } from '@/app/opds/utils/opdsReq';
+
+vi.mock('@/services/environment', () => ({
+ isWebAppPlatform: vi.fn(() => false),
+ isTauriAppPlatform: vi.fn(() => true),
+ getAPIBaseUrl: () => '/api',
+ getNodeAPIBaseUrl: () => '/node-api',
+}));
+
+vi.mock('@tauri-apps/plugin-http', () => ({
+ fetch: vi.fn(),
+}));
+
+vi.mock('@/app/opds/utils/opdsReq', () => ({
+ fetchWithAuth: vi.fn(),
+ needsProxy: vi.fn(() => false),
+ getProxiedURL: vi.fn((url: string) => url),
+}));
+
+// --- Copyparty-style feed fixtures -----------------------------------------
+// Reproduces https://github.com/readest/readest/issues/4272: copyparty
+// exposes each directory as an OPDS feed where subdirectories are
+// rel="subsection" navigation entries and files are acquisition entries.
+// There is no "by newest" feed and no pagination.
+
+const dirEntry = (name: string, href: string) => `
+
+ urn:uuid:dir-${name}
+ ${name}/
+
+ 2026-03-28T05:37:03Z
+ `;
+
+const bookEntry = (id: string, title: string, href: string) => `
+
+ ${id}
+ ${title}
+ 2025-11-02T17:50:21Z
+
+ `;
+
+const feedXML = (
+ title: string,
+ body: string,
+ extraLinks = '',
+) => `
+
+ urn:uuid:feed-${title}
+ ${title}
+ ${extraLinks}
+ ${body}
+`;
+
+const BASE = 'https://files.example.com/books/Kids/?opds';
+const RAMONA_URL = 'https://files.example.com/books/Kids/Ramona/?opds';
+
+// URL → XML served by the mocked fetchWithAuth; set per test.
+let feeds: Record;
+
+const makeCatalog = (): OPDSCatalog => ({ id: 'cat-1', name: 'Kids', url: BASE });
+
+const emptyState = (): OPDSSubscriptionState => ({
+ catalogId: 'cat-1',
+ lastCheckedAt: 0,
+ knownEntryIds: [],
+ failedEntries: [],
+});
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ feeds = {};
+ vi.mocked(fetchWithAuth).mockImplementation(async (url: string) => {
+ const xml = feeds[url];
+ if (!xml) {
+ return {
+ ok: false,
+ url,
+ status: 404,
+ statusText: 'Not Found',
+ text: async () => '',
+ } as unknown as Response;
+ }
+ return {
+ ok: true,
+ url,
+ status: 200,
+ statusText: 'OK',
+ text: async () => xml,
+ } as unknown as Response;
+ });
+});
+
+describe('checkFeedForNewItems directory crawl (#4272)', () => {
+ it('collects books from subdirectories of a directory-style catalog', async () => {
+ feeds[BASE] = feedXML(
+ 'Kids',
+ dirEntry('Ramona', '/books/Kids/Ramona/?opds') +
+ bookEntry('urn:uuid:rosie', 'Rosie Revere.epub', '/books/Kids/Rosie%20Revere.epub?dl'),
+ );
+ feeds[RAMONA_URL] = feedXML(
+ 'Ramona',
+ bookEntry(
+ 'urn:uuid:ramona-1',
+ 'Complete Ramona Collection.epub',
+ '/books/Kids/Ramona/Complete%20Ramona%20Collection.epub?dl',
+ ),
+ );
+
+ const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+
+ const ids = items.map((i) => i.entryId);
+ expect(ids).toContain('urn:uuid:rosie');
+ expect(ids).toContain('urn:uuid:ramona-1');
+ // The subdirectory book must resolve its href against the sub-feed URL.
+ const ramonaItem = items.find((i) => i.entryId === 'urn:uuid:ramona-1')!;
+ expect(ramonaItem.baseURL).toBe(RAMONA_URL);
+ });
+
+ it('crawls a folder that contains only subfolders (no top-level books)', async () => {
+ feeds[BASE] = feedXML(
+ 'Kids',
+ dirEntry('Ramona', '/books/Kids/Ramona/?opds') + dirEntry('Beaty', '/books/Kids/Beaty/?opds'),
+ );
+ feeds[RAMONA_URL] = feedXML(
+ 'Ramona',
+ bookEntry('urn:uuid:ramona-1', 'Ramona.epub', '/books/Kids/Ramona/Ramona.epub?dl'),
+ );
+ feeds['https://files.example.com/books/Kids/Beaty/?opds'] = feedXML(
+ 'Beaty',
+ bookEntry('urn:uuid:beaty-1', 'Beaty.epub', '/books/Kids/Beaty/Beaty.epub?dl'),
+ );
+
+ const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+ expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:beaty-1', 'urn:uuid:ramona-1']);
+ });
+
+ it('collects nested subdirectories (depth 2)', async () => {
+ const SUB2 = 'https://files.example.com/books/Kids/Ramona/Extras/?opds';
+ feeds[BASE] = feedXML('Kids', dirEntry('Ramona', '/books/Kids/Ramona/?opds'));
+ feeds[RAMONA_URL] = feedXML(
+ 'Ramona',
+ dirEntry('Extras', '/books/Kids/Ramona/Extras/?opds') +
+ bookEntry('urn:uuid:ramona-1', 'Ramona.epub', '/books/Kids/Ramona/Ramona.epub?dl'),
+ );
+ feeds[SUB2] = feedXML(
+ 'Extras',
+ bookEntry('urn:uuid:extra-1', 'Extra.epub', '/books/Kids/Ramona/Extras/Extra.epub?dl'),
+ );
+
+ const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+ expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:extra-1', 'urn:uuid:ramona-1']);
+ });
+
+ it('does not crawl navigation when the catalog has a "by newest" feed', async () => {
+ const NEWEST_URL = 'https://library.example.com/opds/new';
+ const BY_AUTHOR_URL = 'https://library.example.com/opds/author';
+ const ROOT = 'https://library.example.com/opds';
+
+ feeds[ROOT] = feedXML(
+ 'Library',
+ dirEntry('author', '/opds/author'),
+ '',
+ );
+ feeds[NEWEST_URL] = feedXML(
+ 'Newest',
+ bookEntry('urn:uuid:new-1', 'New Book.epub', '/dl/new-1.epub'),
+ );
+ feeds[BY_AUTHOR_URL] = feedXML(
+ 'By Author',
+ bookEntry('urn:uuid:author-1', 'Author Book.epub', '/dl/author-1.epub'),
+ );
+
+ const catalog: OPDSCatalog = { id: 'cat-1', name: 'Library', url: ROOT };
+ const items = await checkFeedForNewItems(catalog, emptyState());
+
+ expect(items.map((i) => i.entryId)).toEqual(['urn:uuid:new-1']);
+ const fetchedURLs = vi.mocked(fetchWithAuth).mock.calls.map((c) => c[0]);
+ expect(fetchedURLs).not.toContain(BY_AUTHOR_URL);
+ });
+
+ it('falls back to root publications without crawling when the newest feed is broken', async () => {
+ const ROOT = 'https://library.example.com/opds';
+ const BY_AUTHOR_URL = 'https://library.example.com/opds/author';
+
+ // sort/new link points at a 404; root itself has publications + navigation.
+ feeds[ROOT] = feedXML(
+ 'Library',
+ dirEntry('author', '/opds/author') +
+ bookEntry('urn:uuid:root-1', 'Root Book.epub', '/dl/root-1.epub'),
+ '',
+ );
+ feeds[BY_AUTHOR_URL] = feedXML(
+ 'By Author',
+ bookEntry('urn:uuid:author-1', 'Author Book.epub', '/dl/author-1.epub'),
+ );
+
+ const catalog: OPDSCatalog = { id: 'cat-1', name: 'Library', url: ROOT };
+ const items = await checkFeedForNewItems(catalog, emptyState());
+
+ expect(items.map((i) => i.entryId)).toEqual(['urn:uuid:root-1']);
+ const fetchedURLs = vi.mocked(fetchWithAuth).mock.calls.map((c) => c[0]);
+ expect(fetchedURLs).not.toContain(BY_AUTHOR_URL);
+ });
+
+ it('stops descending at MAX_CRAWL_DEPTH', async () => {
+ // Chain: root -> d1 -> d2 -> ... each level has one book and one subdir.
+ const dirURL = (i: number) => `https://files.example.com/books/Kids/${'d/'.repeat(i)}?opds`;
+ const chainLen = MAX_CRAWL_DEPTH + 2;
+ feeds[BASE] = feedXML('Kids', dirEntry('d1', dirURL(1)));
+ for (let i = 1; i <= chainLen; i++) {
+ feeds[dirURL(i)] = feedXML(
+ `d${i}`,
+ bookEntry(`urn:uuid:book-${i}`, `Book ${i}.epub`, `/dl/book-${i}.epub`) +
+ dirEntry(`d${i + 1}`, dirURL(i + 1)),
+ );
+ }
+
+ const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+ const ids = items.map((i) => i.entryId);
+ expect(ids).toContain(`urn:uuid:book-${MAX_CRAWL_DEPTH}`);
+ expect(ids).not.toContain(`urn:uuid:book-${MAX_CRAWL_DEPTH + 1}`);
+ });
+
+ it('stops fetching after MAX_FEEDS_PER_CRAWL feeds', async () => {
+ const subCount = MAX_FEEDS_PER_CRAWL + 10;
+ let rootBody = '';
+ for (let i = 0; i < subCount; i++) {
+ const url = `https://files.example.com/books/Kids/sub${i}/?opds`;
+ rootBody += dirEntry(`sub${i}`, url);
+ feeds[url] = feedXML(
+ `sub${i}`,
+ bookEntry(`urn:uuid:sub-${i}`, `Sub ${i}.epub`, `/dl/sub-${i}.epub`),
+ );
+ }
+ feeds[BASE] = feedXML('Kids', rootBody);
+
+ const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+ expect(vi.mocked(fetchWithAuth).mock.calls.length).toBeLessThanOrEqual(MAX_FEEDS_PER_CRAWL);
+ // Root fetch consumes one slot from the budget.
+ expect(items).toHaveLength(MAX_FEEDS_PER_CRAWL - 1);
+ });
+
+ it('does not loop on subdirectories that link back to the root', async () => {
+ feeds[BASE] = feedXML(
+ 'Kids',
+ dirEntry('Ramona', '/books/Kids/Ramona/?opds') +
+ bookEntry('urn:uuid:rosie', 'Rosie.epub', '/dl/rosie.epub'),
+ );
+ feeds[RAMONA_URL] = feedXML(
+ 'Ramona',
+ dirEntry('Kids', '/books/Kids/?opds') +
+ bookEntry('urn:uuid:ramona-1', 'Ramona.epub', '/dl/ramona.epub'),
+ );
+
+ const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+ expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:ramona-1', 'urn:uuid:rosie']);
+ // Root fetched exactly once.
+ const rootFetches = vi.mocked(fetchWithAuth).mock.calls.filter((c) => c[0] === BASE);
+ expect(rootFetches).toHaveLength(1);
+ });
+
+ it('returns a subdirectory book only once when listed in two feeds', async () => {
+ feeds[BASE] = feedXML(
+ 'Kids',
+ dirEntry('Ramona', '/books/Kids/Ramona/?opds') +
+ bookEntry('urn:uuid:dup', 'Dup.epub', '/dl/dup.epub'),
+ );
+ feeds[RAMONA_URL] = feedXML('Ramona', bookEntry('urn:uuid:dup', 'Dup.epub', '/dl/dup.epub'));
+
+ const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+ expect(items.filter((i) => i.entryId === 'urn:uuid:dup')).toHaveLength(1);
+ });
+
+ it('skips already-known entries found in subdirectories', async () => {
+ feeds[BASE] = feedXML('Kids', dirEntry('Ramona', '/books/Kids/Ramona/?opds'));
+ feeds[RAMONA_URL] = feedXML(
+ 'Ramona',
+ bookEntry('urn:uuid:known', 'Known.epub', '/dl/known.epub') +
+ bookEntry('urn:uuid:fresh', 'Fresh.epub', '/dl/fresh.epub'),
+ );
+
+ const state = { ...emptyState(), knownEntryIds: ['urn:uuid:known'] };
+ const items = await checkFeedForNewItems(makeCatalog(), state);
+ expect(items.map((i) => i.entryId)).toEqual(['urn:uuid:fresh']);
+ });
+
+ it('follows rel=next pagination inside crawled subdirectories', async () => {
+ const PAGE2 = 'https://files.example.com/books/Kids/Ramona/?opds&page=2';
+ feeds[BASE] = feedXML('Kids', dirEntry('Ramona', '/books/Kids/Ramona/?opds'));
+ feeds[RAMONA_URL] = feedXML(
+ 'Ramona',
+ bookEntry('urn:uuid:p1', 'Page1.epub', '/dl/p1.epub'),
+ ``,
+ );
+ feeds[PAGE2] = feedXML('Ramona p2', bookEntry('urn:uuid:p2', 'Page2.epub', '/dl/p2.epub'));
+
+ const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+ expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:p1', 'urn:uuid:p2']);
+ });
+});
+
+describe('getSubsectionURLs', () => {
+ const baseURL = 'https://files.example.com/books/Kids/?opds';
+
+ it('returns resolved URLs for subsection navigation entries', () => {
+ const feed: OPDSFeed = {
+ metadata: { title: 'Kids' },
+ links: [],
+ navigation: [
+ {
+ title: 'Ramona/',
+ href: '/books/Kids/Ramona/?opds',
+ rel: 'subsection',
+ type: 'application/atom+xml;profile=opds-catalog',
+ properties: {},
+ },
+ ],
+ };
+ expect(getSubsectionURLs(feed, baseURL)).toEqual([RAMONA_URL]);
+ });
+
+ it('accepts navigation entries without a type', () => {
+ const feed: OPDSFeed = {
+ metadata: { title: 'Kids' },
+ links: [],
+ navigation: [{ title: 'Sub', href: 'Sub/?opds', properties: {} }],
+ };
+ expect(getSubsectionURLs(feed, baseURL)).toEqual([
+ 'https://files.example.com/books/Kids/Sub/?opds',
+ ]);
+ });
+
+ it('skips entries with non-catalog types', () => {
+ const feed: OPDSFeed = {
+ metadata: { title: 'Kids' },
+ links: [],
+ navigation: [{ title: 'Readme', href: '/readme.html', type: 'text/html', properties: {} }],
+ };
+ expect(getSubsectionURLs(feed, baseURL)).toEqual([]);
+ });
+
+ it('skips facet, self, up, search and start rels', () => {
+ const catalogType = 'application/atom+xml;profile=opds-catalog';
+ const feed: OPDSFeed = {
+ metadata: { title: 'Kids' },
+ links: [],
+ navigation: [
+ {
+ title: 'F',
+ href: '/f',
+ rel: 'http://opds-spec.org/facet',
+ type: catalogType,
+ properties: {},
+ },
+ { title: 'S', href: '/s', rel: 'self', type: catalogType, properties: {} },
+ { title: 'U', href: '/u', rel: 'up', type: catalogType, properties: {} },
+ { title: 'Q', href: '/q', rel: 'search', type: catalogType, properties: {} },
+ { title: 'T', href: '/t', rel: 'start', type: catalogType, properties: {} },
+ ],
+ };
+ expect(getSubsectionURLs(feed, baseURL)).toEqual([]);
+ });
+
+ it('returns empty for feeds without navigation', () => {
+ const feed: OPDSFeed = { metadata: { title: 'Kids' }, links: [] };
+ expect(getSubsectionURLs(feed, baseURL)).toEqual([]);
+ });
+});
diff --git a/apps/readest-app/src/services/opds/feedChecker.ts b/apps/readest-app/src/services/opds/feedChecker.ts
index 516f683c..0f8c326b 100644
--- a/apps/readest-app/src/services/opds/feedChecker.ts
+++ b/apps/readest-app/src/services/opds/feedChecker.ts
@@ -21,7 +21,7 @@ import {
} from '@/app/opds/utils/opdsUtils';
import { normalizeOPDSCustomHeaders } from '@/app/opds/utils/customHeaders';
import type { OPDSSubscriptionState, PendingItem } from './types';
-import { MAX_PAGES_PER_FEED } from './types';
+import { MAX_CRAWL_DEPTH, MAX_FEEDS_PER_CRAWL, MAX_PAGES_PER_FEED } from './types';
const SORT_NEW_REL = 'http://opds-spec.org/sort/new';
@@ -268,9 +268,10 @@ function hasRel(item: LinkLike, target: string): boolean {
/**
* Find the catalog's "by newest" feed URL — the one that lists publications
- * in reverse-chronological order. Auto-download follows only this feed (plus
- * its rel=next pages); we deliberately don't crawl the rest of the navigation
- * tree because subscribing to a whole catalog is rarely what the user wants.
+ * in reverse-chronological order. For library catalogs auto-download follows
+ * only this feed (plus its rel=next pages); we deliberately don't crawl the
+ * rest of the navigation tree because subscribing to a whole library catalog
+ * is rarely what the user wants.
*
* Detection order:
* 1. Authoritative: any link or navigation entry with
@@ -280,8 +281,8 @@ function hasRel(item: LinkLike, target: string): boolean {
* 3. Href heuristics: ?sort_order=release_date (Project Gutenberg),
* /new-releases, /recently-added, ?sort=new, etc.
*
- * Returns undefined when no candidate matches — the caller should treat the
- * catalog as not auto-download-capable rather than fall back to a deep crawl.
+ * Returns undefined when no candidate matches — the caller then treats the
+ * catalog as a directory-style feed and crawls its subsections instead.
*/
export function findNewestFeedURL(feed: OPDSFeed, baseURL: string): string | undefined {
const candidates: LinkLike[] = [...(feed.links ?? []), ...(feed.navigation ?? [])];
@@ -312,43 +313,110 @@ function feedHasContent(feed: OPDSFeed): boolean {
return false;
}
+// Rels that must not be treated as subdirectories when crawling a
+// directory-style catalog: facets and structural links (self/up/start/…)
+// would re-list the same feed under another sort order or escape the
+// subscribed folder into its parent.
+const CRAWL_SKIP_RELS = [REL.FACET, 'self', 'start', 'up', 'top', 'search'];
+
/**
- * Resolve a catalog URL down to its acquisition feed:
- * - If the root already contains publications, use it as-is.
- * - Otherwise look for a "by newest" link and follow it.
- *
- * Returns null when no acquisition feed can be found — auto-download will
- * skip the catalog rather than crawl through every navigation branch.
+ * Collect sub-catalog URLs from a feed's navigation entries — the
+ * subdirectories of a directory-style catalog (copyparty and other file
+ * servers expose folders as rel="subsection" entries). Entries with a
+ * non-catalog media type or a facet/structural rel are skipped; a missing
+ * type is accepted since many servers omit it on navigation entries.
*/
-async function resolveAcquisitionFeed(
- url: string,
- username: string,
- password: string,
- customHeaders: Record,
- visited: Set,
-): Promise<{ feed: OPDSFeed; baseURL: string } | null> {
- visited.add(url);
- const root = await fetchFeed(url, username, password, customHeaders);
- if (!root) return null;
-
- const newestURL = findNewestFeedURL(root.feed, root.baseURL);
- if (newestURL && !visited.has(newestURL)) {
- visited.add(newestURL);
- const newest = await fetchFeed(newestURL, username, password, customHeaders);
- if (newest && feedHasContent(newest.feed)) return newest;
+export function getSubsectionURLs(feed: OPDSFeed, baseURL: string): string[] {
+ const urls: string[] = [];
+ for (const item of feed.navigation ?? []) {
+ if (!item.href) continue;
+ const rels = Array.isArray(item.rel) ? item.rel : [item.rel ?? ''];
+ if (rels.some((rel) => CRAWL_SKIP_RELS.includes(rel))) continue;
+ if (item.type && !isOPDSCatalog(item.type)) continue;
+ urls.push(resolveURL(item.href, baseURL));
}
+ return urls;
+}
- if (feedHasContent(root.feed)) return root;
- return null;
+interface CrawlContext {
+ catalog: OPDSCatalog;
+ knownIds: Set;
+ username: string;
+ password: string;
+ customHeaders: Record;
+ visited: Set;
+ /** Follow subsection navigation entries (directory-style catalogs). */
+ crawlNav: boolean;
}
/**
- * Check a catalog's "by newest" feed for new items.
+ * Walk feeds breadth-first from an already-fetched start feed, collecting
+ * new PendingItems. Every feed's rel=next chain is followed up to
+ * MAX_PAGES_PER_FEED pages. When ctx.crawlNav is set, subsection navigation
+ * entries are followed too, at most MAX_CRAWL_DEPTH levels below the start
+ * feed and MAX_FEEDS_PER_CRAWL fetches in total.
+ */
+async function crawlFeeds(
+ start: { feed: OPDSFeed; baseURL: string },
+ ctx: CrawlContext,
+): Promise {
+ const items: PendingItem[] = [];
+ const queue: Array<{ url: string; depth: number; page: number }> = [];
+
+ const processFeed = (feed: OPDSFeed, baseURL: string, depth: number, page: number) => {
+ const newItems = collectNewEntries(feed, ctx.knownIds, baseURL);
+ // Mark as known so a book listed by several crawled feeds is only
+ // collected once.
+ for (const item of newItems) ctx.knownIds.add(item.entryId);
+ items.push(...newItems);
+
+ const nextHref = getNextPageUrl(feed);
+ if (nextHref && page < MAX_PAGES_PER_FEED) {
+ const nextURL = resolveURL(nextHref, baseURL);
+ if (!ctx.visited.has(nextURL)) {
+ ctx.visited.add(nextURL);
+ queue.push({ url: nextURL, depth, page: page + 1 });
+ }
+ }
+
+ if (ctx.crawlNav && depth < MAX_CRAWL_DEPTH) {
+ for (const subURL of getSubsectionURLs(feed, baseURL)) {
+ if (ctx.visited.has(subURL)) continue;
+ ctx.visited.add(subURL);
+ queue.push({ url: subURL, depth: depth + 1, page: 1 });
+ }
+ }
+ };
+
+ processFeed(start.feed, start.baseURL, 0, 1);
+
+ let fetches = 1; // the start feed was already fetched by the caller
+ while (queue.length > 0 && fetches < MAX_FEEDS_PER_CRAWL) {
+ const node = queue.shift()!;
+ const next = await fetchFeed(node.url, ctx.username, ctx.password, ctx.customHeaders);
+ fetches++;
+ if (!next) continue;
+ processFeed(next.feed, next.baseURL, node.depth, node.page);
+ }
+ if (queue.length > 0) {
+ console.warn(
+ `OPDS sync: catalog "${ctx.catalog.name}" crawl budget exhausted; ${queue.length} sub-feed(s) skipped`,
+ );
+ }
+
+ return items;
+}
+
+/**
+ * Check a catalog for new items. Pure discovery — no downloads, no state
+ * mutations.
*
- * Pure discovery — no downloads, no state mutations. Resolves the catalog
- * URL to its newest-acquisition feed (see resolveAcquisitionFeed) and walks
- * up to MAX_PAGES_PER_FEED of rel=next pagination, collecting entries that
- * aren't already in knownEntryIds.
+ * Library catalogs (those exposing a "by newest" feed, see
+ * findNewestFeedURL) are checked through that feed and its rel=next pages
+ * only. Catalogs without one are directory-style listings (e.g. copyparty):
+ * the subscribed URL itself is the acquisition feed and its subsection
+ * navigation entries are subdirectories, which are crawled breadth-first so
+ * books in subfolders are downloaded too (#4272).
*/
export async function checkFeedForNewItems(
catalog: OPDSCatalog,
@@ -358,42 +426,38 @@ export async function checkFeedForNewItems(
const customHeaders = normalizeOPDSCustomHeaders(catalog.customHeaders);
const username = catalog.username ?? '';
const password = catalog.password ?? '';
- const visited = new Set();
+ const visited = new Set([catalog.url]);
- const acquisition = await resolveAcquisitionFeed(
- catalog.url,
+ const root = await fetchFeed(catalog.url, username, password, customHeaders);
+ if (!root) return [];
+
+ const ctx: CrawlContext = {
+ catalog,
+ knownIds,
username,
password,
customHeaders,
visited,
- );
- if (!acquisition) {
+ crawlNav: false,
+ };
+
+ const newestURL = findNewestFeedURL(root.feed, root.baseURL);
+ if (newestURL) {
+ if (!visited.has(newestURL)) {
+ visited.add(newestURL);
+ const newest = await fetchFeed(newestURL, username, password, customHeaders);
+ if (newest && feedHasContent(newest.feed)) return crawlFeeds(newest, ctx);
+ }
+ // Broken or empty "by newest" feed: fall back to the root feed's own
+ // publications, still without crawling navigation.
+ return crawlFeeds(root, ctx);
+ }
+
+ if (!feedHasContent(root.feed) && getSubsectionURLs(root.feed, root.baseURL).length === 0) {
console.warn(
- `OPDS sync: catalog "${catalog.name}" has no recognizable "by newest" feed; skipping`,
+ `OPDS sync: catalog "${catalog.name}" has no publications or subdirectories; skipping`,
);
return [];
}
-
- const items: PendingItem[] = [];
- let { feed, baseURL } = acquisition;
- items.push(...collectNewEntries(feed, knownIds, baseURL));
-
- let pageCount = 1;
- while (pageCount < MAX_PAGES_PER_FEED) {
- const nextHref = getNextPageUrl(feed);
- if (!nextHref) break;
- const nextUrl = resolveURL(nextHref, baseURL);
- if (visited.has(nextUrl)) break;
- visited.add(nextUrl);
-
- const next = await fetchFeed(nextUrl, username, password, customHeaders);
- if (!next) break;
-
- items.push(...collectNewEntries(next.feed, knownIds, next.baseURL));
- feed = next.feed;
- baseURL = next.baseURL;
- pageCount++;
- }
-
- return items;
+ return crawlFeeds(root, { ...ctx, crawlNav: true });
}
diff --git a/apps/readest-app/src/services/opds/types.ts b/apps/readest-app/src/services/opds/types.ts
index ca97261b..3905131a 100644
--- a/apps/readest-app/src/services/opds/types.ts
+++ b/apps/readest-app/src/services/opds/types.ts
@@ -3,6 +3,11 @@ import type { Book } from '@/types/book';
// --- Constants ---
export const MAX_PAGES_PER_FEED = 5;
+// Directory-style catalogs (e.g. copyparty file listings) expose subfolders
+// as rel="subsection" navigation entries. When a catalog has no "by newest"
+// feed those subsections are crawled breadth-first, bounded by these caps.
+export const MAX_CRAWL_DEPTH = 5;
+export const MAX_FEEDS_PER_CRAWL = 50;
export const MAX_KNOWN_ENTRIES = 2000;
export const MAX_RETRY_ATTEMPTS = 3;
export const RETRY_BACKOFF_MS = 60_000;