fix(opds): crawl subdirectories when auto-downloading directory-style catalogs (#4948)

Copyparty and other file servers expose each folder as an OPDS feed where
subfolders are rel="subsection" navigation entries. Auto-download only
followed the catalog's "by newest" feed or the subscribed feed itself, so
books in subfolders were never discovered, and a folder containing only
subfolders was skipped entirely.

When a catalog has no "by newest" feed, treat it as a directory-style
listing and crawl its subsection navigation entries breadth-first, bounded
by MAX_CRAWL_DEPTH levels, MAX_FEEDS_PER_CRAWL fetches, and the visited
set. Library catalogs with a "by newest" feed keep the previous behavior
and are never crawled. Facet and structural rels (self, up, start, top,
search) are excluded so the crawl cannot escape the subscribed folder.

Fixes #4272

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

apps/readest-app/src/__tests__/services/opds-feed-crawl.test.ts

@@ -0,0 +1,377 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import type { OPDSCatalog, OPDSFeed } from '@/types/opds';
+import type { OPDSSubscriptionState } from '@/services/opds/types';
+import { MAX_CRAWL_DEPTH, MAX_FEEDS_PER_CRAWL } from '@/services/opds/types';
+import { checkFeedForNewItems, getSubsectionURLs } from '@/services/opds/feedChecker';
+import { fetchWithAuth } from '@/app/opds/utils/opdsReq';
+
+vi.mock('@/services/environment', () => ({
+  isWebAppPlatform: vi.fn(() => false),
+  isTauriAppPlatform: vi.fn(() => true),
+  getAPIBaseUrl: () => '/api',
+  getNodeAPIBaseUrl: () => '/node-api',
+}));
+
+vi.mock('@tauri-apps/plugin-http', () => ({
+  fetch: vi.fn(),
+}));
+
+vi.mock('@/app/opds/utils/opdsReq', () => ({
+  fetchWithAuth: vi.fn(),
+  needsProxy: vi.fn(() => false),
+  getProxiedURL: vi.fn((url: string) => url),
+}));
+
+// --- Copyparty-style feed fixtures -----------------------------------------
+// Reproduces https://github.com/readest/readest/issues/4272: copyparty
+// exposes each directory as an OPDS feed where subdirectories are
+// rel="subsection" navigation entries and files are acquisition entries.
+// There is no "by newest" feed and no pagination.
+
+const dirEntry = (name: string, href: string) => `
+  <entry>
+    <id>urn:uuid:dir-${name}</id>
+    <title>${name}/</title>
+    <link rel="subsection"
+      href="${href}"
+      type="application/atom+xml;profile=opds-catalog"/>
+    <updated>2026-03-28T05:37:03Z</updated>
+  </entry>`;
+
+const bookEntry = (id: string, title: string, href: string) => `
+  <entry>
+    <id>${id}</id>
+    <title>${title}</title>
+    <updated>2025-11-02T17:50:21Z</updated>
+    <link rel="http://opds-spec.org/acquisition"
+      href="${href}"
+      type="application/epub+zip"/>
+  </entry>`;
+
+const feedXML = (
+  title: string,
+  body: string,
+  extraLinks = '',
+) => `<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <id>urn:uuid:feed-${title}</id>
+  <title>${title}</title>
+  ${extraLinks}
+  ${body}
+</feed>`;
+
+const BASE = 'https://files.example.com/books/Kids/?opds';
+const RAMONA_URL = 'https://files.example.com/books/Kids/Ramona/?opds';
+
+// URL → XML served by the mocked fetchWithAuth; set per test.
+let feeds: Record<string, string>;
+
+const makeCatalog = (): OPDSCatalog => ({ id: 'cat-1', name: 'Kids', url: BASE });
+
+const emptyState = (): OPDSSubscriptionState => ({
+  catalogId: 'cat-1',
+  lastCheckedAt: 0,
+  knownEntryIds: [],
+  failedEntries: [],
+});
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  feeds = {};
+  vi.mocked(fetchWithAuth).mockImplementation(async (url: string) => {
+    const xml = feeds[url];
+    if (!xml) {
+      return {
+        ok: false,
+        url,
+        status: 404,
+        statusText: 'Not Found',
+        text: async () => '',
+      } as unknown as Response;
+    }
+    return {
+      ok: true,
+      url,
+      status: 200,
+      statusText: 'OK',
+      text: async () => xml,
+    } as unknown as Response;
+  });
+});
+
+describe('checkFeedForNewItems directory crawl (#4272)', () => {
+  it('collects books from subdirectories of a directory-style catalog', async () => {
+    feeds[BASE] = feedXML(
+      'Kids',
+      dirEntry('Ramona', '/books/Kids/Ramona/?opds') +
+        bookEntry('urn:uuid:rosie', 'Rosie Revere.epub', '/books/Kids/Rosie%20Revere.epub?dl'),
+    );
+    feeds[RAMONA_URL] = feedXML(
+      'Ramona',
+      bookEntry(
+        'urn:uuid:ramona-1',
+        'Complete Ramona Collection.epub',
+        '/books/Kids/Ramona/Complete%20Ramona%20Collection.epub?dl',
+      ),
+    );
+
+    const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+
+    const ids = items.map((i) => i.entryId);
+    expect(ids).toContain('urn:uuid:rosie');
+    expect(ids).toContain('urn:uuid:ramona-1');
+    // The subdirectory book must resolve its href against the sub-feed URL.
+    const ramonaItem = items.find((i) => i.entryId === 'urn:uuid:ramona-1')!;
+    expect(ramonaItem.baseURL).toBe(RAMONA_URL);
+  });
+
+  it('crawls a folder that contains only subfolders (no top-level books)', async () => {
+    feeds[BASE] = feedXML(
+      'Kids',
+      dirEntry('Ramona', '/books/Kids/Ramona/?opds') + dirEntry('Beaty', '/books/Kids/Beaty/?opds'),
+    );
+    feeds[RAMONA_URL] = feedXML(
+      'Ramona',
+      bookEntry('urn:uuid:ramona-1', 'Ramona.epub', '/books/Kids/Ramona/Ramona.epub?dl'),
+    );
+    feeds['https://files.example.com/books/Kids/Beaty/?opds'] = feedXML(
+      'Beaty',
+      bookEntry('urn:uuid:beaty-1', 'Beaty.epub', '/books/Kids/Beaty/Beaty.epub?dl'),
+    );
+
+    const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+    expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:beaty-1', 'urn:uuid:ramona-1']);
+  });
+
+  it('collects nested subdirectories (depth 2)', async () => {
+    const SUB2 = 'https://files.example.com/books/Kids/Ramona/Extras/?opds';
+    feeds[BASE] = feedXML('Kids', dirEntry('Ramona', '/books/Kids/Ramona/?opds'));
+    feeds[RAMONA_URL] = feedXML(
+      'Ramona',
+      dirEntry('Extras', '/books/Kids/Ramona/Extras/?opds') +
+        bookEntry('urn:uuid:ramona-1', 'Ramona.epub', '/books/Kids/Ramona/Ramona.epub?dl'),
+    );
+    feeds[SUB2] = feedXML(
+      'Extras',
+      bookEntry('urn:uuid:extra-1', 'Extra.epub', '/books/Kids/Ramona/Extras/Extra.epub?dl'),
+    );
+
+    const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+    expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:extra-1', 'urn:uuid:ramona-1']);
+  });
+
+  it('does not crawl navigation when the catalog has a "by newest" feed', async () => {
+    const NEWEST_URL = 'https://library.example.com/opds/new';
+    const BY_AUTHOR_URL = 'https://library.example.com/opds/author';
+    const ROOT = 'https://library.example.com/opds';
+
+    feeds[ROOT] = feedXML(
+      'Library',
+      dirEntry('author', '/opds/author'),
+      '<link rel="http://opds-spec.org/sort/new" href="/opds/new" type="application/atom+xml;profile=opds-catalog"/>',
+    );
+    feeds[NEWEST_URL] = feedXML(
+      'Newest',
+      bookEntry('urn:uuid:new-1', 'New Book.epub', '/dl/new-1.epub'),
+    );
+    feeds[BY_AUTHOR_URL] = feedXML(
+      'By Author',
+      bookEntry('urn:uuid:author-1', 'Author Book.epub', '/dl/author-1.epub'),
+    );
+
+    const catalog: OPDSCatalog = { id: 'cat-1', name: 'Library', url: ROOT };
+    const items = await checkFeedForNewItems(catalog, emptyState());
+
+    expect(items.map((i) => i.entryId)).toEqual(['urn:uuid:new-1']);
+    const fetchedURLs = vi.mocked(fetchWithAuth).mock.calls.map((c) => c[0]);
+    expect(fetchedURLs).not.toContain(BY_AUTHOR_URL);
+  });
+
+  it('falls back to root publications without crawling when the newest feed is broken', async () => {
+    const ROOT = 'https://library.example.com/opds';
+    const BY_AUTHOR_URL = 'https://library.example.com/opds/author';
+
+    // sort/new link points at a 404; root itself has publications + navigation.
+    feeds[ROOT] = feedXML(
+      'Library',
+      dirEntry('author', '/opds/author') +
+        bookEntry('urn:uuid:root-1', 'Root Book.epub', '/dl/root-1.epub'),
+      '<link rel="http://opds-spec.org/sort/new" href="/opds/new" type="application/atom+xml;profile=opds-catalog"/>',
+    );
+    feeds[BY_AUTHOR_URL] = feedXML(
+      'By Author',
+      bookEntry('urn:uuid:author-1', 'Author Book.epub', '/dl/author-1.epub'),
+    );
+
+    const catalog: OPDSCatalog = { id: 'cat-1', name: 'Library', url: ROOT };
+    const items = await checkFeedForNewItems(catalog, emptyState());
+
+    expect(items.map((i) => i.entryId)).toEqual(['urn:uuid:root-1']);
+    const fetchedURLs = vi.mocked(fetchWithAuth).mock.calls.map((c) => c[0]);
+    expect(fetchedURLs).not.toContain(BY_AUTHOR_URL);
+  });
+
+  it('stops descending at MAX_CRAWL_DEPTH', async () => {
+    // Chain: root -> d1 -> d2 -> ... each level has one book and one subdir.
+    const dirURL = (i: number) => `https://files.example.com/books/Kids/${'d/'.repeat(i)}?opds`;
+    const chainLen = MAX_CRAWL_DEPTH + 2;
+    feeds[BASE] = feedXML('Kids', dirEntry('d1', dirURL(1)));
+    for (let i = 1; i <= chainLen; i++) {
+      feeds[dirURL(i)] = feedXML(
+        `d${i}`,
+        bookEntry(`urn:uuid:book-${i}`, `Book ${i}.epub`, `/dl/book-${i}.epub`) +
+          dirEntry(`d${i + 1}`, dirURL(i + 1)),
+      );
+    }
+
+    const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+    const ids = items.map((i) => i.entryId);
+    expect(ids).toContain(`urn:uuid:book-${MAX_CRAWL_DEPTH}`);
+    expect(ids).not.toContain(`urn:uuid:book-${MAX_CRAWL_DEPTH + 1}`);
+  });
+
+  it('stops fetching after MAX_FEEDS_PER_CRAWL feeds', async () => {
+    const subCount = MAX_FEEDS_PER_CRAWL + 10;
+    let rootBody = '';
+    for (let i = 0; i < subCount; i++) {
+      const url = `https://files.example.com/books/Kids/sub${i}/?opds`;
+      rootBody += dirEntry(`sub${i}`, url);
+      feeds[url] = feedXML(
+        `sub${i}`,
+        bookEntry(`urn:uuid:sub-${i}`, `Sub ${i}.epub`, `/dl/sub-${i}.epub`),
+      );
+    }
+    feeds[BASE] = feedXML('Kids', rootBody);
+
+    const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+    expect(vi.mocked(fetchWithAuth).mock.calls.length).toBeLessThanOrEqual(MAX_FEEDS_PER_CRAWL);
+    // Root fetch consumes one slot from the budget.
+    expect(items).toHaveLength(MAX_FEEDS_PER_CRAWL - 1);
+  });
+
+  it('does not loop on subdirectories that link back to the root', async () => {
+    feeds[BASE] = feedXML(
+      'Kids',
+      dirEntry('Ramona', '/books/Kids/Ramona/?opds') +
+        bookEntry('urn:uuid:rosie', 'Rosie.epub', '/dl/rosie.epub'),
+    );
+    feeds[RAMONA_URL] = feedXML(
+      'Ramona',
+      dirEntry('Kids', '/books/Kids/?opds') +
+        bookEntry('urn:uuid:ramona-1', 'Ramona.epub', '/dl/ramona.epub'),
+    );
+
+    const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+    expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:ramona-1', 'urn:uuid:rosie']);
+    // Root fetched exactly once.
+    const rootFetches = vi.mocked(fetchWithAuth).mock.calls.filter((c) => c[0] === BASE);
+    expect(rootFetches).toHaveLength(1);
+  });
+
+  it('returns a subdirectory book only once when listed in two feeds', async () => {
+    feeds[BASE] = feedXML(
+      'Kids',
+      dirEntry('Ramona', '/books/Kids/Ramona/?opds') +
+        bookEntry('urn:uuid:dup', 'Dup.epub', '/dl/dup.epub'),
+    );
+    feeds[RAMONA_URL] = feedXML('Ramona', bookEntry('urn:uuid:dup', 'Dup.epub', '/dl/dup.epub'));
+
+    const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+    expect(items.filter((i) => i.entryId === 'urn:uuid:dup')).toHaveLength(1);
+  });
+
+  it('skips already-known entries found in subdirectories', async () => {
+    feeds[BASE] = feedXML('Kids', dirEntry('Ramona', '/books/Kids/Ramona/?opds'));
+    feeds[RAMONA_URL] = feedXML(
+      'Ramona',
+      bookEntry('urn:uuid:known', 'Known.epub', '/dl/known.epub') +
+        bookEntry('urn:uuid:fresh', 'Fresh.epub', '/dl/fresh.epub'),
+    );
+
+    const state = { ...emptyState(), knownEntryIds: ['urn:uuid:known'] };
+    const items = await checkFeedForNewItems(makeCatalog(), state);
+    expect(items.map((i) => i.entryId)).toEqual(['urn:uuid:fresh']);
+  });
+
+  it('follows rel=next pagination inside crawled subdirectories', async () => {
+    const PAGE2 = 'https://files.example.com/books/Kids/Ramona/?opds&page=2';
+    feeds[BASE] = feedXML('Kids', dirEntry('Ramona', '/books/Kids/Ramona/?opds'));
+    feeds[RAMONA_URL] = feedXML(
+      'Ramona',
+      bookEntry('urn:uuid:p1', 'Page1.epub', '/dl/p1.epub'),
+      `<link rel="next" href="/books/Kids/Ramona/?opds&amp;page=2" type="application/atom+xml;profile=opds-catalog"/>`,
+    );
+    feeds[PAGE2] = feedXML('Ramona p2', bookEntry('urn:uuid:p2', 'Page2.epub', '/dl/p2.epub'));
+
+    const items = await checkFeedForNewItems(makeCatalog(), emptyState());
+    expect(items.map((i) => i.entryId).sort()).toEqual(['urn:uuid:p1', 'urn:uuid:p2']);
+  });
+});
+
+describe('getSubsectionURLs', () => {
+  const baseURL = 'https://files.example.com/books/Kids/?opds';
+
+  it('returns resolved URLs for subsection navigation entries', () => {
+    const feed: OPDSFeed = {
+      metadata: { title: 'Kids' },
+      links: [],
+      navigation: [
+        {
+          title: 'Ramona/',
+          href: '/books/Kids/Ramona/?opds',
+          rel: 'subsection',
+          type: 'application/atom+xml;profile=opds-catalog',
+          properties: {},
+        },
+      ],
+    };
+    expect(getSubsectionURLs(feed, baseURL)).toEqual([RAMONA_URL]);
+  });
+
+  it('accepts navigation entries without a type', () => {
+    const feed: OPDSFeed = {
+      metadata: { title: 'Kids' },
+      links: [],
+      navigation: [{ title: 'Sub', href: 'Sub/?opds', properties: {} }],
+    };
+    expect(getSubsectionURLs(feed, baseURL)).toEqual([
+      'https://files.example.com/books/Kids/Sub/?opds',
+    ]);
+  });
+
+  it('skips entries with non-catalog types', () => {
+    const feed: OPDSFeed = {
+      metadata: { title: 'Kids' },
+      links: [],
+      navigation: [{ title: 'Readme', href: '/readme.html', type: 'text/html', properties: {} }],
+    };
+    expect(getSubsectionURLs(feed, baseURL)).toEqual([]);
+  });
+
+  it('skips facet, self, up, search and start rels', () => {
+    const catalogType = 'application/atom+xml;profile=opds-catalog';
+    const feed: OPDSFeed = {
+      metadata: { title: 'Kids' },
+      links: [],
+      navigation: [
+        {
+          title: 'F',
+          href: '/f',
+          rel: 'http://opds-spec.org/facet',
+          type: catalogType,
+          properties: {},
+        },
+        { title: 'S', href: '/s', rel: 'self', type: catalogType, properties: {} },
+        { title: 'U', href: '/u', rel: 'up', type: catalogType, properties: {} },
+        { title: 'Q', href: '/q', rel: 'search', type: catalogType, properties: {} },
+        { title: 'T', href: '/t', rel: 'start', type: catalogType, properties: {} },
+      ],
+    };
+    expect(getSubsectionURLs(feed, baseURL)).toEqual([]);
+  });
+
+  it('returns empty for feeds without navigation', () => {
+    const feed: OPDSFeed = { metadata: { title: 'Kids' }, links: [] };
+    expect(getSubsectionURLs(feed, baseURL)).toEqual([]);
+  });
+});

apps/readest-app/src/services/opds/feedChecker.ts

@@ -21,7 +21,7 @@ import {
 } from '@/app/opds/utils/opdsUtils';
 import { normalizeOPDSCustomHeaders } from '@/app/opds/utils/customHeaders';
 import type { OPDSSubscriptionState, PendingItem } from './types';
-import { MAX_PAGES_PER_FEED } from './types';
+import { MAX_CRAWL_DEPTH, MAX_FEEDS_PER_CRAWL, MAX_PAGES_PER_FEED } from './types';
 
 const SORT_NEW_REL = 'http://opds-spec.org/sort/new';
 
@@ -268,9 +268,10 @@ function hasRel(item: LinkLike, target: string): boolean {
 
 /**
  * Find the catalog's "by newest" feed URL — the one that lists publications
- * in reverse-chronological order. Auto-download follows only this feed (plus
+ * in reverse-chronological order. For library catalogs auto-download follows
- * its rel=next pages); we deliberately don't crawl the rest of the navigation
+ * only this feed (plus its rel=next pages); we deliberately don't crawl the
- * tree because subscribing to a whole catalog is rarely what the user wants.
+ * rest of the navigation tree because subscribing to a whole library catalog
+ * is rarely what the user wants.
  *
  * Detection order:
  *  1. Authoritative: any link or navigation entry with
@@ -280,8 +281,8 @@ function hasRel(item: LinkLike, target: string): boolean {
  *  3. Href heuristics: ?sort_order=release_date (Project Gutenberg),
  *     /new-releases, /recently-added, ?sort=new, etc.
  *
- * Returns undefined when no candidate matches — the caller should treat the
+ * Returns undefined when no candidate matches — the caller then treats the
- * catalog as not auto-download-capable rather than fall back to a deep crawl.
+ * catalog as a directory-style feed and crawls its subsections instead.
  */
 export function findNewestFeedURL(feed: OPDSFeed, baseURL: string): string | undefined {
   const candidates: LinkLike[] = [...(feed.links ?? []), ...(feed.navigation ?? [])];
@@ -312,43 +313,110 @@ function feedHasContent(feed: OPDSFeed): boolean {
   return false;
 }
 
+// Rels that must not be treated as subdirectories when crawling a
+// directory-style catalog: facets and structural links (self/up/start/…)
+// would re-list the same feed under another sort order or escape the
+// subscribed folder into its parent.
+const CRAWL_SKIP_RELS = [REL.FACET, 'self', 'start', 'up', 'top', 'search'];
+
 /**
- * Resolve a catalog URL down to its acquisition feed:
+ * Collect sub-catalog URLs from a feed's navigation entries — the
- * - If the root already contains publications, use it as-is.
+ * subdirectories of a directory-style catalog (copyparty and other file
- * - Otherwise look for a "by newest" link and follow it.
+ * servers expose folders as rel="subsection" entries). Entries with a
- *
+ * non-catalog media type or a facet/structural rel are skipped; a missing
- * Returns null when no acquisition feed can be found — auto-download will
+ * type is accepted since many servers omit it on navigation entries.
- * skip the catalog rather than crawl through every navigation branch.
  */
-async function resolveAcquisitionFeed(
+export function getSubsectionURLs(feed: OPDSFeed, baseURL: string): string[] {
-  url: string,
+  const urls: string[] = [];
-  username: string,
+  for (const item of feed.navigation ?? []) {
-  password: string,
+    if (!item.href) continue;
-  customHeaders: Record<string, string>,
+    const rels = Array.isArray(item.rel) ? item.rel : [item.rel ?? ''];
-  visited: Set<string>,
+    if (rels.some((rel) => CRAWL_SKIP_RELS.includes(rel))) continue;
-): Promise<{ feed: OPDSFeed; baseURL: string } | null> {
+    if (item.type && !isOPDSCatalog(item.type)) continue;
-  visited.add(url);
+    urls.push(resolveURL(item.href, baseURL));
-  const root = await fetchFeed(url, username, password, customHeaders);
-  if (!root) return null;
-
-  const newestURL = findNewestFeedURL(root.feed, root.baseURL);
-  if (newestURL && !visited.has(newestURL)) {
-    visited.add(newestURL);
-    const newest = await fetchFeed(newestURL, username, password, customHeaders);
-    if (newest && feedHasContent(newest.feed)) return newest;
   }
+  return urls;
+}
 
-  if (feedHasContent(root.feed)) return root;
+interface CrawlContext {
-  return null;
+  catalog: OPDSCatalog;
+  knownIds: Set<string>;
+  username: string;
+  password: string;
+  customHeaders: Record<string, string>;
+  visited: Set<string>;
+  /** Follow subsection navigation entries (directory-style catalogs). */
+  crawlNav: boolean;
 }
 
 /**
- * Check a catalog's "by newest" feed for new items.
+ * Walk feeds breadth-first from an already-fetched start feed, collecting
+ * new PendingItems. Every feed's rel=next chain is followed up to
+ * MAX_PAGES_PER_FEED pages. When ctx.crawlNav is set, subsection navigation
+ * entries are followed too, at most MAX_CRAWL_DEPTH levels below the start
+ * feed and MAX_FEEDS_PER_CRAWL fetches in total.
+ */
+async function crawlFeeds(
+  start: { feed: OPDSFeed; baseURL: string },
+  ctx: CrawlContext,
+): Promise<PendingItem[]> {
+  const items: PendingItem[] = [];
+  const queue: Array<{ url: string; depth: number; page: number }> = [];
+
+  const processFeed = (feed: OPDSFeed, baseURL: string, depth: number, page: number) => {
+    const newItems = collectNewEntries(feed, ctx.knownIds, baseURL);
+    // Mark as known so a book listed by several crawled feeds is only
+    // collected once.
+    for (const item of newItems) ctx.knownIds.add(item.entryId);
+    items.push(...newItems);
+
+    const nextHref = getNextPageUrl(feed);
+    if (nextHref && page < MAX_PAGES_PER_FEED) {
+      const nextURL = resolveURL(nextHref, baseURL);
+      if (!ctx.visited.has(nextURL)) {
+        ctx.visited.add(nextURL);
+        queue.push({ url: nextURL, depth, page: page + 1 });
+      }
+    }
+
+    if (ctx.crawlNav && depth < MAX_CRAWL_DEPTH) {
+      for (const subURL of getSubsectionURLs(feed, baseURL)) {
+        if (ctx.visited.has(subURL)) continue;
+        ctx.visited.add(subURL);
+        queue.push({ url: subURL, depth: depth + 1, page: 1 });
+      }
+    }
+  };
+
+  processFeed(start.feed, start.baseURL, 0, 1);
+
+  let fetches = 1; // the start feed was already fetched by the caller
+  while (queue.length > 0 && fetches < MAX_FEEDS_PER_CRAWL) {
+    const node = queue.shift()!;
+    const next = await fetchFeed(node.url, ctx.username, ctx.password, ctx.customHeaders);
+    fetches++;
+    if (!next) continue;
+    processFeed(next.feed, next.baseURL, node.depth, node.page);
+  }
+  if (queue.length > 0) {
+    console.warn(
+      `OPDS sync: catalog "${ctx.catalog.name}" crawl budget exhausted; ${queue.length} sub-feed(s) skipped`,
+    );
+  }
+
+  return items;
+}
+
+/**
+ * Check a catalog for new items. Pure discovery — no downloads, no state
+ * mutations.
  *
- * Pure discovery — no downloads, no state mutations. Resolves the catalog
+ * Library catalogs (those exposing a "by newest" feed, see
- * URL to its newest-acquisition feed (see resolveAcquisitionFeed) and walks
+ * findNewestFeedURL) are checked through that feed and its rel=next pages
- * up to MAX_PAGES_PER_FEED of rel=next pagination, collecting entries that
+ * only. Catalogs without one are directory-style listings (e.g. copyparty):
- * aren't already in knownEntryIds.
+ * the subscribed URL itself is the acquisition feed and its subsection
+ * navigation entries are subdirectories, which are crawled breadth-first so
+ * books in subfolders are downloaded too (#4272).
  */
 export async function checkFeedForNewItems(
   catalog: OPDSCatalog,
@@ -358,42 +426,38 @@ export async function checkFeedForNewItems(
   const customHeaders = normalizeOPDSCustomHeaders(catalog.customHeaders);
   const username = catalog.username ?? '';
   const password = catalog.password ?? '';
-  const visited = new Set<string>();
+  const visited = new Set<string>([catalog.url]);
 
-  const acquisition = await resolveAcquisitionFeed(
+  const root = await fetchFeed(catalog.url, username, password, customHeaders);
-    catalog.url,
+  if (!root) return [];
+
+  const ctx: CrawlContext = {
+    catalog,
+    knownIds,
     username,
     password,
     customHeaders,
     visited,
-  );
+    crawlNav: false,
-  if (!acquisition) {
+  };
+
+  const newestURL = findNewestFeedURL(root.feed, root.baseURL);
+  if (newestURL) {
+    if (!visited.has(newestURL)) {
+      visited.add(newestURL);
+      const newest = await fetchFeed(newestURL, username, password, customHeaders);
+      if (newest && feedHasContent(newest.feed)) return crawlFeeds(newest, ctx);
+    }
+    // Broken or empty "by newest" feed: fall back to the root feed's own
+    // publications, still without crawling navigation.
+    return crawlFeeds(root, ctx);
+  }
+
+  if (!feedHasContent(root.feed) && getSubsectionURLs(root.feed, root.baseURL).length === 0) {
     console.warn(
-      `OPDS sync: catalog "${catalog.name}" has no recognizable "by newest" feed; skipping`,
+      `OPDS sync: catalog "${catalog.name}" has no publications or subdirectories; skipping`,
     );
     return [];
   }
-
+  return crawlFeeds(root, { ...ctx, crawlNav: true });
-  const items: PendingItem[] = [];
-  let { feed, baseURL } = acquisition;
-  items.push(...collectNewEntries(feed, knownIds, baseURL));
-
-  let pageCount = 1;
-  while (pageCount < MAX_PAGES_PER_FEED) {
-    const nextHref = getNextPageUrl(feed);
-    if (!nextHref) break;
-    const nextUrl = resolveURL(nextHref, baseURL);
-    if (visited.has(nextUrl)) break;
-    visited.add(nextUrl);
-
-    const next = await fetchFeed(nextUrl, username, password, customHeaders);
-    if (!next) break;
-
-    items.push(...collectNewEntries(next.feed, knownIds, next.baseURL));
-    feed = next.feed;
-    baseURL = next.baseURL;
-    pageCount++;
-  }
-
-  return items;
 }

apps/readest-app/src/services/opds/types.ts

@@ -3,6 +3,11 @@ import type { Book } from '@/types/book';
 // --- Constants ---
 
 export const MAX_PAGES_PER_FEED = 5;
+// Directory-style catalogs (e.g. copyparty file listings) expose subfolders
+// as rel="subsection" navigation entries. When a catalog has no "by newest"
+// feed those subsections are crawled breadth-first, bounded by these caps.
+export const MAX_CRAWL_DEPTH = 5;
+export const MAX_FEEDS_PER_CRAWL = 50;
 export const MAX_KNOWN_ENTRIES = 2000;
 export const MAX_RETRY_ATTEMPTS = 3;
 export const RETRY_BACKOFF_MS = 60_000;