refactor(toc): cache navigable structure per book (#3869)
* refactor(toc): cache TOC + section fragments per book
Moves the TOC regrouping and section-fragment computation out of
foliate-js/epub.js #updateSubItems into the readest client as
computeBookNav / hydrateBookNav in utils/toc.ts. The result is
persisted to Books/{hash}/nav.json — capturing the book's full
navigable structure (TOC hierarchy + sections with hierarchical
fragments). Compute once, persist locally, hydrate on subsequent
opens. Designed to serve current human-facing navigation (TOC
sidebar, progress math) and future agentic navigation (LLM-driven
seeking by structural location).
Versioned by BOOK_NAV_VERSION for forward invalidation. Existing
books regenerate transparently on next open.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
* chore: update worktree scripts
---------
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Huang Xin
@@ -35,6 +35,9 @@ export const getCoverFilename = (book: Book) => {
|
||||
export const getConfigFilename = (book: Book) => {
|
||||
return `${book.hash}/config.json`;
|
||||
};
|
||||
export const getBookNavFilename = (book: Book) => {
|
||||
return `${book.hash}/nav.json`;
|
||||
};
|
||||
export const isBookFile = (filename: string) => {
|
||||
return Object.values(EXTS).includes(filename.split('.').pop()!);
|
||||
};
|
||||
|
||||
@@ -1,8 +1,31 @@
|
||||
import { ConvertChineseVariant } from '@/types/book';
|
||||
import { SectionItem, TOCItem, CFI, BookDoc } from '@/libs/document';
|
||||
import { SectionFragment, SectionItem, TOCItem, CFI, BookDoc } from '@/libs/document';
|
||||
import { initSimpleCC, runSimpleCC } from '@/utils/simplecc';
|
||||
import { SIZE_PER_LOC } from '@/services/constants';
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Book navigation artifact (persisted to Books/{hash}/nav.json).
|
||||
// Bump BOOK_NAV_VERSION whenever computeBookNav output semantics change
|
||||
// (TOC grouping heuristic, fragment CFI/size math, hierarchy rules).
|
||||
// v2: fragment CFIs are derived from the section DOM via CFI.joinIndir instead
|
||||
// of inherited from the TOC item (ported from foliate-js 317051e).
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
export const BOOK_NAV_VERSION = 2;
|
||||
|
||||
export type { SectionFragment };
|
||||
|
||||
export interface BookNavSection {
|
||||
id: string;
|
||||
fragments: SectionFragment[];
|
||||
}
|
||||
|
||||
export interface BookNav {
|
||||
version: number;
|
||||
toc: TOCItem[];
|
||||
sections: Record<string, BookNavSection>;
|
||||
}
|
||||
|
||||
export const findParentPath = (toc: TOCItem[], href: string): TOCItem[] => {
|
||||
for (const item of toc) {
|
||||
if (item.href === href) {
|
||||
@@ -58,34 +81,34 @@ const calculateCumulativeSizes = (sections: SectionItem[]): number[] => {
|
||||
}, []);
|
||||
};
|
||||
|
||||
// Helper: Process subitems recursively to assign locations
|
||||
const processSubitemLocations = (
|
||||
subitems: SectionItem[],
|
||||
// Helper: Process fragments recursively to assign locations
|
||||
const processFragmentLocations = (
|
||||
fragments: SectionFragment[],
|
||||
parentByteOffset: number,
|
||||
parentLocation: { current: number; next: number; total: number },
|
||||
totalLocations: number,
|
||||
) => {
|
||||
let currentByteOffset = parentByteOffset;
|
||||
|
||||
subitems.forEach((subitem, index) => {
|
||||
const nextSubitem = index < subitems.length - 1 ? subitems[index + 1] : null;
|
||||
fragments.forEach((fragment, index) => {
|
||||
const nextFragment = index < fragments.length - 1 ? fragments[index + 1] : null;
|
||||
|
||||
currentByteOffset += subitem.size || 0;
|
||||
const nextByteOffset = nextSubitem
|
||||
? currentByteOffset + (nextSubitem.size || 0)
|
||||
currentByteOffset += fragment.size || 0;
|
||||
const nextByteOffset = nextFragment
|
||||
? currentByteOffset + (nextFragment.size || 0)
|
||||
: parentLocation.next * SIZE_PER_LOC;
|
||||
|
||||
subitem.location = {
|
||||
fragment.location = {
|
||||
current: Math.floor(currentByteOffset / SIZE_PER_LOC),
|
||||
next: Math.floor(nextByteOffset / SIZE_PER_LOC),
|
||||
total: totalLocations,
|
||||
};
|
||||
|
||||
if (subitem.subitems?.length) {
|
||||
processSubitemLocations(
|
||||
subitem.subitems,
|
||||
if (fragment.fragments?.length) {
|
||||
processFragmentLocations(
|
||||
fragment.fragments,
|
||||
currentByteOffset,
|
||||
subitem.location,
|
||||
fragment.location,
|
||||
totalLocations,
|
||||
);
|
||||
}
|
||||
@@ -108,29 +131,38 @@ const updateSectionLocations = (
|
||||
total: totalLocations,
|
||||
};
|
||||
|
||||
if (section.subitems?.length) {
|
||||
processSubitemLocations(section.subitems, baseOffset, section.location, totalLocations);
|
||||
if (section.fragments?.length) {
|
||||
processFragmentLocations(section.fragments, baseOffset, section.location, totalLocations);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Helper: Recursively add subitems to sections map
|
||||
const addSubitemsToMap = (subitems: SectionItem[], map: Record<string, SectionItem>) => {
|
||||
for (const subitem of subitems) {
|
||||
if (subitem.href) map[subitem.href] = subitem;
|
||||
if (subitem.subitems?.length) addSubitemsToMap(subitem.subitems, map);
|
||||
// Narrow type that both SectionItem and SectionFragment satisfy — the fields
|
||||
// read by updateTocLocation when mapping TOC items to their section/fragment.
|
||||
interface LocatedEntry {
|
||||
id: string;
|
||||
href?: string;
|
||||
cfi: string;
|
||||
location?: SectionFragment['location'];
|
||||
}
|
||||
|
||||
// Helper: Recursively add fragments to sections map
|
||||
const addFragmentsToMap = (fragments: SectionFragment[], map: Record<string, LocatedEntry>) => {
|
||||
for (const fragment of fragments) {
|
||||
if (fragment.href) map[fragment.href] = fragment;
|
||||
if (fragment.fragments?.length) addFragmentsToMap(fragment.fragments, map);
|
||||
}
|
||||
};
|
||||
|
||||
// Helper: Create sections lookup map including all subitems
|
||||
// Helper: Create sections lookup map including all fragments
|
||||
type Href = string;
|
||||
type SectionsMap = Record<Href, SectionItem>;
|
||||
type SectionsMap = Record<Href, LocatedEntry>;
|
||||
const createSectionsMap = (sections: SectionItem[]) => {
|
||||
const map: SectionsMap = {};
|
||||
|
||||
for (const section of sections) {
|
||||
map[section.id] = section;
|
||||
if (section.subitems?.length) addSubitemsToMap(section.subitems, map);
|
||||
if (section.fragments?.length) addFragmentsToMap(section.fragments, map);
|
||||
}
|
||||
|
||||
return map;
|
||||
@@ -160,7 +192,7 @@ export const updateToc = async (
|
||||
const totalSize = cumulativeSizes[cumulativeSizes.length - 1]! + sizes[sizes.length - 1]!;
|
||||
const totalLocations = Math.floor(totalSize / SIZE_PER_LOC);
|
||||
|
||||
// Step 3: Update locations to sections and subitems
|
||||
// Step 3: Update locations to sections and fragments
|
||||
updateSectionLocations(sections, cumulativeSizes, sizes, totalLocations);
|
||||
|
||||
// Step 4: Create sections map and update TOC locations
|
||||
@@ -223,3 +255,273 @@ const sortTocItems = (items: TOCItem[]): void => {
|
||||
return 0;
|
||||
});
|
||||
};
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// computeBookNav / hydrateBookNav
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
const cloneTocItems = (items: TOCItem[]): TOCItem[] =>
|
||||
items.map((item) => ({
|
||||
...item,
|
||||
subitems: item.subitems ? cloneTocItems(item.subitems) : undefined,
|
||||
}));
|
||||
|
||||
// Ported from foliate-js: restructure TOC so that fragment-linked subitems under
|
||||
// the same section are regrouped under a natural parent when one exists.
|
||||
const groupTocSubitems = (bookDoc: BookDoc, items: TOCItem[]): void => {
|
||||
const splitHref = (href: string) => bookDoc.splitTOCHref(href);
|
||||
|
||||
const groupBySection = (subitems: TOCItem[]) => {
|
||||
const grouped = new Map<string, TOCItem[]>();
|
||||
for (const subitem of subitems) {
|
||||
const [sectionId] = splitHref(subitem.href) as [string | undefined];
|
||||
const key = sectionId ?? '';
|
||||
const bucket = grouped.get(key) ?? [];
|
||||
bucket.push(subitem);
|
||||
grouped.set(key, bucket);
|
||||
}
|
||||
return grouped;
|
||||
};
|
||||
|
||||
const separateParentAndFragments = (sectionId: string, subitems: TOCItem[]) => {
|
||||
let parent: TOCItem | null = null;
|
||||
const fragments: TOCItem[] = [];
|
||||
for (const subitem of subitems) {
|
||||
const [, fragmentId] = splitHref(subitem.href) as [string | undefined, string | undefined];
|
||||
if (!fragmentId || subitem.href === sectionId) {
|
||||
parent = subitem;
|
||||
} else {
|
||||
fragments.push(subitem);
|
||||
}
|
||||
}
|
||||
return { parent, fragments };
|
||||
};
|
||||
|
||||
for (const item of items) {
|
||||
if (!item.subitems?.length) continue;
|
||||
|
||||
const groupedBySection = groupBySection(item.subitems);
|
||||
if (groupedBySection.size <= 3) continue;
|
||||
|
||||
const newSubitems: TOCItem[] = [];
|
||||
for (const [sectionId, subitems] of groupedBySection.entries()) {
|
||||
if (item.href === sectionId) {
|
||||
newSubitems.push(...subitems);
|
||||
continue;
|
||||
}
|
||||
if (subitems.length === 1) {
|
||||
newSubitems.push(subitems[0]!);
|
||||
} else {
|
||||
const { parent, fragments } = separateParentAndFragments(sectionId, subitems);
|
||||
if (parent) {
|
||||
parent.subitems = fragments.length > 0 ? fragments : parent.subitems;
|
||||
newSubitems.push(parent);
|
||||
} else {
|
||||
newSubitems.push(...subitems);
|
||||
}
|
||||
}
|
||||
}
|
||||
item.subitems = newSubitems;
|
||||
}
|
||||
};
|
||||
|
||||
const findFragmentPosition = (html: string, fragmentId: string | undefined): number => {
|
||||
if (!fragmentId) return html.length;
|
||||
const patterns = [
|
||||
new RegExp(`\\sid=["']${CSS.escape(fragmentId)}["']`, 'i'),
|
||||
new RegExp(`\\sname=["']${CSS.escape(fragmentId)}["']`, 'i'),
|
||||
];
|
||||
for (const pattern of patterns) {
|
||||
const match = html.match(pattern);
|
||||
if (match && typeof match.index === 'number') return match.index;
|
||||
}
|
||||
return -1;
|
||||
};
|
||||
|
||||
const calculateFragmentSize = (
|
||||
content: string,
|
||||
fragmentId: string | undefined,
|
||||
prevFragmentId: string | undefined,
|
||||
): number => {
|
||||
const endPos = findFragmentPosition(content, fragmentId);
|
||||
if (endPos < 0) return 0;
|
||||
const startPos = prevFragmentId ? findFragmentPosition(content, prevFragmentId) : 0;
|
||||
const validStartPos = Math.max(0, startPos);
|
||||
if (endPos < validStartPos) return 0;
|
||||
return new Blob([content.substring(validStartPos, endPos)]).size;
|
||||
};
|
||||
|
||||
const collectAllTocItems = (items: TOCItem[]): TOCItem[] => {
|
||||
const out: TOCItem[] = [];
|
||||
const walk = (xs: TOCItem[]) => {
|
||||
for (const x of xs) {
|
||||
out.push(x);
|
||||
if (x.subitems?.length) walk(x.subitems);
|
||||
}
|
||||
};
|
||||
walk(items);
|
||||
return out;
|
||||
};
|
||||
|
||||
interface SectionGroup {
|
||||
base: TOCItem | null;
|
||||
fragments: TOCItem[];
|
||||
}
|
||||
|
||||
const groupItemsBySection = (bookDoc: BookDoc, items: TOCItem[]): Map<string, SectionGroup> => {
|
||||
const groups = new Map<string, SectionGroup>();
|
||||
for (const item of items) {
|
||||
if (!item.href) continue;
|
||||
const [sectionId, fragmentId] = bookDoc.splitTOCHref(item.href) as [
|
||||
string | undefined,
|
||||
string | undefined,
|
||||
];
|
||||
if (!sectionId) continue;
|
||||
let group = groups.get(sectionId);
|
||||
if (!group) {
|
||||
group = { base: null, fragments: [] };
|
||||
groups.set(sectionId, group);
|
||||
}
|
||||
const isBase = !fragmentId || item.href === sectionId;
|
||||
if (isBase) group.base = item;
|
||||
else group.fragments.push(item);
|
||||
}
|
||||
return groups;
|
||||
};
|
||||
|
||||
const getHTMLFragmentElement = (doc: Document, id: string | undefined): Element | null => {
|
||||
if (!id) return null;
|
||||
return doc.getElementById(id) ?? doc.querySelector(`[name="${CSS.escape(id)}"]`);
|
||||
};
|
||||
|
||||
type CFIModule = {
|
||||
joinIndir: (...xs: string[]) => string;
|
||||
fromElements: (elements: Element[]) => string[];
|
||||
};
|
||||
|
||||
const buildFragmentCfi = (section: SectionItem, element: Element | null): string => {
|
||||
const cfiLib = CFI as unknown as CFIModule;
|
||||
const rel = element ? (cfiLib.fromElements([element])[0] ?? '') : '';
|
||||
return cfiLib.joinIndir(section.cfi, rel);
|
||||
};
|
||||
|
||||
const buildSectionFragments = (
|
||||
section: SectionItem,
|
||||
fragments: TOCItem[],
|
||||
base: TOCItem | null,
|
||||
content: string,
|
||||
doc: Document,
|
||||
splitHref: (href: string) => Array<string | number>,
|
||||
): SectionFragment[] => {
|
||||
const out: SectionFragment[] = [];
|
||||
for (let i = 0; i < fragments.length; i++) {
|
||||
const fragment = fragments[i]!;
|
||||
const [, rawFragmentId] = splitHref(fragment.href) as [string | undefined, string | undefined];
|
||||
const fragmentId = rawFragmentId;
|
||||
|
||||
const prev = i > 0 ? fragments[i - 1] : base;
|
||||
const [, rawPrevFragmentId] = prev
|
||||
? (splitHref(prev.href) as [string | undefined, string | undefined])
|
||||
: [undefined, undefined];
|
||||
const prevFragmentId = rawPrevFragmentId;
|
||||
|
||||
const element = getHTMLFragmentElement(doc, fragmentId);
|
||||
const cfi = buildFragmentCfi(section, element);
|
||||
const size = calculateFragmentSize(content, fragmentId, prevFragmentId);
|
||||
|
||||
out.push({
|
||||
id: fragment.href,
|
||||
href: fragment.href,
|
||||
cfi,
|
||||
size,
|
||||
linear: section.linear,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
/**
|
||||
* Compute a per-book navigation artifact from a freshly opened BookDoc.
|
||||
* Expensive: for each referenced section, loads the HTML text and parses the
|
||||
* XHTML DOM to compute per-fragment CFIs (via CFI.joinIndir) and byte-size
|
||||
* offsets between TOC-fragment anchors. Intended to run on cache miss; the
|
||||
* result is persisted to Books/{hash}/nav.json and replayed via
|
||||
* hydrateBookNav on subsequent opens.
|
||||
*/
|
||||
export const computeBookNav = async (bookDoc: BookDoc): Promise<BookNav> => {
|
||||
const tocClone = cloneTocItems(bookDoc.toc ?? []);
|
||||
const sections: Record<string, BookNavSection> = {};
|
||||
|
||||
if (tocClone.length) {
|
||||
groupTocSubitems(bookDoc, tocClone);
|
||||
}
|
||||
|
||||
const bookSections = bookDoc.sections ?? [];
|
||||
if (!tocClone.length || !bookSections.length) {
|
||||
return { version: BOOK_NAV_VERSION, toc: tocClone, sections };
|
||||
}
|
||||
|
||||
const sectionMap = new Map(bookSections.map((s) => [s.id, s]));
|
||||
const allItems = collectAllTocItems(tocClone);
|
||||
const groups = groupItemsBySection(bookDoc, allItems);
|
||||
const splitHref = (href: string) => bookDoc.splitTOCHref(href);
|
||||
|
||||
for (const [sectionId, { base, fragments }] of groups.entries()) {
|
||||
const section = sectionMap.get(sectionId);
|
||||
if (!section || fragments.length === 0) continue;
|
||||
if (!section.loadText) continue;
|
||||
|
||||
const content = await section.loadText();
|
||||
if (!content) continue;
|
||||
|
||||
let doc: Document | null = null;
|
||||
try {
|
||||
doc = await section.createDocument();
|
||||
} catch (e) {
|
||||
console.warn(`Failed to parse section ${sectionId} for fragment CFIs:`, e);
|
||||
}
|
||||
if (!doc) continue;
|
||||
|
||||
const sectionFragments = buildSectionFragments(
|
||||
section,
|
||||
fragments,
|
||||
base,
|
||||
content,
|
||||
doc,
|
||||
splitHref,
|
||||
);
|
||||
if (sectionFragments.length > 0) {
|
||||
sections[sectionId] = { id: sectionId, fragments: sectionFragments };
|
||||
}
|
||||
}
|
||||
|
||||
return { version: BOOK_NAV_VERSION, toc: tocClone, sections };
|
||||
};
|
||||
|
||||
const cloneSectionFragments = (fragments: SectionFragment[]): SectionFragment[] =>
|
||||
fragments.map((f) => ({
|
||||
id: f.id,
|
||||
href: f.href,
|
||||
cfi: f.cfi,
|
||||
size: f.size,
|
||||
linear: f.linear,
|
||||
fragments: f.fragments ? cloneSectionFragments(f.fragments) : undefined,
|
||||
}));
|
||||
|
||||
/**
|
||||
* Apply a cached BookNav onto a freshly opened BookDoc, replacing its TOC
|
||||
* with the cached (post-grouping) version and attaching per-section
|
||||
* fragments to the corresponding Section objects. No I/O.
|
||||
*/
|
||||
export const hydrateBookNav = (bookDoc: BookDoc, bookNav: BookNav): void => {
|
||||
bookDoc.toc = cloneTocItems(bookNav.toc);
|
||||
const bookSections = bookDoc.sections ?? [];
|
||||
for (const section of bookSections) {
|
||||
const cached = bookNav.sections[section.id];
|
||||
if (cached?.fragments?.length) {
|
||||
section.fragments = cloneSectionFragments(cached.fragments);
|
||||
} else {
|
||||
section.fragments = undefined;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user