akai/readest
2
1
Fork 1
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

txt: more robust TXT parsing (#1706)

Browse Source
This commit is contained in:
Huang Xin
2025-07-30 02:06:55 +08:00
committed by GitHub
parent 63ed763499
commit c581f8ab47
3 changed files with 54 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/readest-app/src/app/library/hooks/useBooksSync.ts
+1 -1
View File
@@ -72,6 +72,7 @@ export const useBooksSync = ({ onSyncStart, onSyncEnd }: UseBooksSyncProps) => {
if (matchingBook) {
if (!matchingBook.deletedAt && matchingBook.uploadedAt && !oldBook.coverDownloadedAt) {
await appService?.downloadBook(oldBook, true);
oldBook.coverImageUrl = await appService?.generateCoverImageUrl(oldBook);
}
const mergedBook =
matchingBook.updatedAt > oldBook.updatedAt
@@ -92,7 +93,6 @@ export const useBooksSync = ({ onSyncStart, onSyncEnd }: UseBooksSyncProps) => {
console.error('Failed to download book:', newBook);
} finally {
newBook.coverImageUrl = await appService?.generateCoverImageUrl(newBook);
newBook.coverImageUrl += '?timestamp=' + Date.now();
updatedLibrary.push(newBook);
setLibrary(updatedLibrary);
}
apps/readest-app/src/services/webAppService.ts
+1
View File
@@ -51,6 +51,7 @@ const indexedDBFileSystem: FileSystem = {
const content = await this.readFile(path, base, 'binary');
return URL.createObjectURL(new Blob([content]));
} catch {
console.warn(`Failed to get Blob URL for path: ${path} in base: ${base}`);
return path;
}
},
apps/readest-app/src/utils/txt.ts
+52 -27
View File
@@ -21,7 +21,7 @@ interface Txt2EpubOptions {
interface ExtractChapterOptions {
linesBetweenSegments: number;
paragraphsPerChapter?: number;
fallbackParagraphsPerChapter: number;
}
interface ConversionResult {
@@ -68,9 +68,10 @@ export class TxtToEpubConverter {
const metadata = { bookTitle, author, language, identifier };
let chapters: Chapter[] = [];
for (let i = 4; i >= 3; i--) {
for (let i = 8; i >= 6; i--) {
chapters = this.extractChapters(txtContent, metadata, {
linesBetweenSegments: i,
fallbackParagraphsPerChapter: 100,
});
if (chapters.length === 0) {
@@ -79,12 +80,6 @@ export class TxtToEpubConverter {
break;
}
}
if (chapters.length === 1) {
chapters = this.extractChapters(txtContent, metadata, {
linesBetweenSegments: 4,
paragraphsPerChapter: 100,
});
}
const blob = await this.createEpub(chapters, metadata);
return {
@@ -101,24 +96,38 @@ export class TxtToEpubConverter {
option: ExtractChapterOptions,
): Chapter[] {
const { language } = metadata;
const { linesBetweenSegments, paragraphsPerChapter } = option;
const { linesBetweenSegments, fallbackParagraphsPerChapter } = option;
const segmentRegex = new RegExp(`(?:\\r?\\n){${linesBetweenSegments},}|-{8,}\r?\n`);
let chapterRegex: RegExp;
const chapterRegexps: RegExp[] = [];
if (language === 'zh') {
chapterRegex = new RegExp(
String.raw`(?:^|\n|\s)` +
'(' +
[
String.raw`第[零〇一二三四五六七八九十0-9][零〇一二三四五六七八九十百千万0-9]*(?:[章卷节回讲篇封])(?:[：:、 　\(\)0-9]*[^\n-]{0,24})(?!\S)`,
String.raw`(?:^|\n|\s|《[^》]+》)[一二三四五六七八九十][零〇一二三四五六七八九十百千万]*(?:[：: 　][^\n-]{0,24})(?!\S)`,
String.raw`(?:楔子|前言|引言|序言|序章|总论|概论)(?:[：: 　][^\n-]{0,24})?(?!\S)`,
].join('|') +
')',
'gu',
chapterRegexps.push(
new RegExp(
String.raw`(?:^|\n)\s*` +
'(' +
[
String.raw`第[零〇一二三四五六七八九十0-9][零〇一二三四五六七八九十百千万0-9]*(?:[章卷节回讲篇封])(?:[：:、 　\(\)0-9]*[^\n-]{0,24})(?!\S)`,
String.raw`(?:楔子|前言|简介|引言|序言|序章|总论|概论)(?:[：: 　][^\n-]{0,24})?(?!\S)`,
].join('|') +
')',
'gu',
),
);
chapterRegexps.push(
new RegExp(
String.raw`(?:^|\n)\s*` +
'(' +
[
String.raw`[一二三四五六七八九十][零〇一二三四五六七八九十百千万]?[：:、 　][^\n-]{0,24}(?=\n|$)`,
String.raw`[0-9]+[^\n]{0,16}(?=\n|$)`,
].join('|') +
')',
'gu',
),
);
} else {
chapterRegex =
/(?:^|\n|\s)(?:(Chapter|Part)\s+(\d+|[IVXLCDM]+)(?:[:.\-–—]?\s+[^\n]*)?|(?:Prologue|Epilogue|Introduction|Foreword)(?:[:.\-–—]?\s+[^\n]*)?)(?=\s|$)/gi;
chapterRegexps.push(
/(?:^|\n|\s)(?:(Chapter|Part)\s+(\d+|[IVXLCDM]+)(?:[:.\-–—]?\s+[^\n]*)?|(?:Prologue|Epilogue|Introduction|Foreword)(?:[:.\-–—]?\s+[^\n]*)?)(?=\s|$)/gi,
);
}
const formatSegment = (segment: string): string => {
@@ -149,17 +158,35 @@ export class TxtToEpubConverter {
return acc;
}, []);
const isGoodMatches = (matches: string[], maxLength: number = 100000): boolean => {
const meaningfulParts = matches.filter((part) => part.trim().length > 0);
if (meaningfulParts.length <= 1) return false;
const hasLongParts = meaningfulParts.some((part) => part.length > maxLength);
return !hasLongParts;
};
const chapters: Chapter[] = [];
const segments = txtContent.split(segmentRegex);
for (const segment of segments) {
const trimmedSegment = segment.replace(/<!--.*?-->/g, '').trim();
if (!trimmedSegment) continue;
if (paragraphsPerChapter && paragraphsPerChapter > 0) {
const segmentChapters = [];
let matches: string[] = [];
for (const chapterRegex of chapterRegexps) {
const tryMatches = trimmedSegment.split(chapterRegex);
if (isGoodMatches(tryMatches)) {
matches = joinAroundUndefined(tryMatches);
break;
}
}
if (matches.length === 0 && fallbackParagraphsPerChapter > 0) {
const paragraphs = trimmedSegment.split(/\n+/);
const totalParagraphs = paragraphs.length;
for (let i = 0; i < totalParagraphs; i += paragraphsPerChapter) {
const chunks = paragraphs.slice(i, i + paragraphsPerChapter);
for (let i = 0; i < totalParagraphs; i += fallbackParagraphsPerChapter) {
const chunks = paragraphs.slice(i, i + fallbackParagraphsPerChapter);
const formattedSegment = formatSegment(chunks.join('\n'));
const title = `${chapters.length + 1}`;
const content = `<h2>${title}</h2><p>${formattedSegment}</p>`;
@@ -168,8 +195,6 @@ export class TxtToEpubConverter {
continue;
}
const segmentChapters = [];
const matches = joinAroundUndefined(trimmedSegment.split(chapterRegex));
for (let j = 1; j < matches.length; j += 2) {
const title = matches[j]?.trim() || '';
const content = matches[j + 1]?.trim() || '';