Add bilingual EPUB filter
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
import clsx from 'clsx';
|
||||
import * as React from 'react';
|
||||
import { useState } from 'react';
|
||||
import { LuLanguages } from 'react-icons/lu';
|
||||
import { PiX } from 'react-icons/pi';
|
||||
import { useKeyDownActions } from '@/hooks/useKeyDownActions';
|
||||
import { useTranslation } from '@/hooks/useTranslation';
|
||||
import type { BilingualFilterMode, BilingualFilterOptions } from '@/services/bilingualEpubFilter';
|
||||
|
||||
interface BilingualFilterAlertProps {
|
||||
bookTitle: string;
|
||||
safeAreaBottom: number;
|
||||
processing: boolean;
|
||||
onCancel: () => void;
|
||||
onFilter: (options: BilingualFilterOptions) => void;
|
||||
}
|
||||
|
||||
const BilingualFilterAlert: React.FC<BilingualFilterAlertProps> = ({
|
||||
bookTitle,
|
||||
safeAreaBottom,
|
||||
processing,
|
||||
onCancel,
|
||||
onFilter,
|
||||
}) => {
|
||||
const _ = useTranslation();
|
||||
const divRef = useKeyDownActions({ onCancel });
|
||||
const [mode, setMode] = useState<BilingualFilterMode>('auto');
|
||||
const [removeUnknown, setRemoveUnknown] = useState(false);
|
||||
|
||||
const handleFilter = (removeLanguage: BilingualFilterOptions['removeLanguage']) => {
|
||||
if (processing) return;
|
||||
onFilter({ removeLanguage, mode, removeUnknown });
|
||||
};
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={divRef}
|
||||
className='fixed bottom-0 left-0 right-0 z-50 flex justify-center px-3'
|
||||
style={{ paddingBottom: `${safeAreaBottom + 16}px` }}
|
||||
>
|
||||
<div
|
||||
className={clsx(
|
||||
'border-base-content/10 bg-base-200/95 flex w-full max-w-xl flex-col gap-4 rounded-2xl border p-4 shadow-lg backdrop-blur-sm',
|
||||
processing && 'pointer-events-none opacity-80',
|
||||
)}
|
||||
>
|
||||
<div className='relative flex min-w-0 items-center justify-center gap-2 pr-8'>
|
||||
<LuLanguages className='size-5 shrink-0' />
|
||||
<div className='min-w-0 truncate text-center text-sm font-medium'>
|
||||
{_('Bilingual EPUB')}: {bookTitle}
|
||||
</div>
|
||||
<button
|
||||
className={clsx(
|
||||
'absolute right-0 flex items-center justify-center rounded-full p-1.5',
|
||||
'text-base-content/70 transition-colors hover:text-base-content',
|
||||
)}
|
||||
onClick={onCancel}
|
||||
aria-label={_('Cancel')}
|
||||
disabled={processing}
|
||||
>
|
||||
<PiX className='size-5' />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className='grid grid-cols-1 gap-3 sm:grid-cols-[minmax(0,1fr)_auto] sm:items-center'>
|
||||
<label className='flex min-w-0 items-center gap-2'>
|
||||
<span className='text-neutral-content shrink-0 text-xs'>{_('Detection')}</span>
|
||||
<select
|
||||
className='select select-bordered select-sm min-w-0 flex-1'
|
||||
value={mode}
|
||||
disabled={processing}
|
||||
onChange={(event) => setMode(event.target.value as BilingualFilterMode)}
|
||||
>
|
||||
<option value='auto'>{_('Auto')}</option>
|
||||
<option value='style'>{_('Style')}</option>
|
||||
<option value='script'>{_('Script')}</option>
|
||||
</select>
|
||||
</label>
|
||||
<label className='flex cursor-pointer items-center gap-2 justify-self-start sm:justify-self-end'>
|
||||
<input
|
||||
type='checkbox'
|
||||
className='checkbox checkbox-sm'
|
||||
checked={removeUnknown}
|
||||
disabled={processing}
|
||||
onChange={(event) => setRemoveUnknown(event.target.checked)}
|
||||
/>
|
||||
<span className='text-sm'>{_('Remove uncertain text')}</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div className='grid grid-cols-1 gap-2 sm:grid-cols-3'>
|
||||
<button
|
||||
className='btn btn-primary btn-sm'
|
||||
disabled={processing}
|
||||
onClick={() => handleFilter('ja')}
|
||||
>
|
||||
{_('Keep Chinese')}
|
||||
</button>
|
||||
<button
|
||||
className='btn btn-secondary btn-sm'
|
||||
disabled={processing}
|
||||
onClick={() => handleFilter('zh')}
|
||||
>
|
||||
{_('Keep Japanese')}
|
||||
</button>
|
||||
<button className='btn btn-ghost btn-sm' disabled={processing} onClick={onCancel}>
|
||||
{processing ? _('Processing...') : _('Cancel')}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default BilingualFilterAlert;
|
||||
@@ -63,6 +63,11 @@ import GroupingModal from './GroupingModal';
|
||||
import SetStatusAlert from './SetStatusAlert';
|
||||
import RecentShelf, { RECENT_SHELF_BOOK_COUNT } from './RecentShelf';
|
||||
import { useOpenBook } from '../hooks/useOpenBook';
|
||||
import BilingualFilterAlert from './BilingualFilterAlert';
|
||||
import {
|
||||
filterBilingualEpubFile,
|
||||
type BilingualFilterOptions,
|
||||
} from '@/services/bilingualEpubFilter';
|
||||
|
||||
interface BookshelfProps {
|
||||
libraryBooks: Book[];
|
||||
@@ -199,6 +204,8 @@ const Bookshelf: React.FC<BookshelfProps> = ({
|
||||
const [showDeleteAlert, setShowDeleteAlert] = useState(false);
|
||||
const [showStatusAlert, setShowStatusAlert] = useState(false);
|
||||
const [showGroupingModal, setShowGroupingModal] = useState(false);
|
||||
const [showBilingualFilterAlert, setShowBilingualFilterAlert] = useState(false);
|
||||
const [bilingualFilterProcessing, setBilingualFilterProcessing] = useState(false);
|
||||
const [importBookUrl] = useState(searchParams?.get('url') || '');
|
||||
|
||||
const abortDeletionRef = useRef(false);
|
||||
@@ -452,6 +459,91 @@ const Bookshelf: React.FC<BookshelfProps> = ({
|
||||
setShowStatusAlert(true);
|
||||
};
|
||||
|
||||
const getSingleSelectedBook = () => {
|
||||
const ids = getSelectedBooks();
|
||||
if (ids.length !== 1) return;
|
||||
return filteredBooks.find((book) => book.hash === ids[0]);
|
||||
};
|
||||
|
||||
const showBilingualFilterSelection = () => {
|
||||
const book = getSingleSelectedBook();
|
||||
if (!book || book.format !== 'EPUB') return;
|
||||
setShowSelectModeActions(false);
|
||||
setShowBilingualFilterAlert(true);
|
||||
};
|
||||
|
||||
const closeBilingualFilterSelection = () => {
|
||||
if (bilingualFilterProcessing) return;
|
||||
setShowBilingualFilterAlert(false);
|
||||
setShowSelectModeActions(true);
|
||||
};
|
||||
|
||||
const runBilingualFilter = async (options: BilingualFilterOptions) => {
|
||||
const book = getSingleSelectedBook();
|
||||
if (!book || !appService) return;
|
||||
if (book.format !== 'EPUB') {
|
||||
eventDispatcher.dispatch('toast', {
|
||||
type: 'warning',
|
||||
message: _('Only EPUB books can be filtered'),
|
||||
timeout: 2500,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
setBilingualFilterProcessing(true);
|
||||
setLoading(true);
|
||||
|
||||
try {
|
||||
if (!(await appService.isBookAvailable(book))) {
|
||||
if (!book.uploadedAt || !(await handleBookDownload(book, { queued: false }))) {
|
||||
throw new Error('Book file is not available locally');
|
||||
}
|
||||
}
|
||||
|
||||
const { file } = await appService.loadBookContent(book);
|
||||
try {
|
||||
const result = await filterBilingualEpubFile(file, options);
|
||||
const importedBook = await appService.importBook(result.file, libraryBooks);
|
||||
if (!importedBook) throw new Error('Failed to import generated EPUB');
|
||||
|
||||
importedBook.group = book.group;
|
||||
importedBook.groupId = book.groupId;
|
||||
importedBook.groupName = book.groupName;
|
||||
importedBook.tags = book.tags;
|
||||
importedBook.updatedAt = Date.now();
|
||||
|
||||
await updateBooks(envConfig, [importedBook]);
|
||||
handlePushLibrary();
|
||||
setSelectedBooks([]);
|
||||
setShowBilingualFilterAlert(false);
|
||||
handleSetSelectMode(false);
|
||||
eventDispatcher.dispatch('toast', {
|
||||
type: 'success',
|
||||
message: _('Created {{title}}. Removed {{count}} paragraph(s).', {
|
||||
title: importedBook.title || result.title,
|
||||
count: result.stats.removed,
|
||||
}),
|
||||
timeout: 3500,
|
||||
});
|
||||
} finally {
|
||||
const closable = file as File & { close?: () => Promise<void> | void };
|
||||
await closable.close?.();
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to filter bilingual EPUB:', error);
|
||||
eventDispatcher.dispatch('toast', {
|
||||
type: 'error',
|
||||
message: _('Failed to filter bilingual EPUB'),
|
||||
timeout: 3000,
|
||||
});
|
||||
setShowBilingualFilterAlert(false);
|
||||
setShowSelectModeActions(true);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
setBilingualFilterProcessing(false);
|
||||
}
|
||||
};
|
||||
|
||||
const sendSelectedBook = async () => {
|
||||
// "Send" hands the actual book file (epub/pdf/...) to the OS share
|
||||
// sheet (UIActivityViewController on iOS, Intent.ACTION_SEND on
|
||||
@@ -678,6 +770,11 @@ const Bookshelf: React.FC<BookshelfProps> = ({
|
||||
);
|
||||
|
||||
const selectedBooks = getSelectedBooks();
|
||||
const selectedBilingualBook =
|
||||
selectedBooks.length === 1
|
||||
? filteredBooks.find((book) => book.hash === selectedBooks[0])
|
||||
: null;
|
||||
const bilingualFilterEnabled = !!selectedBilingualBook && selectedBilingualBook.format === 'EPUB';
|
||||
const isGridMode = viewMode === 'grid';
|
||||
const hasItems = sortedBookshelfItems.length > 0;
|
||||
// In grid mode the Import-Books "+" tile is rendered as an extra grid cell
|
||||
@@ -889,9 +986,20 @@ const Bookshelf: React.FC<BookshelfProps> = ({
|
||||
onGroup={groupSelectedBooks}
|
||||
onDetails={openBookDetails}
|
||||
onStatus={showStatusSelection}
|
||||
onBilingualFilter={showBilingualFilterSelection}
|
||||
onSend={sendSelectedBook}
|
||||
onDelete={deleteSelectedBooks}
|
||||
onCancel={() => handleSetSelectMode(false)}
|
||||
bilingualFilterEnabled={bilingualFilterEnabled}
|
||||
/>
|
||||
)}
|
||||
{showBilingualFilterAlert && selectedBilingualBook && (
|
||||
<BilingualFilterAlert
|
||||
bookTitle={selectedBilingualBook.title}
|
||||
safeAreaBottom={safeAreaInsets?.bottom || 0}
|
||||
processing={bilingualFilterProcessing}
|
||||
onCancel={closeBilingualFilterSelection}
|
||||
onFilter={runBilingualFilter}
|
||||
/>
|
||||
)}
|
||||
{showGroupingModal && selectedBooks.length > 0 && (
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
MdCheckCircleOutline,
|
||||
} from 'react-icons/md';
|
||||
import { IoShareSocialOutline } from 'react-icons/io5';
|
||||
import { LuFolderPlus } from 'react-icons/lu';
|
||||
import { LuFolderPlus, LuLanguages } from 'react-icons/lu';
|
||||
import { useKeyDownActions } from '@/hooks/useKeyDownActions';
|
||||
import { useTranslation } from '@/hooks/useTranslation';
|
||||
import { isMd5 } from '@/utils/md5';
|
||||
@@ -25,6 +25,7 @@ interface SelectModeActionsProps {
|
||||
onGroup: () => void;
|
||||
onDetails: () => void;
|
||||
onStatus: () => void;
|
||||
onBilingualFilter: () => void;
|
||||
// The macOS / iPad share popover is anchored to the selected book's
|
||||
// cover (located via its data-book-hash attribute), not to this
|
||||
// button — the user's visual focus is on the cover they just tapped.
|
||||
@@ -32,6 +33,7 @@ interface SelectModeActionsProps {
|
||||
onSend: () => void;
|
||||
onDelete: () => void;
|
||||
onCancel: () => void;
|
||||
bilingualFilterEnabled?: boolean;
|
||||
}
|
||||
|
||||
const SelectModeActions: React.FC<SelectModeActionsProps> = ({
|
||||
@@ -42,9 +44,11 @@ const SelectModeActions: React.FC<SelectModeActionsProps> = ({
|
||||
onGroup,
|
||||
onDetails,
|
||||
onStatus,
|
||||
onBilingualFilter,
|
||||
onSend,
|
||||
onDelete,
|
||||
onCancel,
|
||||
bilingualFilterEnabled = false,
|
||||
}) => {
|
||||
const _ = useTranslation();
|
||||
|
||||
@@ -110,13 +114,21 @@ const SelectModeActions: React.FC<SelectModeActionsProps> = ({
|
||||
<MdInfoOutline />
|
||||
<div>{_('Details')}</div>
|
||||
</button>
|
||||
<button
|
||||
onClick={onBilingualFilter}
|
||||
className={clsx(
|
||||
'flex flex-col items-center justify-center gap-1',
|
||||
!bilingualFilterEnabled && 'btn-disabled opacity-50',
|
||||
)}
|
||||
>
|
||||
<LuLanguages />
|
||||
<div>{_('Bilingual')}</div>
|
||||
</button>
|
||||
{sendEnabled && (
|
||||
<button
|
||||
onClick={onSend}
|
||||
className={clsx(
|
||||
'flex flex-col items-center justify-center gap-1',
|
||||
// Wraps to the start of the second row on narrow viewports.
|
||||
'max-[500px]:col-start-1',
|
||||
(!hasSingleSelection || !hasValidBooks) && 'btn-disabled opacity-50',
|
||||
)}
|
||||
>
|
||||
@@ -128,12 +140,6 @@ const SelectModeActions: React.FC<SelectModeActionsProps> = ({
|
||||
onClick={onDelete}
|
||||
className={clsx(
|
||||
'flex flex-col items-center justify-center gap-1',
|
||||
// Without Send (Linux/Windows/web), Delete needs an explicit
|
||||
// col-start-2 so the wrapped row {Delete, Cancel} stays centred
|
||||
// under the 4-col grid. With Send present, the layout is
|
||||
// {Send, Delete, Cancel} starting at col-start-1, so Delete
|
||||
// naturally lands in col-start-2 without an override.
|
||||
!sendEnabled && 'max-[500px]:col-start-2',
|
||||
!hasSelection && 'btn-disabled opacity-50',
|
||||
)}
|
||||
>
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
import { makeSafeFilename } from '@/utils/misc';
|
||||
import { getBaseFilename } from '@/utils/path';
|
||||
|
||||
export type BilingualFilterLanguage = 'ja' | 'zh';
|
||||
export type BilingualFilterMode = 'auto' | 'style' | 'script';
|
||||
|
||||
export interface BilingualFilterOptions {
|
||||
removeLanguage: BilingualFilterLanguage;
|
||||
mode?: BilingualFilterMode;
|
||||
removeUnknown?: boolean;
|
||||
}
|
||||
|
||||
export interface BilingualFilterFileStats {
|
||||
path: string;
|
||||
paragraphs: number;
|
||||
removed: number;
|
||||
kept: number;
|
||||
unknown: number;
|
||||
anchorsMoved: number;
|
||||
byLanguage: Record<BilingualFilterLanguage | 'unknown', number>;
|
||||
}
|
||||
|
||||
export interface BilingualFilterStats {
|
||||
filesSeen: number;
|
||||
htmlFiles: number;
|
||||
changedFiles: number;
|
||||
paragraphs: number;
|
||||
removed: number;
|
||||
kept: number;
|
||||
unknown: number;
|
||||
anchorsMoved: number;
|
||||
fileStats: BilingualFilterFileStats[];
|
||||
}
|
||||
|
||||
export interface BilingualFilterResult {
|
||||
file: File;
|
||||
filename: string;
|
||||
title: string;
|
||||
stats: BilingualFilterStats;
|
||||
}
|
||||
|
||||
const HTML_EXTENSIONS = ['.html', '.htm', '.xhtml'];
|
||||
const TEXT_BLOCK_RE = /<p\b[^>]*>.*?<\/p>/gis;
|
||||
const BODY_END_RE = /<\/body\s*>/i;
|
||||
const ANCHOR_RE =
|
||||
/<a\b(?=[^>]*(?:\bid\s*=\s*['"][^'"]+['"]|\bname\s*=\s*['"][^'"]+['"]))[^>]*>\s*<\/a\s*>/gis;
|
||||
const TAG_RE = /<[^>]+>/g;
|
||||
const OPF_EXT = '.opf';
|
||||
|
||||
const makeEmptyFileStats = (path: string): BilingualFilterFileStats => ({
|
||||
path,
|
||||
paragraphs: 0,
|
||||
removed: 0,
|
||||
kept: 0,
|
||||
unknown: 0,
|
||||
anchorsMoved: 0,
|
||||
byLanguage: { ja: 0, zh: 0, unknown: 0 },
|
||||
});
|
||||
|
||||
const makeEmptyRunStats = (): BilingualFilterStats => ({
|
||||
filesSeen: 0,
|
||||
htmlFiles: 0,
|
||||
changedFiles: 0,
|
||||
paragraphs: 0,
|
||||
removed: 0,
|
||||
kept: 0,
|
||||
unknown: 0,
|
||||
anchorsMoved: 0,
|
||||
fileStats: [],
|
||||
});
|
||||
|
||||
const isHtmlPath = (path: string) => {
|
||||
const lower = path.toLowerCase();
|
||||
return HTML_EXTENSIONS.some((ext) => lower.endsWith(ext));
|
||||
};
|
||||
|
||||
const decodeHtmlEntities = (text: string) => {
|
||||
if (typeof document !== 'undefined') {
|
||||
const textarea = document.createElement('textarea');
|
||||
textarea.innerHTML = text;
|
||||
return textarea.value;
|
||||
}
|
||||
return text
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'");
|
||||
};
|
||||
|
||||
const stripTags = (fragment: string) => decodeHtmlEntities(fragment.replace(TAG_RE, '')).trim();
|
||||
|
||||
const getOpeningTag = (fragment: string) => fragment.match(/^<p\b[^>]*>/is)?.[0] ?? '';
|
||||
|
||||
const isImageOrEmptyParagraph = (fragment: string) => {
|
||||
if (stripTags(fragment)) return false;
|
||||
return /<(img|svg|math)\b/i.test(fragment);
|
||||
};
|
||||
|
||||
const getStyleAttribute = (openingTag: string) =>
|
||||
openingTag.match(/\bstyle\s*=\s*(["'])(.*?)\1/is)?.[2] ?? '';
|
||||
|
||||
const getClassAttribute = (openingTag: string) =>
|
||||
openingTag.match(/\bclass\s*=\s*(["'])(.*?)\1/is)?.[2] ?? '';
|
||||
|
||||
const hasGrayColor = (style: string) => {
|
||||
const compact = style.replace(/\s+/g, '').toLowerCase();
|
||||
return (
|
||||
/(^|;)color:(gray|grey)(;|$)/i.test(style) ||
|
||||
compact.includes('color:#808080') ||
|
||||
compact.includes('color:#888') ||
|
||||
compact.includes('color:rgb(128,128,128)') ||
|
||||
compact.includes('color:rgba(128,128,128,')
|
||||
);
|
||||
};
|
||||
|
||||
const isProbablyJapaneseByStyle = (fragment: string) => {
|
||||
const openingTag = getOpeningTag(fragment);
|
||||
const style = getStyleAttribute(openingTag);
|
||||
const className = getClassAttribute(openingTag).toLowerCase();
|
||||
return (
|
||||
/\bopacity\s*:\s*0(?:\.40*|\.4|\.5|\.50*)\b/i.test(style) ||
|
||||
hasGrayColor(style) ||
|
||||
/\b(japanese|source|original|src|ja|jp)\b/.test(className)
|
||||
);
|
||||
};
|
||||
|
||||
const isProbablyChineseByStyle = (fragment: string) => {
|
||||
const openingTag = getOpeningTag(fragment);
|
||||
const className = getClassAttribute(openingTag).toLowerCase();
|
||||
return /\b(chinese|translation|translated|target|zh|cn)\b/.test(className);
|
||||
};
|
||||
|
||||
const scriptCounts = (text: string) => {
|
||||
const counts = { hiragana: 0, katakana: 0, kana: 0, cjk: 0, ascii: 0 };
|
||||
for (const ch of text) {
|
||||
const code = ch.codePointAt(0) ?? 0;
|
||||
if (code >= 0x3040 && code <= 0x309f) {
|
||||
counts.hiragana += 1;
|
||||
counts.kana += 1;
|
||||
} else if (
|
||||
(code >= 0x30a0 && code <= 0x30ff) ||
|
||||
(code >= 0x31f0 && code <= 0x31ff) ||
|
||||
(code >= 0xff66 && code <= 0xff9d)
|
||||
) {
|
||||
counts.katakana += 1;
|
||||
counts.kana += 1;
|
||||
} else if (
|
||||
(code >= 0x4e00 && code <= 0x9fff) ||
|
||||
(code >= 0x3400 && code <= 0x4dbf) ||
|
||||
(code >= 0xf900 && code <= 0xfaff)
|
||||
) {
|
||||
counts.cjk += 1;
|
||||
} else if (/^[a-z]$/i.test(ch)) {
|
||||
counts.ascii += 1;
|
||||
}
|
||||
}
|
||||
return counts;
|
||||
};
|
||||
|
||||
const classifyByScript = (fragment: string): BilingualFilterLanguage | 'unknown' => {
|
||||
const text = stripTags(fragment);
|
||||
if (!text) return 'unknown';
|
||||
|
||||
const counts = scriptCounts(text);
|
||||
const meaningful = counts.kana + counts.cjk + counts.ascii;
|
||||
if (meaningful === 0) return 'unknown';
|
||||
|
||||
if (counts.kana >= 2) return 'ja';
|
||||
if (counts.kana === 1 && counts.cjk <= 2) return 'ja';
|
||||
if (counts.cjk >= 2 && counts.kana === 0) return 'zh';
|
||||
|
||||
return 'unknown';
|
||||
};
|
||||
|
||||
const classifyParagraph = (
|
||||
fragment: string,
|
||||
mode: BilingualFilterMode,
|
||||
): BilingualFilterLanguage | 'unknown' => {
|
||||
if (isImageOrEmptyParagraph(fragment)) return 'unknown';
|
||||
if (mode === 'auto' || mode === 'style') {
|
||||
if (isProbablyJapaneseByStyle(fragment)) return 'ja';
|
||||
if (isProbablyChineseByStyle(fragment)) return 'zh';
|
||||
}
|
||||
if (mode === 'style') {
|
||||
return stripTags(fragment) ? 'zh' : 'unknown';
|
||||
}
|
||||
return classifyByScript(fragment);
|
||||
};
|
||||
|
||||
const looksLikeStyleBilingual = (text: string) => {
|
||||
const paragraphs = text.match(TEXT_BLOCK_RE) ?? [];
|
||||
if (paragraphs.length < 10) return false;
|
||||
const textBlocks = paragraphs.filter((paragraph) => stripTags(paragraph)).length;
|
||||
if (textBlocks === 0) return false;
|
||||
const styled = paragraphs.filter(isProbablyJapaneseByStyle).length;
|
||||
const ratio = styled / textBlocks;
|
||||
return styled >= 5 && ratio >= 0.15 && ratio <= 0.85;
|
||||
};
|
||||
|
||||
const extractEmptyAnchors = (fragment: string) => fragment.match(ANCHOR_RE) ?? [];
|
||||
|
||||
const insertAnchorsIntoParagraph = (paragraph: string, anchors: string[]) => {
|
||||
if (anchors.length === 0) return paragraph;
|
||||
const insertion = anchors.join('');
|
||||
const match = paragraph.match(/^<p\b[^>]*>/is);
|
||||
if (!match) return insertion + paragraph;
|
||||
const index = match[0].length;
|
||||
return paragraph.slice(0, index) + insertion + paragraph.slice(index);
|
||||
};
|
||||
|
||||
const insertPendingAnchorsBeforeBodyEnd = (text: string, anchors: string[]) => {
|
||||
if (anchors.length === 0) return text;
|
||||
const fallback = `<p style="display:none;">${anchors.join('')}</p>\n`;
|
||||
const match = text.match(BODY_END_RE);
|
||||
if (!match || match.index === undefined) return `${text}\n${fallback}`;
|
||||
return text.slice(0, match.index) + fallback + text.slice(match.index);
|
||||
};
|
||||
|
||||
const shouldRemove = (
|
||||
language: BilingualFilterLanguage | 'unknown',
|
||||
removeLanguage: BilingualFilterLanguage,
|
||||
removeUnknown: boolean,
|
||||
) => language === removeLanguage || (removeUnknown && language === 'unknown');
|
||||
|
||||
const filterHtmlText = (text: string, options: Required<BilingualFilterOptions>, path: string) => {
|
||||
const stats = makeEmptyFileStats(path);
|
||||
const output: string[] = [];
|
||||
let pendingAnchors: string[] = [];
|
||||
let position = 0;
|
||||
let changed = false;
|
||||
const mode = options.mode === 'auto' && looksLikeStyleBilingual(text) ? 'style' : options.mode;
|
||||
|
||||
for (const match of text.matchAll(TEXT_BLOCK_RE)) {
|
||||
if (match.index === undefined) continue;
|
||||
output.push(text.slice(position, match.index));
|
||||
|
||||
let paragraph = match[0];
|
||||
const language = classifyParagraph(paragraph, mode);
|
||||
stats.paragraphs += 1;
|
||||
stats.byLanguage[language] += 1;
|
||||
|
||||
if (shouldRemove(language, options.removeLanguage, options.removeUnknown)) {
|
||||
const anchors = extractEmptyAnchors(paragraph);
|
||||
pendingAnchors = [...pendingAnchors, ...anchors];
|
||||
stats.removed += 1;
|
||||
stats.anchorsMoved += anchors.length;
|
||||
changed = true;
|
||||
} else {
|
||||
if (language === 'unknown') stats.unknown += 1;
|
||||
stats.kept += 1;
|
||||
if (pendingAnchors.length > 0) {
|
||||
paragraph = insertAnchorsIntoParagraph(paragraph, pendingAnchors);
|
||||
pendingAnchors = [];
|
||||
changed = true;
|
||||
}
|
||||
output.push(paragraph);
|
||||
}
|
||||
|
||||
position = match.index + match[0].length;
|
||||
}
|
||||
|
||||
output.push(text.slice(position));
|
||||
let newText = output.join('');
|
||||
if (pendingAnchors.length > 0) {
|
||||
newText = insertPendingAnchorsBeforeBodyEnd(newText, pendingAnchors);
|
||||
changed = true;
|
||||
}
|
||||
|
||||
return { text: changed ? newText : text, stats };
|
||||
};
|
||||
|
||||
const escapeXml = (value: string) =>
|
||||
value
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
|
||||
const decodeText = (data: Uint8Array) => {
|
||||
const encodings = ['utf-8', 'shift_jis', 'gb18030'];
|
||||
for (const encoding of encodings) {
|
||||
try {
|
||||
const decoder = new TextDecoder(encoding, { fatal: true });
|
||||
return decoder.decode(data);
|
||||
} catch {
|
||||
// Try the next common EPUB encoding.
|
||||
}
|
||||
}
|
||||
return new TextDecoder('utf-8').decode(data);
|
||||
};
|
||||
|
||||
const setDocumentLanguageAndTitle = (text: string, title: string, language: string) => {
|
||||
let updated = text;
|
||||
if (/<title\b[^>]*>.*?<\/title>/is.test(updated)) {
|
||||
updated = updated.replace(/<title\b[^>]*>.*?<\/title>/is, `<title>${escapeXml(title)}</title>`);
|
||||
}
|
||||
if (/\bxml:lang\s*=\s*["'][^"']*["']/i.test(updated)) {
|
||||
updated = updated.replace(/\bxml:lang\s*=\s*(["'])[^"']*\1/i, `xml:lang="${language}"`);
|
||||
}
|
||||
if (/\blang\s*=\s*["'][^"']*["']/i.test(updated)) {
|
||||
updated = updated.replace(/\blang\s*=\s*(["'])[^"']*\1/i, `lang="${language}"`);
|
||||
} else {
|
||||
updated = updated.replace(/<html\b/i, `<html lang="${language}"`);
|
||||
}
|
||||
return updated;
|
||||
};
|
||||
|
||||
const extractOpfTitle = (text: string) => {
|
||||
const match = text.match(/<dc:title\b[^>]*>(.*?)<\/dc:title>/is);
|
||||
return match ? stripTags(match[1] ?? '') : '';
|
||||
};
|
||||
|
||||
const getVariantInfo = (
|
||||
sourceName: string,
|
||||
removeLanguage: BilingualFilterLanguage,
|
||||
metadataTitle?: string,
|
||||
) => {
|
||||
const sourceTitle = metadataTitle?.trim() || getBaseFilename(sourceName);
|
||||
const suffix = removeLanguage === 'ja' ? '简中译本' : '日文原文版';
|
||||
const language = removeLanguage === 'ja' ? 'zh-CN' : 'ja';
|
||||
const title = `${sourceTitle}(${suffix})`;
|
||||
const filename = `${makeSafeFilename(`${sourceTitle}_${suffix}`)}.epub`;
|
||||
const identifierSuffix = removeLanguage === 'ja' ? 'readest-no-ja' : 'readest-no-zh';
|
||||
return { title, filename, language, identifierSuffix };
|
||||
};
|
||||
|
||||
const patchOpfMetadata = (
|
||||
text: string,
|
||||
variant: ReturnType<typeof getVariantInfo>,
|
||||
removeLanguage: BilingualFilterLanguage,
|
||||
) => {
|
||||
let updated = text;
|
||||
const title = escapeXml(variant.title);
|
||||
if (/<dc:title\b[^>]*>.*?<\/dc:title>/is.test(updated)) {
|
||||
updated = updated.replace(
|
||||
/<dc:title\b([^>]*)>.*?<\/dc:title>/is,
|
||||
`<dc:title$1>${title}</dc:title>`,
|
||||
);
|
||||
}
|
||||
if (/<dc:language\b[^>]*>.*?<\/dc:language>/is.test(updated)) {
|
||||
updated = updated.replace(
|
||||
/<dc:language\b([^>]*)>.*?<\/dc:language>/is,
|
||||
`<dc:language$1>${variant.language}</dc:language>`,
|
||||
);
|
||||
}
|
||||
if (/<dc:identifier\b[^>]*>.*?<\/dc:identifier>/is.test(updated)) {
|
||||
updated = updated.replace(
|
||||
/<dc:identifier\b([^>]*)>(.*?)<\/dc:identifier>/gis,
|
||||
(_match, attrs: string, value: string) => {
|
||||
const trimmed = value.trim();
|
||||
const suffix = `:${variant.identifierSuffix}`;
|
||||
const nextValue = trimmed.includes(suffix) ? trimmed : `${trimmed}${suffix}`;
|
||||
return `<dc:identifier${attrs}>${escapeXml(nextValue)}</dc:identifier>`;
|
||||
},
|
||||
);
|
||||
} else {
|
||||
updated = updated.replace(
|
||||
/<\/metadata\s*>/i,
|
||||
`<dc:identifier id="readest-filter-id">readest:${removeLanguage}:${Date.now()}</dc:identifier></metadata>`,
|
||||
);
|
||||
}
|
||||
updated = updated.replace(
|
||||
/<meta\b([^>]*\bproperty\s*=\s*["']dcterms:modified["'][^>]*)>.*?<\/meta>/is,
|
||||
`<meta$1>${new Date().toISOString().replace(/\.\d{3}Z$/, 'Z')}</meta>`,
|
||||
);
|
||||
return updated;
|
||||
};
|
||||
|
||||
const updateRunStats = (runStats: BilingualFilterStats, fileStats: BilingualFilterFileStats) => {
|
||||
runStats.fileStats.push(fileStats);
|
||||
runStats.paragraphs += fileStats.paragraphs;
|
||||
runStats.removed += fileStats.removed;
|
||||
runStats.kept += fileStats.kept;
|
||||
runStats.unknown += fileStats.unknown;
|
||||
runStats.anchorsMoved += fileStats.anchorsMoved;
|
||||
};
|
||||
|
||||
export async function filterBilingualEpubFile(
|
||||
sourceFile: File,
|
||||
filterOptions: BilingualFilterOptions,
|
||||
): Promise<BilingualFilterResult> {
|
||||
const options: Required<BilingualFilterOptions> = {
|
||||
mode: 'auto',
|
||||
removeUnknown: false,
|
||||
...filterOptions,
|
||||
};
|
||||
const {
|
||||
BlobReader,
|
||||
BlobWriter,
|
||||
TextReader,
|
||||
Uint8ArrayReader,
|
||||
Uint8ArrayWriter,
|
||||
ZipReader,
|
||||
ZipWriter,
|
||||
} = await import('@zip.js/zip.js');
|
||||
|
||||
const reader = new ZipReader(new BlobReader(sourceFile));
|
||||
const writer = new ZipWriter(new BlobWriter('application/epub+zip'));
|
||||
const runStats = makeEmptyRunStats();
|
||||
|
||||
try {
|
||||
const entries = await reader.getEntries();
|
||||
const opfEntry = entries.find((entry) => entry.filename.toLowerCase().endsWith(OPF_EXT));
|
||||
const opfTitle =
|
||||
opfEntry?.getData && !opfEntry.directory
|
||||
? extractOpfTitle(decodeText(await opfEntry.getData(new Uint8ArrayWriter())))
|
||||
: '';
|
||||
const variant = getVariantInfo(sourceFile.name, options.removeLanguage, opfTitle);
|
||||
const seen = new Set<string>();
|
||||
const mimetype = entries.find((entry) => entry.filename.toLowerCase() === 'mimetype');
|
||||
if (mimetype) {
|
||||
await writer.add('mimetype', new TextReader('application/epub+zip'), { level: 0 });
|
||||
seen.add(mimetype.filename);
|
||||
runStats.filesSeen += 1;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (seen.has(entry.filename) || entry.directory || !entry.getData) continue;
|
||||
seen.add(entry.filename);
|
||||
runStats.filesSeen += 1;
|
||||
|
||||
const lowerName = entry.filename.toLowerCase();
|
||||
const rawData = await entry.getData(new Uint8ArrayWriter());
|
||||
let outData: Uint8Array | string = rawData;
|
||||
|
||||
if (isHtmlPath(lowerName)) {
|
||||
runStats.htmlFiles += 1;
|
||||
const originalText = decodeText(rawData);
|
||||
const filtered = filterHtmlText(originalText, options, entry.filename);
|
||||
const patchedText = setDocumentLanguageAndTitle(
|
||||
filtered.text,
|
||||
variant.title,
|
||||
variant.language,
|
||||
);
|
||||
updateRunStats(runStats, filtered.stats);
|
||||
if (patchedText !== originalText) runStats.changedFiles += 1;
|
||||
outData = patchedText;
|
||||
} else if (lowerName.endsWith(OPF_EXT)) {
|
||||
const originalText = decodeText(rawData);
|
||||
const patchedText = patchOpfMetadata(originalText, variant, options.removeLanguage);
|
||||
if (patchedText !== originalText) runStats.changedFiles += 1;
|
||||
outData = patchedText;
|
||||
}
|
||||
|
||||
const writerReader =
|
||||
typeof outData === 'string'
|
||||
? new TextReader(outData)
|
||||
: new Uint8ArrayReader(outData as Uint8Array);
|
||||
await writer.add(entry.filename, writerReader, zipOptionsForEntry(entry));
|
||||
}
|
||||
|
||||
const blob = await writer.close();
|
||||
const file = new File([blob], variant.filename, { type: 'application/epub+zip' });
|
||||
return { file, filename: variant.filename, title: variant.title, stats: runStats };
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
}
|
||||
|
||||
const zipOptionsForEntry = (entry: import('@zip.js/zip.js').Entry) =>
|
||||
entry.filename.toLowerCase() === 'mimetype' ? { level: 0 } : {};
|
||||
Reference in New Issue
Block a user