From 3e5e4d29464453672adaf1e3712a7d6b96eda14d Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Mon, 4 Aug 2025 20:55:18 +0800 Subject: [PATCH] compat: detect book language from text for invalid language code in metadata (#1743) --- .../app/reader/components/FoliateViewer.tsx | 2 +- .../src/services/transformers/index.ts | 2 ++ .../src/services/transformers/language.ts | 28 +++++++++++++++++++ apps/readest-app/src/store/readerStore.ts | 8 +++--- apps/readest-app/src/utils/book.ts | 11 ++++++-- apps/readest-app/src/utils/lang.ts | 7 +++-- 6 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 apps/readest-app/src/services/transformers/language.ts diff --git a/apps/readest-app/src/app/reader/components/FoliateViewer.tsx b/apps/readest-app/src/app/reader/components/FoliateViewer.tsx index cbaffb7f..d71dec94 100644 --- a/apps/readest-app/src/app/reader/components/FoliateViewer.tsx +++ b/apps/readest-app/src/app/reader/components/FoliateViewer.tsx @@ -105,7 +105,7 @@ const FoliateViewer: React.FC<{ bookKey, viewSettings, content: data, - transformers: ['punctuation', 'footnote'], + transformers: ['punctuation', 'footnote', 'language'], }; return Promise.resolve(transformContent(ctx)); } diff --git a/apps/readest-app/src/services/transformers/index.ts b/apps/readest-app/src/services/transformers/index.ts index b73d380c..71eba61d 100644 --- a/apps/readest-app/src/services/transformers/index.ts +++ b/apps/readest-app/src/services/transformers/index.ts @@ -1,9 +1,11 @@ import type { Transformer } from './types'; import { footnoteTransformer } from './footnote'; +import { languageTransformer } from './language'; import { punctuationTransformer } from './punctuation'; export const availableTransformers: Transformer[] = [ punctuationTransformer, footnoteTransformer, + languageTransformer, // Add more transformers here ]; diff --git a/apps/readest-app/src/services/transformers/language.ts b/apps/readest-app/src/services/transformers/language.ts new file mode 100644 index 00000000..dc0c1ee7 --- /dev/null +++ b/apps/readest-app/src/services/transformers/language.ts @@ -0,0 +1,28 @@ +import { detectLanguage, isValidLang } from '@/utils/lang'; +import type { Transformer } from './types'; + +export const languageTransformer: Transformer = { + name: 'language', + + transform: async (ctx) => { + let result = ctx.content; + const attrsMatch = result.match(/]*)>/i); + if (attrsMatch) { + let attrs = attrsMatch[1] || ''; + const langRegex = / lang="([^"]*)"/i; + const xmlLangRegex = / xml:lang="([^"]*)"/i; + const xmlLangMatch = attrs.match(xmlLangRegex); + const langMatch = attrs.match(langRegex); + if (!isValidLang(langMatch?.[1] || xmlLangMatch?.[1])) { + const mainContent = result.replace(/<[^>]+>/g, ' '); + const lang = detectLanguage(mainContent); + const newLangAttr = ` lang="${lang}"`; + const newXmlLangAttr = ` xml:lang="${lang}"`; + attrs = langMatch ? attrs.replace(langRegex, newLangAttr) : attrs + newLangAttr; + attrs = xmlLangMatch ? attrs.replace(xmlLangRegex, newXmlLangAttr) : attrs + newXmlLangAttr; + result = result.replace(attrsMatch[0], ``); + } + } + return result; + }, +}; diff --git a/apps/readest-app/src/store/readerStore.ts b/apps/readest-app/src/store/readerStore.ts index 6cafe96b..dee0b4be 100644 --- a/apps/readest-app/src/store/readerStore.ts +++ b/apps/readest-app/src/store/readerStore.ts @@ -138,6 +138,10 @@ export const useReaderStore = create((set, get) => ({ console.log('Loading book', key); const { book: bookDoc } = await new DocumentLoader(file).open(); updateToc(bookDoc, config.viewSettings?.sortedTOC ?? false); + if (!bookDoc.metadata.title) { + bookDoc.metadata.title = getBaseFilename(file.name); + } + book.sourceTitle = formatTitle(bookDoc.metadata.title); // Correct language codes mistakenly set with language names if (typeof bookDoc.metadata?.language === 'string') { if (bookDoc.metadata.language in SUPPORTED_LANGNAMES) { @@ -145,10 +149,6 @@ export const useReaderStore = create((set, get) => ({ } } // Set the book's language for formerly imported books, newly imported books have this field set - if (!bookDoc.metadata.title) { - bookDoc.metadata.title = getBaseFilename(file.name); - } - book.sourceTitle = formatTitle(bookDoc.metadata.title); const primaryLanguage = getPrimaryLanguage(bookDoc.metadata.language); book.primaryLanguage = book.primaryLanguage ?? primaryLanguage; book.metadata = book.metadata ?? bookDoc.metadata; diff --git a/apps/readest-app/src/utils/book.ts b/apps/readest-app/src/utils/book.ts index 8cfbd8f5..a19085ac 100644 --- a/apps/readest-app/src/utils/book.ts +++ b/apps/readest-app/src/utils/book.ts @@ -1,9 +1,10 @@ import { EXTS } from '@/libs/document'; import { Book, BookConfig, BookProgress, WritingMode } from '@/types/book'; +import { SUPPORTED_LANGS } from '@/services/constants'; import { getUserLang, isContentURI, isFileURI, isValidURL, makeSafeFilename } from './misc'; import { getStorageType } from './storage'; import { getDirFromLanguage } from './rtl'; -import { SUPPORTED_LANGS } from '@/services/constants'; +import { code6392to6391, isValidLang, normalizedLangCode } from './lang'; export const getDir = (book: Book) => { return `${book.hash}`; @@ -137,8 +138,14 @@ export const formatLanguage = (lang: string | string[] | undefined): string => { : langCodeToLangName(lang || ''); }; +// Should return valid ISO-639-1 language code, fallback to 'en' if not valid export const getPrimaryLanguage = (lang: string | string[] | undefined) => { - return Array.isArray(lang) ? lang[0] : lang; + const primaryLang = Array.isArray(lang) ? lang[0] : lang; + if (isValidLang(primaryLang)) { + const normalizedLang = normalizedLangCode(primaryLang); + return code6392to6391(normalizedLang) || normalizedLang; + } + return 'en'; }; export const formatDate = (date: string | number | Date | null | undefined, isUTC = false) => { diff --git a/apps/readest-app/src/utils/lang.ts b/apps/readest-app/src/utils/lang.ts index e0c05533..729e886f 100644 --- a/apps/readest-app/src/utils/lang.ts +++ b/apps/readest-app/src/utils/lang.ts @@ -8,8 +8,8 @@ export const isCJKStr = (str: string) => { export const isCJKLang = (lang: string | null | undefined): boolean => { if (!lang) return false; - const normalizedLang = lang.split('-')[0]!.toLowerCase(); - return ['zh', 'ja', 'ko'].includes(normalizedLang); + const normalizedLang = normalizedLangCode(lang); + return ['zh', 'ja', 'ko', 'zho', 'jpn', 'kor'].includes(normalizedLang); }; export const normalizeToFullLang = (langCode: string): string => { @@ -84,7 +84,8 @@ export const isSameLang = (lang1?: string | null, lang2?: string | null): boolea export const isValidLang = (lang?: string) => { if (!lang) return false; - const code = lang.split('-')[0]!.toLowerCase(); + if (typeof lang !== 'string') return false; + const code = normalizedLangCode(lang); return iso6392.some((l) => l.iso6391 === code || l.iso6392B === code); };