compat: detect book language from text for invalid language code in metadata (#1743)

apps/readest-app/src/app/reader/components/FoliateViewer.tsx

@@ -105,7 +105,7 @@ const FoliateViewer: React.FC<{
               bookKey,
               viewSettings,
               content: data,
-              transformers: ['punctuation', 'footnote'],
+              transformers: ['punctuation', 'footnote', 'language'],
             };
             return Promise.resolve(transformContent(ctx));
           }

apps/readest-app/src/services/transformers/index.ts

@@ -1,9 +1,11 @@
 import type { Transformer } from './types';
 import { footnoteTransformer } from './footnote';
+import { languageTransformer } from './language';
 import { punctuationTransformer } from './punctuation';
 
 export const availableTransformers: Transformer[] = [
   punctuationTransformer,
   footnoteTransformer,
+  languageTransformer,
   // Add more transformers here
 ];

apps/readest-app/src/services/transformers/language.ts

@@ -0,0 +1,28 @@
+import { detectLanguage, isValidLang } from '@/utils/lang';
+import type { Transformer } from './types';
+
+export const languageTransformer: Transformer = {
+  name: 'language',
+
+  transform: async (ctx) => {
+    let result = ctx.content;
+    const attrsMatch = result.match(/<html\b([^>]*)>/i);
+    if (attrsMatch) {
+      let attrs = attrsMatch[1] || '';
+      const langRegex = / lang="([^"]*)"/i;
+      const xmlLangRegex = / xml:lang="([^"]*)"/i;
+      const xmlLangMatch = attrs.match(xmlLangRegex);
+      const langMatch = attrs.match(langRegex);
+      if (!isValidLang(langMatch?.[1] || xmlLangMatch?.[1])) {
+        const mainContent = result.replace(/<[^>]+>/g, ' ');
+        const lang = detectLanguage(mainContent);
+        const newLangAttr = ` lang="${lang}"`;
+        const newXmlLangAttr = ` xml:lang="${lang}"`;
+        attrs = langMatch ? attrs.replace(langRegex, newLangAttr) : attrs + newLangAttr;
+        attrs = xmlLangMatch ? attrs.replace(xmlLangRegex, newXmlLangAttr) : attrs + newXmlLangAttr;
+        result = result.replace(attrsMatch[0], `<html${attrs}>`);
+      }
+    }
+    return result;
+  },
+};

apps/readest-app/src/store/readerStore.ts

@@ -138,6 +138,10 @@ export const useReaderStore = create<ReaderStore>((set, get) => ({
         console.log('Loading book', key);
         const { book: bookDoc } = await new DocumentLoader(file).open();
         updateToc(bookDoc, config.viewSettings?.sortedTOC ?? false);
+        if (!bookDoc.metadata.title) {
+          bookDoc.metadata.title = getBaseFilename(file.name);
+        }
+        book.sourceTitle = formatTitle(bookDoc.metadata.title);
         // Correct language codes mistakenly set with language names
         if (typeof bookDoc.metadata?.language === 'string') {
           if (bookDoc.metadata.language in SUPPORTED_LANGNAMES) {
@@ -145,10 +149,6 @@ export const useReaderStore = create<ReaderStore>((set, get) => ({
           }
         }
         // Set the book's language for formerly imported books, newly imported books have this field set
-        if (!bookDoc.metadata.title) {
-          bookDoc.metadata.title = getBaseFilename(file.name);
-        }
-        book.sourceTitle = formatTitle(bookDoc.metadata.title);
         const primaryLanguage = getPrimaryLanguage(bookDoc.metadata.language);
         book.primaryLanguage = book.primaryLanguage ?? primaryLanguage;
         book.metadata = book.metadata ?? bookDoc.metadata;

apps/readest-app/src/utils/book.ts

@@ -1,9 +1,10 @@
 import { EXTS } from '@/libs/document';
 import { Book, BookConfig, BookProgress, WritingMode } from '@/types/book';
+import { SUPPORTED_LANGS } from '@/services/constants';
 import { getUserLang, isContentURI, isFileURI, isValidURL, makeSafeFilename } from './misc';
 import { getStorageType } from './storage';
 import { getDirFromLanguage } from './rtl';
-import { SUPPORTED_LANGS } from '@/services/constants';
+import { code6392to6391, isValidLang, normalizedLangCode } from './lang';
 
 export const getDir = (book: Book) => {
   return `${book.hash}`;
@@ -137,8 +138,14 @@ export const formatLanguage = (lang: string | string[] | undefined): string => {
     : langCodeToLangName(lang || '');
 };
 
+// Should return valid ISO-639-1 language code, fallback to 'en' if not valid
 export const getPrimaryLanguage = (lang: string | string[] | undefined) => {
-  return Array.isArray(lang) ? lang[0] : lang;
+  const primaryLang = Array.isArray(lang) ? lang[0] : lang;
+  if (isValidLang(primaryLang)) {
+    const normalizedLang = normalizedLangCode(primaryLang);
+    return code6392to6391(normalizedLang) || normalizedLang;
+  }
+  return 'en';
 };
 
 export const formatDate = (date: string | number | Date | null | undefined, isUTC = false) => {

apps/readest-app/src/utils/lang.ts

@@ -8,8 +8,8 @@ export const isCJKStr = (str: string) => {
 
 export const isCJKLang = (lang: string | null | undefined): boolean => {
   if (!lang) return false;
-  const normalizedLang = lang.split('-')[0]!.toLowerCase();
-  return ['zh', 'ja', 'ko'].includes(normalizedLang);
+  const normalizedLang = normalizedLangCode(lang);
+  return ['zh', 'ja', 'ko', 'zho', 'jpn', 'kor'].includes(normalizedLang);
 };
 
 export const normalizeToFullLang = (langCode: string): string => {
@@ -84,7 +84,8 @@ export const isSameLang = (lang1?: string | null, lang2?: string | null): boolea
 
 export const isValidLang = (lang?: string) => {
   if (!lang) return false;
-  const code = lang.split('-')[0]!.toLowerCase();
+  if (typeof lang !== 'string') return false;
+  const code = normalizedLangCode(lang);
   return iso6392.some((l) => l.iso6391 === code || l.iso6392B === code);
 };