txt: fix language detection when parsing TXT files, closes #1720 (#1723)

.github/workflows/pull-request.yml

@@ -58,6 +58,11 @@ jobs:
         run: |
           pnpm install && pnpm setup-pdfjs
 
+      - name: run tests
+        working-directory: apps/readest-app
+        run: |
+          pnpm test -- --watchAll=false
+
       - name: build the web App
         if: matrix.config.platform == 'web'
         working-directory: apps/readest-app

apps/readest-app/package.json

@@ -73,11 +73,13 @@
     "cssbeautify": "^0.3.1",
     "dayjs": "^1.11.13",
     "foliate-js": "workspace:*",
+    "franc-min": "^6.2.0",
     "highlight.js": "^11.11.1",
     "i18next": "^24.2.0",
     "i18next-browser-languagedetector": "^8.0.2",
     "i18next-http-backend": "^3.0.1",
     "iso-639-2": "^3.0.2",
+    "iso-639-3": "^3.0.1",
     "js-md5": "^0.8.3",
     "jwt-decode": "^4.0.0",
     "marked": "^15.0.12",

apps/readest-app/src/__tests__/api/iap-verify.test.ts

@@ -3,6 +3,7 @@ import { POST } from '@/app/api/apple/iap-verify/route';
 import { NextRequest } from 'next/server';
 import { setupSupabaseMocks } from '../helpers/supabase-mock';
 
+const SKIP_IAP_API_TESTS = !process.env['ENABLE_IAP_API_TESTS'];
 vi.mock('@/utils/supabase', () => ({
   supabase: {
     auth: {
@@ -14,7 +15,7 @@ vi.mock('@/utils/supabase', () => ({
   createSupabaseAdminClient: vi.fn(),
 }));
 
-describe('/api/apple/iap-verify', () => {
+describe.skipIf(SKIP_IAP_API_TESTS)('/api/apple/iap-verify', () => {
   it('should verify a valid transaction', async () => {
     setupSupabaseMocks();
     const request = new NextRequest('http://localhost:3000/api/apple/iap-verify', {

apps/readest-app/src/__tests__/utils/detect-language.test.ts

@@ -0,0 +1,273 @@
+import { describe, it, expect } from 'vitest';
+
+import { detectLanguage } from '@/utils/lang';
+
+describe('detectLanguage - Result Tests', () => {
+  describe('English text detection', () => {
+    it('should detect English text', () => {
+      const englishTexts = [
+        'This is a sample English text for language detection testing.',
+        'The quick brown fox jumps over the lazy dog.',
+        'Hello world! This is an English sentence with punctuation.',
+        'Chapter 1: Introduction to Language Detection',
+        'In the beginning was the Word, and the Word was with God.',
+        'To be or not to be, that is the question.',
+      ];
+
+      englishTexts.forEach((text) => {
+        const result = detectLanguage(text);
+        expect(result).toBe('en');
+      });
+    });
+  });
+
+  describe('Chinese text detection', () => {
+    it('should detect Chinese text', () => {
+      const chineseTexts = [
+        '这是一个中文文本的示例，用于语言检测测试。',
+        '天下大势，分久必合，合久必分。',
+        '学而时习之，不亦说乎？有朋自远方来，不亦乐乎？',
+        '春眠不觉晓，处处闻啼鸟。夜来风雨声，花落知多少。',
+        '那时候，天空很蓝，云彩很白，我们都还很年轻。',
+        '人工智能技术的发展正在改变我们的生活方式和工作方式。',
+      ];
+
+      chineseTexts.forEach((text) => {
+        const result = detectLanguage(text);
+        expect(result).toBe('zh');
+      });
+    });
+
+    it('should detect Traditional Chinese text', () => {
+      const traditionalChineseTexts = [
+        '這是一個正體中文文本的示例，用於語言檢測測試。',
+        '天下大勢，分久必合，合久必分。',
+        '學而時習之，不亦說乎？有朋自遠方來，不亦樂乎？',
+        '春眠不覺曉，處處聞啼鳥。夜來風雨聲，花落知多少。',
+        '那時候，天空很藍，雲彩很白，我們都還很年輕。',
+        '人工智慧技術的發展正在改變我們的生活方式和工作方式。',
+      ];
+
+      traditionalChineseTexts.forEach((text) => {
+        const result = detectLanguage(text);
+        expect(result).toBe('zh');
+      });
+    });
+  });
+
+  describe('Japanese text detection', () => {
+    it('should detect Japanese text', () => {
+      const japaneseTexts = [
+        'これは日本語のテキストサンプルです。',
+        '第一章：言語検出について',
+        'こんにちは、世界！これは日本語の文章です。',
+        '桜の花が咲く季節になりました。とても美しいです。',
+        'ひらがな、カタカナ、漢字を使って日本語を書きます。',
+        '東京は日本の首都であり、多くの人々が住んでいます。',
+      ];
+
+      japaneseTexts.forEach((text) => {
+        const result = detectLanguage(text);
+        expect(result).toBe('ja');
+      });
+    });
+  });
+
+  describe('French text detection', () => {
+    it('should detect French text', () => {
+      const frenchTexts = [
+        'Ceci est un exemple de texte français pour tester la détection de langue.',
+        'Chapitre 1: Introduction à la détection de langue.',
+        'Bonjour le monde! Ceci est une phrase en français.',
+        'Les Champs-Élysées sont une avenue célèbre à Paris, souvent appelée la plus belle avenue du monde.',
+        'Victor Hugo était un écrivain français très célèbre du XIXe siècle,',
+        "La littérature française possède une richesse extraordinaire qui s'étend sur plusieurs siècles.",
+      ];
+
+      frenchTexts.forEach((text) => {
+        const result = detectLanguage(text);
+        expect(result).toBe('fr');
+      });
+    });
+  });
+
+  describe('Spanish text detection', () => {
+    it('should detect Spanish text', () => {
+      const spanishTexts = [
+        'Este es un ejemplo de texto en español para probar la detección de idioma.',
+        'Capítulo 1: Introducción a la detección de idiomas',
+        '¡Hola mundo! Esta es una oración en español.',
+        'España es un país ubicado en la península ibérica.',
+        'El flamenco es una forma de arte español muy famosa.',
+        'Don Quijote de La Mancha es una obra clásica de la literatura española.',
+      ];
+
+      spanishTexts.forEach((text) => {
+        const result = detectLanguage(text);
+        expect(result).toBe('es');
+      });
+    });
+  });
+
+  describe('German text detection', () => {
+    it('should detect German text', () => {
+      const germanTexts = [
+        'Dies ist ein Beispieltext auf Deutsch für die Spracherkennung.',
+        'Kapitel 1: Einführung in die Spracherkennung',
+        'Hallo Welt! Dies ist ein deutscher Satz.',
+        'Deutschland ist ein Land in Mitteleuropa.',
+        'Das Oktoberfest ist ein berühmtes deutsches Festival.',
+        'Die deutsche Sprache hat viele zusammengesetzte Wörter.',
+      ];
+
+      germanTexts.forEach((text) => {
+        const result = detectLanguage(text);
+        expect(result).toBe('de');
+      });
+    });
+  });
+
+  describe('Arabic text detection', () => {
+    it('should detect Arabic text', () => {
+      const arabicTexts = [
+        'هذا نص عربي للاختبار. يجب أن يتم اكتشاف هذا النص كعربي.',
+        'الفصل الأول: مقدمة في اكتشاف اللغة',
+        'مرحبا بالعالم! هذه جملة باللغة العربية.',
+        'العربية لغة جميلة يتحدث بها ملايين الناس.',
+        'القرآن الكريم نزل باللغة العربية.',
+        'دمشق هي عاصمة سوريا وواحدة من أقدم المدن في العالم.',
+      ];
+
+      arabicTexts.forEach((text) => {
+        const result = detectLanguage(text);
+        expect(result).toBe('ar');
+      });
+    });
+  });
+
+  describe('Korean text detection', () => {
+    it('should detect Korean text', () => {
+      const koreanTexts = [
+        '이것은 언어 감지 테스트를 위한 한국어 텍스트 샘플입니다.',
+        '제1장: 언어 감지 소개',
+        '안녕하세요 세계! 이것은 한국어 문장입니다.',
+        '서울은 대한민국의 수도이며 많은 사람들이 살고 있습니다.',
+        '한글은 세종대왕이 만든 우리나라의 고유한 문자입니다.',
+        '김치는 한국의 전통 음식 중 하나입니다.',
+      ];
+
+      koreanTexts.forEach((text) => {
+        const result = detectLanguage(text);
+        expect(result).toBe('ko');
+      });
+    });
+  });
+
+  describe('fallback behavior', () => {
+    it('should return "en" for very short text', () => {
+      const shortTexts = ['', ' ', 'Hi', 'OK', '123', '!@#'];
+
+      shortTexts.forEach((text) => {
+        const result = detectLanguage(text);
+        expect(result).toBe('en');
+      });
+    });
+
+    it('should return "en" for unrecognizable text', () => {
+      const unrecognizableTexts = [
+        '123456789012345678901234567890',
+        '!@#$%^&*()_+{}|:"<>?[]\\;\',./',
+        '1111111111222222222233333333334444444444',
+      ];
+
+      unrecognizableTexts.forEach((text) => {
+        const result = detectLanguage(text);
+        expect(result).toBe('en');
+      });
+    });
+  });
+
+  describe('mixed content', () => {
+    it('should detect predominant language in mixed content', () => {
+      const testCases = [
+        {
+          text: 'This is mostly English text with some 中文 words mixed in but English dominates.',
+          expected: 'en',
+        },
+        {
+          text: '这主要是中文文本，虽然有一些 English words 混合在其中，但中文占主导地位。',
+          expected: 'zh',
+        },
+        {
+          text: 'Ceci est principalement du français avec quelques English words mélangés.',
+          expected: 'fr',
+        },
+      ];
+
+      testCases.forEach(({ text, expected }) => {
+        const result = detectLanguage(text);
+        expect(result).toBe(expected);
+      });
+    });
+  });
+
+  describe('long text handling', () => {
+    it('should handle long text (over 1000 characters)', () => {
+      const longEnglishText = 'This is a very long English text. '.repeat(100); // ~3400 chars
+      const longChineseText = '这是一个很长的中文文本。'.repeat(100); // ~1200 chars
+      const longFrenchText = 'Ceci est un très long texte français. '.repeat(100); // ~3800 chars
+
+      expect(detectLanguage(longEnglishText)).toBe('en');
+      expect(detectLanguage(longChineseText)).toBe('zh');
+      expect(detectLanguage(longFrenchText)).toBe('fr');
+    });
+  });
+
+  describe('edge cases with whitespace and formatting', () => {
+    it('should handle text with lots of whitespace', () => {
+      const textWithWhitespace = '   This   is   English   text   with   lots   of   spaces   ';
+      const result = detectLanguage(textWithWhitespace);
+      expect(result).toBe('en');
+    });
+
+    it('should handle text with newlines and tabs', () => {
+      const textWithFormatting = 'This is English text\nwith newlines\tand tabs\nfor formatting.';
+      const result = detectLanguage(textWithFormatting);
+      expect(result).toBe('en');
+    });
+
+    it('should handle text with punctuation', () => {
+      const textWithPunctuation = 'Hello, world! How are you? I am fine. Thank you very much!!!';
+      const result = detectLanguage(textWithPunctuation);
+      expect(result).toBe('en');
+    });
+  });
+});
+
+describe('detectLanguage - Real World Samples', () => {
+  it('should detect language in book excerpts', () => {
+    const bookExcerpts = [
+      {
+        text: 'It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness.',
+        expected: 'en',
+      },
+      {
+        text: 'Dans une trou dans la terre vivait un hobbit. Pas un trou déplaisant, sale et humide.',
+        expected: 'fr',
+      },
+      {
+        text: '昔々、あるところにおじいさんとおばあさんが住んでいました。',
+        expected: 'ja',
+      },
+      {
+        text: '很久很久以前，有一个美丽的公主住在一座城堡里。',
+        expected: 'zh',
+      },
+    ];
+
+    bookExcerpts.forEach(({ text, expected }) => {
+      const result = detectLanguage(text);
+      expect(result).toBe(expected);
+    });
+  });
+});

apps/readest-app/src/utils/lang.ts

@@ -1,4 +1,6 @@
+import { franc } from 'franc-min';
 import { iso6392 } from 'iso-639-2';
+import { iso6393To1 } from 'iso-639-3';
 
 export const isCJKStr = (str: string) => {
   return /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u.test(str ?? '');
@@ -79,6 +81,23 @@ export const code6392to6391 = (code: string): string => {
   return lang?.iso6391 || '';
 };
 
+const commonIndivToMacro: Record<string, string> = {
+  cmn: 'zho',
+  arb: 'ara',
+  arz: 'ara',
+  ind: 'msa',
+  zsm: 'msa',
+  nob: 'nor',
+  nno: 'nor',
+  pes: 'fas',
+  quy: 'que',
+};
+
+export const code6393to6391 = (code: string): string => {
+  const macro = commonIndivToMacro[code] || code;
+  return iso6393To1[macro] || '';
+};
+
 export const getLanguageName = (code: string): string => {
   const lang = normalizedLangCode(code);
   const language = iso6392.find((l) => l.iso6391 === lang || l.iso6392B === lang);
@@ -97,3 +116,14 @@ export const inferLangFromScript = (text: string, lang: string): string => {
   }
   return lang;
 };
+
+export const detectLanguage = (content: string): string => {
+  try {
+    const iso6393Lang = franc(content.substring(0, 1000));
+    const iso6391Lang = code6393to6391(iso6393Lang) || 'en';
+    return iso6391Lang;
+  } catch {
+    console.warn('Language detection failed, defaulting to en.');
+    return 'en';
+  }
+};

apps/readest-app/src/utils/txt.ts

@@ -1,5 +1,6 @@
-import { getBaseFilename } from './book';
 import { partialMD5 } from './md5';
+import { getBaseFilename } from './book';
+import { detectLanguage } from './lang';
 
 interface Metadata {
   bookTitle: string;
@@ -11,6 +12,7 @@ interface Metadata {
 interface Chapter {
   title: string;
   content: string;
+  text: string;
 }
 
 interface Txt2EpubOptions {
@@ -67,7 +69,8 @@ export class TxtToEpubConverter {
       matchedAuthor = matchedAuthor.replace(/^[\p{P}\p{S}]+|[\p{P}\p{S}]+$/gu, '');
     } catch {}
     const author = matchedAuthor || providedAuthor || '';
-    const language = providedLanguage || this.detectLanguage(fileHeader);
+    const language = providedLanguage || detectLanguage(fileHeader);
+    console.log(`Detected language: ${language}`);
     const identifier = await partialMD5(txtFile);
     const metadata = { bookTitle, author, language, identifier };
 
@@ -176,7 +179,7 @@ export class TxtToEpubConverter {
       const trimmedSegment = segment.replace(/<!--.*?-->/g, '').trim();
       if (!trimmedSegment) continue;
 
-      const segmentChapters = [];
+      const segmentChapters: Chapter[] = [];
       let matches: string[] = [];
       for (const chapterRegex of chapterRegexps) {
         const tryMatches = trimmedSegment.split(chapterRegex);
@@ -194,7 +197,7 @@ export class TxtToEpubConverter {
           const formattedSegment = formatSegment(chunks.join('\n'));
           const title = `${chapters.length + 1}`;
           const content = `<h2>${title}</h2><p>${formattedSegment}</p>`;
-          chapters.push({ title, content });
+          chapters.push({ title, content, text: chunks.join('\n') });
         }
         continue;
       }
@@ -215,6 +218,7 @@ export class TxtToEpubConverter {
         segmentChapters.push({
           title: escapeXml(title),
           content: `${headTitle}<p>${formattedSegment}</p>`,
+          text: content,
         });
       }
 
@@ -228,6 +232,7 @@ export class TxtToEpubConverter {
         segmentChapters.unshift({
           title: escapeXml(segmentTitle),
           content: `<h3></h3><p>${formattedSegment}</p>`,
+          text: initialContent,
         });
       }
       chapters.push(...segmentChapters);
@@ -322,9 +327,10 @@ export class TxtToEpubConverter {
     // Add chapter files
     for (let i = 0; i < chapters.length; i++) {
       const chapter = chapters[i]!;
+      const lang = detectLanguage(chapter.text);
       const chapterContent = `<?xml version="1.0" encoding="UTF-8"?>
         <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
-        <html xmlns="http://www.w3.org/1999/xhtml" lang="zh">
+        <html xmlns="http://www.w3.org/1999/xhtml" lang="${lang}" xml:lang="${lang}">
           <head>
             <title>${chapter.title}</title>
             <link rel="stylesheet" type="text/css" href="../style.css"/>
@@ -426,26 +432,6 @@ export class TxtToEpubConverter {
     return 'utf-8';
   }
 
-  private detectLanguage(fileHeader: string): string {
-    const sample = fileHeader;
-    let chineseCount = 0;
-    for (let i = 0; i < sample.length; i++) {
-      const code = sample.charCodeAt(i);
-      if (
-        (code >= 0x4e00 && code <= 0x9fff) ||
-        (code >= 0x3400 && code <= 0x4dbf) ||
-        (code >= 0x20000 && code <= 0x2a6df)
-      ) {
-        chineseCount++;
-      }
-    }
-    if (chineseCount / sample.length > 0.05) {
-      return 'zh';
-    }
-
-    return 'en';
-  }
-
   private extractBookTitle(filename: string): string {
     const match = filename.match(/《([^》]+)》/);
     return match ? match[1]! : filename.split('.')[0]!;

pnpm-lock.yaml

@@ -131,6 +131,9 @@ importers:
       foliate-js:
         specifier: workspace:*
         version: link:../../packages/foliate-js
+      franc-min:
+        specifier: ^6.2.0
+        version: 6.2.0
       highlight.js:
         specifier: ^11.11.1
         version: 11.11.1
@@ -146,6 +149,9 @@ importers:
       iso-639-2:
         specifier: ^3.0.2
         version: 3.0.2
+      iso-639-3:
+        specifier: ^3.0.1
+        version: 3.0.1
       js-md5:
         specifier: ^0.8.3
         version: 0.8.3
@@ -3697,6 +3703,9 @@ packages:
     resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==}
     engines: {node: '>=6'}
 
+  collapse-white-space@2.1.0:
+    resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==}
+
   color-convert@2.0.1:
     resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
     engines: {node: '>=7.0.0'}
@@ -4404,6 +4413,9 @@ packages:
   fraction.js@4.3.7:
     resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==}
 
+  franc-min@6.2.0:
+    resolution: {integrity: sha512-1uDIEUSlUZgvJa2AKYR/dmJC66v/PvGQ9mWfI9nOr/kPpMFyvswK0gPXOwpYJYiYD008PpHLkGfG58SPjQJFxw==}
+
   fresh@0.5.2:
     resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==}
     engines: {node: '>= 0.6'}
@@ -4864,6 +4876,9 @@ packages:
   iso-639-2@3.0.2:
     resolution: {integrity: sha512-tna50aWwcGTIn81S9MzD1NSovHYTpFgmPVszHiLF5Vg/xmXAJ9XAkMOB9a8TH9Vi7qwf/x/8NJy2F+lM5OEwAw==}
 
+  iso-639-3@3.0.1:
+    resolution: {integrity: sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ==}
+
   isobject@3.0.1:
     resolution: {integrity: sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==}
     engines: {node: '>=0.10.0'}
@@ -5209,6 +5224,9 @@ packages:
   mz@2.7.0:
     resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==}
 
+  n-gram@2.0.2:
+    resolution: {integrity: sha512-S24aGsn+HLBxUGVAUFOwGpKs7LBcG4RudKU//eWzt/mQ97/NMKQxDWHyHx63UNWk/OOdihgmzoETn1tf5nQDzQ==}
+
   nanoid@3.3.11:
     resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==}
     engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
@@ -6340,6 +6358,9 @@ packages:
     resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==}
     engines: {node: '>=18'}
 
+  trigram-utils@2.0.1:
+    resolution: {integrity: sha512-nfWIXHEaB+HdyslAfMxSqWKDdmqY9I32jS7GnqpdWQnLH89r6A5sdk3fDVYqGAZ0CrT8ovAFSAo6HRiWcWNIGQ==}
+
   ts-api-utils@1.4.3:
     resolution: {integrity: sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==}
     engines: {node: '>=16'}
@@ -11468,6 +11489,8 @@ snapshots:
 
   clsx@2.1.1: {}
 
+  collapse-white-space@2.1.0: {}
+
   color-convert@2.0.1:
     dependencies:
       color-name: 1.1.4
@@ -12315,6 +12338,10 @@ snapshots:
 
   fraction.js@4.3.7: {}
 
+  franc-min@6.2.0:
+    dependencies:
+      trigram-utils: 2.0.1
+
   fresh@0.5.2: {}
 
   fresh@2.0.0: {}
@@ -12780,6 +12807,8 @@ snapshots:
 
   iso-639-2@3.0.2: {}
 
+  iso-639-3@3.0.1: {}
+
   isobject@3.0.1: {}
 
   isows@1.0.7(ws@8.18.2):
@@ -13090,6 +13119,8 @@ snapshots:
       object-assign: 4.1.1
       thenify-all: 1.6.0
 
+  n-gram@2.0.2: {}
+
   nanoid@3.3.11: {}
 
   nanoid@3.3.8: {}
@@ -14270,6 +14301,11 @@ snapshots:
     dependencies:
       punycode: 2.3.1
 
+  trigram-utils@2.0.1:
+    dependencies:
+      collapse-white-space: 2.1.0
+      n-gram: 2.0.2
+
   ts-api-utils@1.4.3(typescript@5.7.2):
     dependencies:
       typescript: 5.7.2