diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 98634517..960f46ec 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -58,6 +58,11 @@ jobs: run: | pnpm install && pnpm setup-pdfjs + - name: run tests + working-directory: apps/readest-app + run: | + pnpm test -- --watchAll=false + - name: build the web App if: matrix.config.platform == 'web' working-directory: apps/readest-app diff --git a/apps/readest-app/package.json b/apps/readest-app/package.json index f4eff27c..b67e2aa0 100644 --- a/apps/readest-app/package.json +++ b/apps/readest-app/package.json @@ -73,11 +73,13 @@ "cssbeautify": "^0.3.1", "dayjs": "^1.11.13", "foliate-js": "workspace:*", + "franc-min": "^6.2.0", "highlight.js": "^11.11.1", "i18next": "^24.2.0", "i18next-browser-languagedetector": "^8.0.2", "i18next-http-backend": "^3.0.1", "iso-639-2": "^3.0.2", + "iso-639-3": "^3.0.1", "js-md5": "^0.8.3", "jwt-decode": "^4.0.0", "marked": "^15.0.12", diff --git a/apps/readest-app/src/__tests__/api/iap-verify.test.ts b/apps/readest-app/src/__tests__/api/iap-verify.test.ts index dea32878..bf9b6df5 100644 --- a/apps/readest-app/src/__tests__/api/iap-verify.test.ts +++ b/apps/readest-app/src/__tests__/api/iap-verify.test.ts @@ -3,6 +3,7 @@ import { POST } from '@/app/api/apple/iap-verify/route'; import { NextRequest } from 'next/server'; import { setupSupabaseMocks } from '../helpers/supabase-mock'; +const SKIP_IAP_API_TESTS = !process.env['ENABLE_IAP_API_TESTS']; vi.mock('@/utils/supabase', () => ({ supabase: { auth: { @@ -14,7 +15,7 @@ vi.mock('@/utils/supabase', () => ({ createSupabaseAdminClient: vi.fn(), })); -describe('/api/apple/iap-verify', () => { +describe.skipIf(SKIP_IAP_API_TESTS)('/api/apple/iap-verify', () => { it('should verify a valid transaction', async () => { setupSupabaseMocks(); const request = new NextRequest('http://localhost:3000/api/apple/iap-verify', { diff --git a/apps/readest-app/src/__tests__/utils/detect-language.test.ts b/apps/readest-app/src/__tests__/utils/detect-language.test.ts new file mode 100644 index 00000000..e6697f74 --- /dev/null +++ b/apps/readest-app/src/__tests__/utils/detect-language.test.ts @@ -0,0 +1,273 @@ +import { describe, it, expect } from 'vitest'; + +import { detectLanguage } from '@/utils/lang'; + +describe('detectLanguage - Result Tests', () => { + describe('English text detection', () => { + it('should detect English text', () => { + const englishTexts = [ + 'This is a sample English text for language detection testing.', + 'The quick brown fox jumps over the lazy dog.', + 'Hello world! This is an English sentence with punctuation.', + 'Chapter 1: Introduction to Language Detection', + 'In the beginning was the Word, and the Word was with God.', + 'To be or not to be, that is the question.', + ]; + + englishTexts.forEach((text) => { + const result = detectLanguage(text); + expect(result).toBe('en'); + }); + }); + }); + + describe('Chinese text detection', () => { + it('should detect Chinese text', () => { + const chineseTexts = [ + '这是一个中文文本的示例，用于语言检测测试。', + '天下大势，分久必合，合久必分。', + '学而时习之，不亦说乎？有朋自远方来，不亦乐乎？', + '春眠不觉晓，处处闻啼鸟。夜来风雨声，花落知多少。', + '那时候，天空很蓝，云彩很白，我们都还很年轻。', + '人工智能技术的发展正在改变我们的生活方式和工作方式。', + ]; + + chineseTexts.forEach((text) => { + const result = detectLanguage(text); + expect(result).toBe('zh'); + }); + }); + + it('should detect Traditional Chinese text', () => { + const traditionalChineseTexts = [ + '這是一個正體中文文本的示例，用於語言檢測測試。', + '天下大勢，分久必合，合久必分。', + '學而時習之，不亦說乎？有朋自遠方來，不亦樂乎？', + '春眠不覺曉，處處聞啼鳥。夜來風雨聲，花落知多少。', + '那時候，天空很藍，雲彩很白，我們都還很年輕。', + '人工智慧技術的發展正在改變我們的生活方式和工作方式。', + ]; + + traditionalChineseTexts.forEach((text) => { + const result = detectLanguage(text); + expect(result).toBe('zh'); + }); + }); + }); + + describe('Japanese text detection', () => { + it('should detect Japanese text', () => { + const japaneseTexts = [ + 'これは日本語のテキストサンプルです。', + '第一章：言語検出について', + 'こんにちは、世界！これは日本語の文章です。', + '桜の花が咲く季節になりました。とても美しいです。', + 'ひらがな、カタカナ、漢字を使って日本語を書きます。', + '東京は日本の首都であり、多くの人々が住んでいます。', + ]; + + japaneseTexts.forEach((text) => { + const result = detectLanguage(text); + expect(result).toBe('ja'); + }); + }); + }); + + describe('French text detection', () => { + it('should detect French text', () => { + const frenchTexts = [ + 'Ceci est un exemple de texte français pour tester la détection de langue.', + 'Chapitre 1: Introduction à la détection de langue.', + 'Bonjour le monde! Ceci est une phrase en français.', + 'Les Champs-Élysées sont une avenue célèbre à Paris, souvent appelée la plus belle avenue du monde.', + 'Victor Hugo était un écrivain français très célèbre du XIXe siècle,', + "La littérature française possède une richesse extraordinaire qui s'étend sur plusieurs siècles.", + ]; + + frenchTexts.forEach((text) => { + const result = detectLanguage(text); + expect(result).toBe('fr'); + }); + }); + }); + + describe('Spanish text detection', () => { + it('should detect Spanish text', () => { + const spanishTexts = [ + 'Este es un ejemplo de texto en español para probar la detección de idioma.', + 'Capítulo 1: Introducción a la detección de idiomas', + '¡Hola mundo! Esta es una oración en español.', + 'España es un país ubicado en la península ibérica.', + 'El flamenco es una forma de arte español muy famosa.', + 'Don Quijote de La Mancha es una obra clásica de la literatura española.', + ]; + + spanishTexts.forEach((text) => { + const result = detectLanguage(text); + expect(result).toBe('es'); + }); + }); + }); + + describe('German text detection', () => { + it('should detect German text', () => { + const germanTexts = [ + 'Dies ist ein Beispieltext auf Deutsch für die Spracherkennung.', + 'Kapitel 1: Einführung in die Spracherkennung', + 'Hallo Welt! Dies ist ein deutscher Satz.', + 'Deutschland ist ein Land in Mitteleuropa.', + 'Das Oktoberfest ist ein berühmtes deutsches Festival.', + 'Die deutsche Sprache hat viele zusammengesetzte Wörter.', + ]; + + germanTexts.forEach((text) => { + const result = detectLanguage(text); + expect(result).toBe('de'); + }); + }); + }); + + describe('Arabic text detection', () => { + it('should detect Arabic text', () => { + const arabicTexts = [ + 'هذا نص عربي للاختبار. يجب أن يتم اكتشاف هذا النص كعربي.', + 'الفصل الأول: مقدمة في اكتشاف اللغة', + 'مرحبا بالعالم! هذه جملة باللغة العربية.', + 'العربية لغة جميلة يتحدث بها ملايين الناس.', + 'القرآن الكريم نزل باللغة العربية.', + 'دمشق هي عاصمة سوريا وواحدة من أقدم المدن في العالم.', + ]; + + arabicTexts.forEach((text) => { + const result = detectLanguage(text); + expect(result).toBe('ar'); + }); + }); + }); + + describe('Korean text detection', () => { + it('should detect Korean text', () => { + const koreanTexts = [ + '이것은 언어 감지 테스트를 위한 한국어 텍스트 샘플입니다.', + '제1장: 언어 감지 소개', + '안녕하세요 세계! 이것은 한국어 문장입니다.', + '서울은 대한민국의 수도이며 많은 사람들이 살고 있습니다.', + '한글은 세종대왕이 만든 우리나라의 고유한 문자입니다.', + '김치는 한국의 전통 음식 중 하나입니다.', + ]; + + koreanTexts.forEach((text) => { + const result = detectLanguage(text); + expect(result).toBe('ko'); + }); + }); + }); + + describe('fallback behavior', () => { + it('should return "en" for very short text', () => { + const shortTexts = ['', ' ', 'Hi', 'OK', '123', '!@#']; + + shortTexts.forEach((text) => { + const result = detectLanguage(text); + expect(result).toBe('en'); + }); + }); + + it('should return "en" for unrecognizable text', () => { + const unrecognizableTexts = [ + '123456789012345678901234567890', + '!@#$%^&*()_+{}|:"<>?[]\\;\',./', + '1111111111222222222233333333334444444444', + ]; + + unrecognizableTexts.forEach((text) => { + const result = detectLanguage(text); + expect(result).toBe('en'); + }); + }); + }); + + describe('mixed content', () => { + it('should detect predominant language in mixed content', () => { + const testCases = [ + { + text: 'This is mostly English text with some 中文 words mixed in but English dominates.', + expected: 'en', + }, + { + text: '这主要是中文文本，虽然有一些 English words 混合在其中，但中文占主导地位。', + expected: 'zh', + }, + { + text: 'Ceci est principalement du français avec quelques English words mélangés.', + expected: 'fr', + }, + ]; + + testCases.forEach(({ text, expected }) => { + const result = detectLanguage(text); + expect(result).toBe(expected); + }); + }); + }); + + describe('long text handling', () => { + it('should handle long text (over 1000 characters)', () => { + const longEnglishText = 'This is a very long English text. '.repeat(100); // ~3400 chars + const longChineseText = '这是一个很长的中文文本。'.repeat(100); // ~1200 chars + const longFrenchText = 'Ceci est un très long texte français. '.repeat(100); // ~3800 chars + + expect(detectLanguage(longEnglishText)).toBe('en'); + expect(detectLanguage(longChineseText)).toBe('zh'); + expect(detectLanguage(longFrenchText)).toBe('fr'); + }); + }); + + describe('edge cases with whitespace and formatting', () => { + it('should handle text with lots of whitespace', () => { + const textWithWhitespace = ' This is English text with lots of spaces '; + const result = detectLanguage(textWithWhitespace); + expect(result).toBe('en'); + }); + + it('should handle text with newlines and tabs', () => { + const textWithFormatting = 'This is English text\nwith newlines\tand tabs\nfor formatting.'; + const result = detectLanguage(textWithFormatting); + expect(result).toBe('en'); + }); + + it('should handle text with punctuation', () => { + const textWithPunctuation = 'Hello, world! How are you? I am fine. Thank you very much!!!'; + const result = detectLanguage(textWithPunctuation); + expect(result).toBe('en'); + }); + }); +}); + +describe('detectLanguage - Real World Samples', () => { + it('should detect language in book excerpts', () => { + const bookExcerpts = [ + { + text: 'It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness.', + expected: 'en', + }, + { + text: 'Dans une trou dans la terre vivait un hobbit. Pas un trou déplaisant, sale et humide.', + expected: 'fr', + }, + { + text: '昔々、あるところにおじいさんとおばあさんが住んでいました。', + expected: 'ja', + }, + { + text: '很久很久以前，有一个美丽的公主住在一座城堡里。', + expected: 'zh', + }, + ]; + + bookExcerpts.forEach(({ text, expected }) => { + const result = detectLanguage(text); + expect(result).toBe(expected); + }); + }); +}); diff --git a/apps/readest-app/src/utils/lang.ts b/apps/readest-app/src/utils/lang.ts index e4dafc45..70ececc0 100644 --- a/apps/readest-app/src/utils/lang.ts +++ b/apps/readest-app/src/utils/lang.ts @@ -1,4 +1,6 @@ +import { franc } from 'franc-min'; import { iso6392 } from 'iso-639-2'; +import { iso6393To1 } from 'iso-639-3'; export const isCJKStr = (str: string) => { return /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u.test(str ?? ''); @@ -79,6 +81,23 @@ export const code6392to6391 = (code: string): string => { return lang?.iso6391 || ''; }; +const commonIndivToMacro: Record = { + cmn: 'zho', + arb: 'ara', + arz: 'ara', + ind: 'msa', + zsm: 'msa', + nob: 'nor', + nno: 'nor', + pes: 'fas', + quy: 'que', +}; + +export const code6393to6391 = (code: string): string => { + const macro = commonIndivToMacro[code] || code; + return iso6393To1[macro] || ''; +}; + export const getLanguageName = (code: string): string => { const lang = normalizedLangCode(code); const language = iso6392.find((l) => l.iso6391 === lang || l.iso6392B === lang); @@ -97,3 +116,14 @@ export const inferLangFromScript = (text: string, lang: string): string => { } return lang; }; + +export const detectLanguage = (content: string): string => { + try { + const iso6393Lang = franc(content.substring(0, 1000)); + const iso6391Lang = code6393to6391(iso6393Lang) || 'en'; + return iso6391Lang; + } catch { + console.warn('Language detection failed, defaulting to en.'); + return 'en'; + } +}; diff --git a/apps/readest-app/src/utils/txt.ts b/apps/readest-app/src/utils/txt.ts index acae2615..10d245f2 100644 --- a/apps/readest-app/src/utils/txt.ts +++ b/apps/readest-app/src/utils/txt.ts @@ -1,5 +1,6 @@ -import { getBaseFilename } from './book'; import { partialMD5 } from './md5'; +import { getBaseFilename } from './book'; +import { detectLanguage } from './lang'; interface Metadata { bookTitle: string; @@ -11,6 +12,7 @@ interface Metadata { interface Chapter { title: string; content: string; + text: string; } interface Txt2EpubOptions { @@ -67,7 +69,8 @@ export class TxtToEpubConverter { matchedAuthor = matchedAuthor.replace(/^[\p{P}\p{S}]+|[\p{P}\p{S}]+$/gu, ''); } catch {} const author = matchedAuthor || providedAuthor || ''; - const language = providedLanguage || this.detectLanguage(fileHeader); + const language = providedLanguage || detectLanguage(fileHeader); + console.log(`Detected language: ${language}`); const identifier = await partialMD5(txtFile); const metadata = { bookTitle, author, language, identifier }; @@ -176,7 +179,7 @@ export class TxtToEpubConverter { const trimmedSegment = segment.replace(//g, '').trim(); if (!trimmedSegment) continue; - const segmentChapters = []; + const segmentChapters: Chapter[] = []; let matches: string[] = []; for (const chapterRegex of chapterRegexps) { const tryMatches = trimmedSegment.split(chapterRegex); @@ -194,7 +197,7 @@ export class TxtToEpubConverter { const formattedSegment = formatSegment(chunks.join('\n')); const title = `${chapters.length + 1}`; const content = `

${title}

${formattedSegment}

`; - chapters.push({ title, content }); + chapters.push({ title, content, text: chunks.join('\n') }); } continue; } @@ -215,6 +218,7 @@ export class TxtToEpubConverter { segmentChapters.push({ title: escapeXml(title), content: `${headTitle}

${formattedSegment}

`, + text: content, }); } @@ -228,6 +232,7 @@ export class TxtToEpubConverter { segmentChapters.unshift({ title: escapeXml(segmentTitle), content: `

${formattedSegment}

`, + text: initialContent, }); } chapters.push(...segmentChapters); @@ -322,9 +327,10 @@ export class TxtToEpubConverter { // Add chapter files for (let i = 0; i < chapters.length; i++) { const chapter = chapters[i]!; + const lang = detectLanguage(chapter.text); const chapterContent = ` - + ${chapter.title} @@ -426,26 +432,6 @@ export class TxtToEpubConverter { return 'utf-8'; } - private detectLanguage(fileHeader: string): string { - const sample = fileHeader; - let chineseCount = 0; - for (let i = 0; i < sample.length; i++) { - const code = sample.charCodeAt(i); - if ( - (code >= 0x4e00 && code <= 0x9fff) || - (code >= 0x3400 && code <= 0x4dbf) || - (code >= 0x20000 && code <= 0x2a6df) - ) { - chineseCount++; - } - } - if (chineseCount / sample.length > 0.05) { - return 'zh'; - } - - return 'en'; - } - private extractBookTitle(filename: string): string { const match = filename.match(/《([^》]+)》/); return match ? match[1]! : filename.split('.')[0]!; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 289260c4..2ccbe1b5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -131,6 +131,9 @@ importers: foliate-js: specifier: workspace:* version: link:../../packages/foliate-js + franc-min: + specifier: ^6.2.0 + version: 6.2.0 highlight.js: specifier: ^11.11.1 version: 11.11.1 @@ -146,6 +149,9 @@ importers: iso-639-2: specifier: ^3.0.2 version: 3.0.2 + iso-639-3: + specifier: ^3.0.1 + version: 3.0.1 js-md5: specifier: ^0.8.3 version: 0.8.3 @@ -3697,6 +3703,9 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + collapse-white-space@2.1.0: + resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -4404,6 +4413,9 @@ packages: fraction.js@4.3.7: resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} + franc-min@6.2.0: + resolution: {integrity: sha512-1uDIEUSlUZgvJa2AKYR/dmJC66v/PvGQ9mWfI9nOr/kPpMFyvswK0gPXOwpYJYiYD008PpHLkGfG58SPjQJFxw==} + fresh@0.5.2: resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} engines: {node: '>= 0.6'} @@ -4864,6 +4876,9 @@ packages: iso-639-2@3.0.2: resolution: {integrity: sha512-tna50aWwcGTIn81S9MzD1NSovHYTpFgmPVszHiLF5Vg/xmXAJ9XAkMOB9a8TH9Vi7qwf/x/8NJy2F+lM5OEwAw==} + iso-639-3@3.0.1: + resolution: {integrity: sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ==} + isobject@3.0.1: resolution: {integrity: sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==} engines: {node: '>=0.10.0'} @@ -5209,6 +5224,9 @@ packages: mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} + n-gram@2.0.2: + resolution: {integrity: sha512-S24aGsn+HLBxUGVAUFOwGpKs7LBcG4RudKU//eWzt/mQ97/NMKQxDWHyHx63UNWk/OOdihgmzoETn1tf5nQDzQ==} + nanoid@3.3.11: resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -6340,6 +6358,9 @@ packages: resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} engines: {node: '>=18'} + trigram-utils@2.0.1: + resolution: {integrity: sha512-nfWIXHEaB+HdyslAfMxSqWKDdmqY9I32jS7GnqpdWQnLH89r6A5sdk3fDVYqGAZ0CrT8ovAFSAo6HRiWcWNIGQ==} + ts-api-utils@1.4.3: resolution: {integrity: sha512-i3eMG77UTMD0hZhgRS562pv83RC6ukSAC2GMNWc+9dieh/+jDM5u5YG+NHX6VNDRHQcHwmsTHctP9LhbC3WxVw==} engines: {node: '>=16'} @@ -11468,6 +11489,8 @@ snapshots: clsx@2.1.1: {} + collapse-white-space@2.1.0: {} + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -12315,6 +12338,10 @@ snapshots: fraction.js@4.3.7: {} + franc-min@6.2.0: + dependencies: + trigram-utils: 2.0.1 + fresh@0.5.2: {} fresh@2.0.0: {} @@ -12780,6 +12807,8 @@ snapshots: iso-639-2@3.0.2: {} + iso-639-3@3.0.1: {} + isobject@3.0.1: {} isows@1.0.7(ws@8.18.2): @@ -13090,6 +13119,8 @@ snapshots: object-assign: 4.1.1 thenify-all: 1.6.0 + n-gram@2.0.2: {} + nanoid@3.3.11: {} nanoid@3.3.8: {} @@ -14270,6 +14301,11 @@ snapshots: dependencies: punycode: 2.3.1 + trigram-utils@2.0.1: + dependencies: + collapse-white-space: 2.1.0 + n-gram: 2.0.2 + ts-api-utils@1.4.3(typescript@5.7.2): dependencies: typescript: 5.7.2