diff --git a/apps/readest-app/src/__tests__/services/transformers/transformers.test.ts b/apps/readest-app/src/__tests__/services/transformers/transformers.test.ts index 3961506e..4fa4567f 100644 --- a/apps/readest-app/src/__tests__/services/transformers/transformers.test.ts +++ b/apps/readest-app/src/__tests__/services/transformers/transformers.test.ts @@ -20,6 +20,7 @@ vi.mock('@/utils/lang', () => ({ getLanguageInfo: vi.fn(() => ({ direction: 'ltr' })), isSameLang: vi.fn(() => true), isValidLang: vi.fn(() => true), + normalizedLangCode: (lang?: string | null) => (lang ? lang.split('-')[0]!.toLowerCase() : ''), })); vi.mock('@/store/settingsStore', () => ({ @@ -769,6 +770,140 @@ describe('simpleccTransformer', () => { }); }); +// ============================================================================= +// nbspTransformer +// ============================================================================= + +describe('nbspTransformer', () => { + let nbspTransformer: typeof import('@/services/transformers/nbsp').nbspTransformer; + + beforeEach(async () => { + ({ nbspTransformer } = await import('@/services/transformers/nbsp')); + }); + + test('has the correct name', () => { + expect(nbspTransformer.name).toBe('nbsp'); + }); + + test('returns content unchanged for unsupported languages', async () => { + const html = '

в доме

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'en' }), + ); + expect(result).toBe(html); + }); + + describe('for Russian books', () => { + const NBSP = '\u00A0'; + + test('glues a single-letter preposition to the next word', async () => { + const html = '

книга в доме

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru' }), + ); + expect(result).toContain(`в${NBSP}доме`); + expect(result).not.toContain('в доме'); + }); + + test('glues a two-letter conjunction', async () => { + const html = '

тепло но ветрено

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru' }), + ); + expect(result).toContain(`но${NBSP}ветрено`); + }); + + test('glues multi-letter function words from the list', async () => { + const html = '

только если хотя дом

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru-RU' }), + ); + expect(result).toContain(`только${NBSP}если${NBSP}хотя${NBSP}дом`); + }); + + test('handles capitalized words at the start of a sentence', async () => { + const html = '

Но что делать

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru' }), + ); + expect(result).toContain(`Но${NBSP}что${NBSP}делать`); + }); + + test('glues consecutive short words', async () => { + const html = '

и в доме

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru' }), + ); + expect(result).toContain(`и${NBSP}в${NBSP}доме`); + }); + + test('does not split a longer word that begins with a short word', async () => { + const html = '

в наказание

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru' }), + ); + expect(result).toContain(`в${NBSP}наказание`); + // "на" inside "наказание" must stay intact (no NBSP injected mid-word) + expect(result).not.toContain(`на${NBSP}`); + }); + + test('leaves text without short function words unchanged', async () => { + const html = '

наш дом большой

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru' }), + ); + expect(result).toBe(html); + }); + + test('does not modify attribute values, only text nodes', async () => { + const html = '

в доме

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru' }), + ); + expect(result).toContain('title="это и то"'); + expect(result).toContain(`в${NBSP}доме`); + }); + + test('skips

в доме

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru' }), + ); + // body text glued, style text left intact (regular space) + expect(result).toContain(''); + expect(result).toContain(`

в${NBSP}доме

`); + }); + + test('does not glue a preposition before a non-Cyrillic word', async () => { + const html = '

читаю в Google

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru' }), + ); + expect(result).toContain('в Google'); + expect(result).not.toContain(`в${NBSP}Google`); + }); + + test('preserves total length so CFIs stay valid', async () => { + const html = '

Он пришёл в дом и сел у окна около 5 часов.

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru' }), + ); + expect(result.length).toBe(html.length); + // length is unchanged, but NBSPs were actually inserted + expect(result).not.toBe(html); + }); + + test('keeps length stable with consecutive short words', async () => { + const html = '

и в доме

'; + const result = await nbspTransformer.transform( + makeCtx({ content: html, primaryLanguage: 'ru' }), + ); + expect(result.length).toBe(html.length); + expect(result).toContain(`и${NBSP}в${NBSP}доме`); + }); + }); +}); + // ============================================================================= // availableTransformers (index) // ============================================================================= @@ -785,6 +920,7 @@ describe('availableTransformers', () => { expect(names).toContain('language'); expect(names).toContain('simplecc'); expect(names).toContain('proofread'); + expect(names).toContain('nbsp'); }); test('each transformer has a name and transform function', async () => { diff --git a/apps/readest-app/src/app/reader/components/FoliateViewer.tsx b/apps/readest-app/src/app/reader/components/FoliateViewer.tsx index cb5a435b..e035f460 100644 --- a/apps/readest-app/src/app/reader/components/FoliateViewer.tsx +++ b/apps/readest-app/src/app/reader/components/FoliateViewer.tsx @@ -254,6 +254,7 @@ const FoliateViewer: React.FC<{ 'language', 'sanitizer', 'simplecc', + 'nbsp', 'proofread', 'warichu', ], diff --git a/apps/readest-app/src/services/transformers/index.ts b/apps/readest-app/src/services/transformers/index.ts index d1b94be8..f205f51c 100644 --- a/apps/readest-app/src/services/transformers/index.ts +++ b/apps/readest-app/src/services/transformers/index.ts @@ -8,6 +8,7 @@ import { simpleccTransformer } from './simplecc'; import { styleTransformer } from './style'; import { proofreadTransformer } from './proofread'; import { warichuTransformer } from './warichu'; +import { nbspTransformer } from './nbsp'; export const availableTransformers: Transformer[] = [ punctuationTransformer, @@ -17,6 +18,7 @@ export const availableTransformers: Transformer[] = [ whitespaceTransformer, sanitizerTransformer, simpleccTransformer, + nbspTransformer, proofreadTransformer, warichuTransformer, // Add more transformers here diff --git a/apps/readest-app/src/services/transformers/nbsp.ts b/apps/readest-app/src/services/transformers/nbsp.ts new file mode 100644 index 00000000..d5947337 --- /dev/null +++ b/apps/readest-app/src/services/transformers/nbsp.ts @@ -0,0 +1,126 @@ +import { normalizedLangCode } from '@/utils/lang'; +import type { Transformer } from './types'; + +interface NbspLanguageConfig { + // Unicode script name (for `\p{Script=...}`). Drives the generic short-word + // rule and the "next char belongs to this language" look-ahead. + script: string; + // Function words (prepositions, conjunctions, particles) of three or more + // letters that must not be left hanging at the end of a line. One- and + // two-letter words are matched generically by the script rule, so only the + // longer ones are listed here. Content words (nouns/verbs) are deliberately + // excluded so we never glue after them. + shortWords: string[]; +} + +// Languages whose short function words must stick to the next word so they +// never hang at the end of a line. Add a new entry to support another language. +// Russian: "висячий предлог" is a typographic error in Russian publishing. +// See issue #4769. +const NBSP_LANGUAGES: Record = { + ru: { + script: 'Cyrillic', + shortWords: [ + // prepositions + 'без', + 'для', + 'близ', + 'под', + 'над', + 'про', + 'при', + 'ради', + 'сквозь', + 'среди', + 'через', + 'около', + 'перед', + 'после', + 'между', + 'кроме', + 'вокруг', + 'против', + 'вместо', + 'внутри', + 'возле', + // conjunctions + 'или', + 'либо', + 'ибо', + 'если', + 'едва', + 'дабы', + 'чтобы', + 'чтоб', + 'хотя', + 'пока', + 'зато', + 'тоже', + 'также', + 'итак', + 'как', + 'что', + 'чем', + 'так', + // particles + 'даже', + 'лишь', + 'ведь', + 'вот', + 'вон', + 'уже', + 'хоть', + 'разве', + 'только', + 'именно', + 'неужели', + ], + }, +}; + +// (boundary)(short word)(regular space)(?=script letter or digit) -> glue. +// `\b` is ASCII-only, so instead of a look-behind (disallowed in this repo) we +// capture and re-emit a non-letter boundary. Requiring a regular space then a +// letter/digit of the language's script ensures we never split a longer word. +const buildGlueRegex = ({ script, shortWords }: NbspLanguageConfig) => + new RegExp( + `(^|[^\\p{L}])(${shortWords.join('|')}|\\p{Script=${script}}{1,2})` + + `\\u0020(?=[\\p{Script=${script}}\\p{N}])`, + 'giu', + ); + +const glueShortWords = (text: string, regex: RegExp): string => { + if (!text.includes(' ')) return text; + let result = text; + let prev: string; + // Loop until stable: a single pass cannot glue runs of consecutive short + // words ("и в доме") because each match consumes the boundary it needs. + do { + prev = result; + // Swap one regular space (U+0020) for one NBSP (U+00A0). Both are single + // UTF-16 code units, so the text length is unchanged: DOM character offsets + // and therefore CFIs stay valid for every word before and after this pass + // (the following `proofread` transformer and stored annotations rely on it). + result = result.replace(regex, '$1$2\u00A0'); + } while (result !== prev); + return result; +}; + +// Match a whole