feat(reader): glue non-breaking spaces after short Russian words (#4769) (#4798)

Russian typography requires short function words (prepositions, conjunctions, particles) to never hang at the end of a line. Add an `nbsp` content transformer that inserts U+00A0 after such words so they stick to the following word. The source file is never modified. The transformer is language-driven via an NBSP_LANGUAGES registry keyed by language code (only `ru` is configured today), so adding another language is a single entry. It runs only for matching books and rewrites text between tags with a regex, leaving tags, attributes, and the XML declaration intact. Runs after whitespace normalization so the inserted spaces are not stripped under the override-layout setting. The space-to-NBSP swap is length-preserving (both are single UTF-16 code units), so DOM character offsets and CFIs stay valid for every word before and after the transform; tests enforce this invariant. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-26 10:52:32 +08:00
parent 4c39d769e6
commit 370a516620
4 changed files with 265 additions and 0 deletions
@@ -20,6 +20,7 @@ vi.mock('@/utils/lang', () => ({
  getLanguageInfo: vi.fn(() => ({ direction: 'ltr' })),
  isSameLang: vi.fn(() => true),
  isValidLang: vi.fn(() => true),
+  normalizedLangCode: (lang?: string | null) => (lang ? lang.split('-')[0]!.toLowerCase() : ''),
 }));

 vi.mock('@/store/settingsStore', () => ({
@@ -769,6 +770,140 @@ describe('simpleccTransformer', () => {
  });
 });

+// =============================================================================
+// nbspTransformer
+// =============================================================================
+
+describe('nbspTransformer', () => {
+  let nbspTransformer: typeof import('@/services/transformers/nbsp').nbspTransformer;
+
+  beforeEach(async () => {
+    ({ nbspTransformer } = await import('@/services/transformers/nbsp'));
+  });
+
+  test('has the correct name', () => {
+    expect(nbspTransformer.name).toBe('nbsp');
+  });
+
+  test('returns content unchanged for unsupported languages', async () => {
+    const html = '<html><body><p>в доме</p></body></html>';
+    const result = await nbspTransformer.transform(
+      makeCtx({ content: html, primaryLanguage: 'en' }),
+    );
+    expect(result).toBe(html);
+  });
+
+  describe('for Russian books', () => {
+    const NBSP = '\u00A0';
+
+    test('glues a single-letter preposition to the next word', async () => {
+      const html = '<html><body><p>книга в доме</p></body></html>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru' }),
+      );
+      expect(result).toContain(`в${NBSP}доме`);
+      expect(result).not.toContain('в доме');
+    });
+
+    test('glues a two-letter conjunction', async () => {
+      const html = '<p>тепло но ветрено</p>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru' }),
+      );
+      expect(result).toContain(`но${NBSP}ветрено`);
+    });
+
+    test('glues multi-letter function words from the list', async () => {
+      const html = '<p>только если хотя дом</p>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru-RU' }),
+      );
+      expect(result).toContain(`только${NBSP}если${NBSP}хотя${NBSP}дом`);
+    });
+
+    test('handles capitalized words at the start of a sentence', async () => {
+      const html = '<p>Но что делать</p>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru' }),
+      );
+      expect(result).toContain(`Но${NBSP}что${NBSP}делать`);
+    });
+
+    test('glues consecutive short words', async () => {
+      const html = '<p>и в доме</p>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru' }),
+      );
+      expect(result).toContain(`и${NBSP}в${NBSP}доме`);
+    });
+
+    test('does not split a longer word that begins with a short word', async () => {
+      const html = '<p>в наказание</p>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru' }),
+      );
+      expect(result).toContain(`в${NBSP}наказание`);
+      // "на" inside "наказание" must stay intact (no NBSP injected mid-word)
+      expect(result).not.toContain(`на${NBSP}`);
+    });
+
+    test('leaves text without short function words unchanged', async () => {
+      const html = '<html><body><p>наш дом большой</p></body></html>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru' }),
+      );
+      expect(result).toBe(html);
+    });
+
+    test('does not modify attribute values, only text nodes', async () => {
+      const html = '<html><body><p title="это и то">в доме</p></body></html>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru' }),
+      );
+      expect(result).toContain('title="это и то"');
+      expect(result).toContain(`в${NBSP}доме`);
+    });
+
+    test('skips <style> and <script> content', async () => {
+      const html = '<html><head><style>в доме</style></head><body><p>в доме</p></body></html>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru' }),
+      );
+      // body text glued, style text left intact (regular space)
+      expect(result).toContain('<style>в доме</style>');
+      expect(result).toContain(`<p>в${NBSP}доме</p>`);
+    });
+
+    test('does not glue a preposition before a non-Cyrillic word', async () => {
+      const html = '<p>читаю в Google</p>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru' }),
+      );
+      expect(result).toContain('в Google');
+      expect(result).not.toContain(`в${NBSP}Google`);
+    });
+
+    test('preserves total length so CFIs stay valid', async () => {
+      const html = '<html><body><p>Он пришёл в дом и сел у окна около 5 часов.</p></body></html>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru' }),
+      );
+      expect(result.length).toBe(html.length);
+      // length is unchanged, but NBSPs were actually inserted
+      expect(result).not.toBe(html);
+    });
+
+    test('keeps length stable with consecutive short words', async () => {
+      const html = '<p>и в доме</p>';
+      const result = await nbspTransformer.transform(
+        makeCtx({ content: html, primaryLanguage: 'ru' }),
+      );
+      expect(result.length).toBe(html.length);
+      expect(result).toContain(`и${NBSP}в${NBSP}доме`);
+    });
+  });
+});
+
 // =============================================================================
 // availableTransformers (index)
 // =============================================================================
@@ -785,6 +920,7 @@ describe('availableTransformers', () => {
    expect(names).toContain('language');
    expect(names).toContain('simplecc');
    expect(names).toContain('proofread');
+    expect(names).toContain('nbsp');
  });

  test('each transformer has a name and transform function', async () => {
@@ -254,6 +254,7 @@ const FoliateViewer: React.FC<{
                'language',
                'sanitizer',
                'simplecc',
+                'nbsp',
                'proofread',
                'warichu',
              ],
@@ -8,6 +8,7 @@ import { simpleccTransformer } from './simplecc';
 import { styleTransformer } from './style';
 import { proofreadTransformer } from './proofread';
 import { warichuTransformer } from './warichu';
+import { nbspTransformer } from './nbsp';

 export const availableTransformers: Transformer[] = [
  punctuationTransformer,
@@ -17,6 +18,7 @@ export const availableTransformers: Transformer[] = [
  whitespaceTransformer,
  sanitizerTransformer,
  simpleccTransformer,
+  nbspTransformer,
  proofreadTransformer,
  warichuTransformer,
  // Add more transformers here
@@ -0,0 +1,126 @@
+import { normalizedLangCode } from '@/utils/lang';
+import type { Transformer } from './types';
+
+interface NbspLanguageConfig {
+  // Unicode script name (for `\p{Script=...}`). Drives the generic short-word
+  // rule and the "next char belongs to this language" look-ahead.
+  script: string;
+  // Function words (prepositions, conjunctions, particles) of three or more
+  // letters that must not be left hanging at the end of a line. One- and
+  // two-letter words are matched generically by the script rule, so only the
+  // longer ones are listed here. Content words (nouns/verbs) are deliberately
+  // excluded so we never glue after them.
+  shortWords: string[];
+}
+
+// Languages whose short function words must stick to the next word so they
+// never hang at the end of a line. Add a new entry to support another language.
+// Russian: "висячий предлог" is a typographic error in Russian publishing.
+// See issue #4769.
+const NBSP_LANGUAGES: Record<string, NbspLanguageConfig> = {
+  ru: {
+    script: 'Cyrillic',
+    shortWords: [
+      // prepositions
+      'без',
+      'для',
+      'близ',
+      'под',
+      'над',
+      'про',
+      'при',
+      'ради',
+      'сквозь',
+      'среди',
+      'через',
+      'около',
+      'перед',
+      'после',
+      'между',
+      'кроме',
+      'вокруг',
+      'против',
+      'вместо',
+      'внутри',
+      'возле',
+      // conjunctions
+      'или',
+      'либо',
+      'ибо',
+      'если',
+      'едва',
+      'дабы',
+      'чтобы',
+      'чтоб',
+      'хотя',
+      'пока',
+      'зато',
+      'тоже',
+      'также',
+      'итак',
+      'как',
+      'что',
+      'чем',
+      'так',
+      // particles
+      'даже',
+      'лишь',
+      'ведь',
+      'вот',
+      'вон',
+      'уже',
+      'хоть',
+      'разве',
+      'только',
+      'именно',
+      'неужели',
+    ],
+  },
+};
+
+// (boundary)(short word)(regular space)(?=script letter or digit) -> glue.
+// `\b` is ASCII-only, so instead of a look-behind (disallowed in this repo) we
+// capture and re-emit a non-letter boundary. Requiring a regular space then a
+// letter/digit of the language's script ensures we never split a longer word.
+const buildGlueRegex = ({ script, shortWords }: NbspLanguageConfig) =>
+  new RegExp(
+    `(^|[^\\p{L}])(${shortWords.join('|')}|\\p{Script=${script}}{1,2})` +
+      `\\u0020(?=[\\p{Script=${script}}\\p{N}])`,
+    'giu',
+  );
+
+const glueShortWords = (text: string, regex: RegExp): string => {
+  if (!text.includes(' ')) return text;
+  let result = text;
+  let prev: string;
+  // Loop until stable: a single pass cannot glue runs of consecutive short
+  // words ("и в доме") because each match consumes the boundary it needs.
+  do {
+    prev = result;
+    // Swap one regular space (U+0020) for one NBSP (U+00A0). Both are single
+    // UTF-16 code units, so the text length is unchanged: DOM character offsets
+    // and therefore CFIs stay valid for every word before and after this pass
+    // (the following `proofread` transformer and stored annotations rely on it).
+    result = result.replace(regex, '$1$2\u00A0');
+  } while (result !== prev);
+  return result;
+};
+
+// Match a whole <style>/<script> block (left untouched) or a run of text
+// between two tags. Operating on the raw string keeps every tag, attribute,
+// entity and the XML declaration byte-for-byte intact; only text nodes change.
+const TEXT_OR_SKIP = /<(style|script)\b[^>]*>[\s\S]*?<\/\1>|>([^<]+)</gi;
+
+export const nbspTransformer: Transformer = {
+  name: 'nbsp',
+
+  transform: async (ctx) => {
+    const config = NBSP_LANGUAGES[normalizedLangCode(ctx.primaryLanguage)];
+    if (!config) return ctx.content;
+
+    const regex = buildGlueRegex(config);
+    return ctx.content.replace(TEXT_OR_SKIP, (match, _skipTag, text) =>
+      text === undefined ? match : `>${glueShortWords(text, regex)}<`,
+    );
+  },
+};