feat(dictionary): lemmatize inflected words before lookup (#4574) (#4582)

Dictionaries that store only base headwords (e.g. Oxford Dictionary of
English) miss inflected selections like `ran`, `mice`, `children`, or
`analyses` even though the lemma (`run`, `mouse`, `child`, `analysis`) is
present. Add a language-aware lemmatizer whose base-form candidates are
appended to the existing lookup candidate chain, after the exact/case
variants, so an exact/case match always wins and the lemma is only tried
once those miss.

- New pluggable `lemmatize/` registry keyed by primary language subtag;
  add a language by registering one lemmatizer, no caller changes.
- English lemmatizer: irregular-form table (suppletive verbs, irregular
  plurals/comparatives) + regular suffix rules (plural/past/gerund/
  comparative/possessive). Over-generates on purpose — the dictionary
  lookup is the validator, so bogus stems simply miss.
- Unknown/missing book language defaults to English (no-op on non-ASCII);
  an explicit non-English language with no registered lemmatizer is a
  no-op.
- Applies centrally to all definition providers (mdict/stardict/dict/slob
  and the online builtins) via `buildLookupCandidates`.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

apps/readest-app/src/__tests__/services/dictionaries/lemmatize/english.test.ts

@@ -0,0 +1,103 @@
+import { describe, it, expect } from 'vitest';
+
+import { lemmatizeEnglish } from '@/services/dictionaries/lemmatize/english';
+
+describe('lemmatizeEnglish', () => {
+  describe('irregular forms (issue test cases)', () => {
+    it('maps irregular verb forms to their base', () => {
+      expect(lemmatizeEnglish('ran')).toContain('run');
+      expect(lemmatizeEnglish('went')).toContain('go');
+      expect(lemmatizeEnglish('gone')).toContain('go');
+    });
+
+    it('maps irregular plurals to their singular', () => {
+      expect(lemmatizeEnglish('mice')).toContain('mouse');
+      expect(lemmatizeEnglish('children')).toContain('child');
+    });
+
+    it('maps irregular comparatives/superlatives to their base adjective', () => {
+      expect(lemmatizeEnglish('better')).toContain('good');
+      expect(lemmatizeEnglish('best')).toContain('good');
+      expect(lemmatizeEnglish('worse')).toContain('bad');
+    });
+  });
+
+  describe('regular suffix rules (issue test cases)', () => {
+    it('reduces Greek/Latin -ses plurals to -sis', () => {
+      expect(lemmatizeEnglish('analyses')).toContain('analysis');
+    });
+
+    it('prefers the -sis noun ahead of the -se verb for -ses words', () => {
+      const candidates = lemmatizeEnglish('analyses');
+      const analysis = candidates.indexOf('analysis');
+      const analyse = candidates.indexOf('analyse');
+      expect(analysis).toBeGreaterThanOrEqual(0);
+      // When both surface, the noun (issue's expected lookup) comes first.
+      if (analyse >= 0) expect(analysis).toBeLessThan(analyse);
+    });
+
+    it('drops a trailing -d from -ed words built on an e-stem', () => {
+      expect(lemmatizeEnglish('realised')).toContain('realise');
+    });
+  });
+
+  describe('regular suffix rule families', () => {
+    it('handles regular plurals and third-person -s/-es', () => {
+      expect(lemmatizeEnglish('cats')).toContain('cat');
+      expect(lemmatizeEnglish('boxes')).toContain('box');
+      expect(lemmatizeEnglish('dishes')).toContain('dish');
+      expect(lemmatizeEnglish('cities')).toContain('city');
+      expect(lemmatizeEnglish('wolves')).toContain('wolf');
+      expect(lemmatizeEnglish('knives')).toContain('knife');
+    });
+
+    it('handles regular past tense, including doubled consonants', () => {
+      expect(lemmatizeEnglish('walked')).toContain('walk');
+      expect(lemmatizeEnglish('studied')).toContain('study');
+      expect(lemmatizeEnglish('stopped')).toContain('stop');
+      expect(lemmatizeEnglish('baked')).toContain('bake');
+    });
+
+    it('handles -ing forms, including e-restoration and doubled consonants', () => {
+      expect(lemmatizeEnglish('walking')).toContain('walk');
+      expect(lemmatizeEnglish('running')).toContain('run');
+      expect(lemmatizeEnglish('making')).toContain('make');
+      expect(lemmatizeEnglish('lying')).toContain('lie');
+    });
+
+    it('handles comparatives and superlatives', () => {
+      expect(lemmatizeEnglish('faster')).toContain('fast');
+      expect(lemmatizeEnglish('fastest')).toContain('fast');
+      expect(lemmatizeEnglish('larger')).toContain('large');
+      expect(lemmatizeEnglish('happier')).toContain('happy');
+      expect(lemmatizeEnglish('happiest')).toContain('happy');
+      expect(lemmatizeEnglish('bigger')).toContain('big');
+    });
+
+    it('strips possessive clitics', () => {
+      expect(lemmatizeEnglish("cat's")).toContain('cat');
+      expect(lemmatizeEnglish("dogs'")).toContain('dog');
+    });
+  });
+
+  describe('guards', () => {
+    it('returns an empty list for multi-word, numeric, or non-ASCII input', () => {
+      expect(lemmatizeEnglish('hello world')).toEqual([]);
+      expect(lemmatizeEnglish('123')).toEqual([]);
+      expect(lemmatizeEnglish('café')).toEqual([]);
+      expect(lemmatizeEnglish('')).toEqual([]);
+      expect(lemmatizeEnglish('   ')).toEqual([]);
+    });
+
+    it('is case-insensitive on the input', () => {
+      expect(lemmatizeEnglish('Ran')).toContain('run');
+      expect(lemmatizeEnglish('MICE')).toContain('mouse');
+    });
+
+    it('never returns the input word itself or single letters', () => {
+      expect(lemmatizeEnglish('run')).not.toContain('run');
+      expect(lemmatizeEnglish('cat')).not.toContain('cat');
+      expect(lemmatizeEnglish('best')).not.toContain('b');
+    });
+  });
+});

apps/readest-app/src/__tests__/services/dictionaries/lemmatize/index.test.ts

@@ -0,0 +1,27 @@
+import { describe, it, expect } from 'vitest';
+
+import { getLemmaCandidates } from '@/services/dictionaries/lemmatize';
+
+describe('getLemmaCandidates', () => {
+  it('lemmatizes words for an English language code', () => {
+    expect(getLemmaCandidates('ran', 'en')).toContain('run');
+  });
+
+  it('normalizes regional/script English subtags to the en lemmatizer', () => {
+    expect(getLemmaCandidates('mice', 'en-US')).toContain('mouse');
+    expect(getLemmaCandidates('mice', 'en-GB')).toContain('mouse');
+  });
+
+  it('defaults to English when the language is missing or empty', () => {
+    expect(getLemmaCandidates('ran')).toContain('run');
+    expect(getLemmaCandidates('ran', undefined)).toContain('run');
+    expect(getLemmaCandidates('ran', '')).toContain('run');
+    expect(getLemmaCandidates('ran', null)).toContain('run');
+  });
+
+  it('returns [] for an explicit language with no registered lemmatizer', () => {
+    expect(getLemmaCandidates('mange', 'fr')).toEqual([]);
+    expect(getLemmaCandidates('rennt', 'de')).toEqual([]);
+    expect(getLemmaCandidates('mice', 'zh')).toEqual([]);
+  });
+});

apps/readest-app/src/__tests__/services/dictionaries/lookupCandidates.test.ts

@@ -8,8 +8,9 @@ describe('buildLookupCandidates', () => {
   });
 
   it('trims leading/trailing whitespace from a double-click selection', () => {
+    // Non-inflecting words keep this focused on trimming + case folding.
     expect(buildLookupCandidates('world ')).toEqual(['world', 'World', 'WORLD']);
-    expect(buildLookupCandidates('  spaced  ')).toEqual(['spaced', 'Spaced', 'SPACED']);
+    expect(buildLookupCandidates('  planet  ')).toEqual(['planet', 'Planet', 'PLANET']);
   });
 
   it('offers a lowercase variant for a sentence-initial capitalized word', () => {
@@ -29,4 +30,47 @@ describe('buildLookupCandidates', () => {
     expect(buildLookupCandidates('')).toEqual([]);
     expect(buildLookupCandidates('   ')).toEqual([]);
   });
+
+  describe('lemmatization fallback', () => {
+    it('appends lemma candidates after the exact/case variants', () => {
+      const candidates = buildLookupCandidates('ran', 'en');
+      expect(candidates[0]).toBe('ran'); // exact selection tried first
+      expect(candidates).toContain('run');
+      expect(candidates.indexOf('run')).toBeGreaterThan(candidates.indexOf('ran'));
+    });
+
+    it('keeps every exact/case variant ahead of any lemma', () => {
+      const candidates = buildLookupCandidates('Mice', 'en');
+      const lastCaseVariant = Math.max(
+        candidates.indexOf('Mice'),
+        candidates.indexOf('mice'),
+        candidates.indexOf('MICE'),
+      );
+      expect(candidates.indexOf('mouse')).toBeGreaterThan(lastCaseVariant);
+    });
+
+    it('resolves the issue test cases to their expected lemma', () => {
+      const cases: Array<[string, string]> = [
+        ['ran', 'run'],
+        ['went', 'go'],
+        ['gone', 'go'],
+        ['mice', 'mouse'],
+        ['children', 'child'],
+        ['better', 'good'],
+        ['analyses', 'analysis'],
+        ['realised', 'realise'],
+      ];
+      for (const [selection, lemma] of cases) {
+        expect(buildLookupCandidates(selection, 'en')).toContain(lemma);
+      }
+    });
+
+    it('defaults to English lemmatization when no language is given', () => {
+      expect(buildLookupCandidates('ran')).toContain('run');
+    });
+
+    it('does not lemmatize for an explicit non-English language', () => {
+      expect(buildLookupCandidates('ran', 'fr')).toEqual(['ran', 'Ran', 'RAN']);
+    });
+  });
 });

apps/readest-app/src/app/reader/components/annotator/DictionaryResultsView.tsx

@@ -211,12 +211,15 @@ export function useDictionaryResults({
           if (!container) {
             outcome = { ok: false, reason: 'error', message: 'no container' };
           } else {
-            // Try normalized query variants (trimmed, case-folded) in
-            // priority order and keep the first hit. Case-sensitive
-            // formats (mdict) otherwise miss `Hello` / `world ` style
-            // selections whose headword is stored lowercased.
+            // Try normalized query variants (trimmed, case-folded) then
+            // language-aware lemma candidates in priority order, keeping the
+            // first hit. Case-sensitive formats (mdict) otherwise miss
+            // `Hello` / `world ` style selections whose headword is stored
+            // lowercased, and dictionaries that store only base headwords
+            // (e.g. Oxford Dictionary of English) miss inflected selections
+            // like `ran` / `mice` / `analyses`.
             outcome = { ok: false, reason: 'empty' };
-            for (const candidate of buildLookupCandidates(currentWord)) {
+            for (const candidate of buildLookupCandidates(currentWord, langCode)) {
               container.replaceChildren();
               outcome = await provider.lookup(candidate, {
                 lang: langCode,

apps/readest-app/src/services/dictionaries/lemmatize/english.ts

@@ -0,0 +1,226 @@
+/**
+ * English lemmatizer for dictionary lookup fallback.
+ *
+ * Given an inflected English word it returns an ordered, de-duplicated list of
+ * candidate base forms (e.g. `ran → run`, `mice → mouse`, `analyses →
+ * analysis`). The list is intentionally *over-generated*: the dictionary
+ * lookup itself is the validator, so a bogus stem simply misses and the caller
+ * moves on. The rules therefore only need to *include* the correct base, not
+ * be linguistically precise.
+ *
+ * Two layers, in priority order:
+ *   1. an irregular-form table (common suppletive verbs, irregular plurals,
+ *      and irregular comparatives) — these can't be derived by rule;
+ *   2. regular suffix rules (plural / past / gerund / comparative / possessive).
+ *
+ * Only single ASCII-alphabetic tokens are lemmatized; phrases, numbers, and
+ * accented/CJK text return `[]`.
+ */
+
+// Base form -> the irregular inflected forms that should resolve to it.
+// Readable as groups; flattened to an inflected->base map at module load.
+const IRREGULAR_GROUPS: Record<string, string[]> = {
+  // Suppletive / highly irregular verbs.
+  be: ['is', 'am', 'are', 'was', 'were', 'been', 'being'],
+  have: ['has', 'had', 'having'],
+  do: ['does', 'did', 'done', 'doing'],
+  go: ['goes', 'went', 'gone', 'going'],
+  say: ['said'],
+  get: ['got', 'gotten'],
+  make: ['made'],
+  know: ['knew', 'known'],
+  think: ['thought'],
+  take: ['took', 'taken'],
+  see: ['saw', 'seen'],
+  come: ['came'],
+  find: ['found'],
+  give: ['gave', 'given'],
+  tell: ['told'],
+  feel: ['felt'],
+  become: ['became'],
+  leave: ['left'],
+  mean: ['meant'],
+  keep: ['kept'],
+  begin: ['began', 'begun'],
+  show: ['showed', 'shown'],
+  hear: ['heard'],
+  run: ['ran'],
+  bring: ['brought'],
+  write: ['wrote', 'written'],
+  sit: ['sat'],
+  stand: ['stood'],
+  lose: ['lost'],
+  pay: ['paid'],
+  meet: ['met'],
+  learn: ['learnt'],
+  lead: ['led'],
+  understand: ['understood'],
+  speak: ['spoke', 'spoken'],
+  spend: ['spent'],
+  grow: ['grew', 'grown'],
+  win: ['won'],
+  teach: ['taught'],
+  buy: ['bought'],
+  send: ['sent'],
+  build: ['built'],
+  fall: ['fell', 'fallen'],
+  catch: ['caught'],
+  draw: ['drew', 'drawn'],
+  choose: ['chose', 'chosen'],
+  drive: ['drove', 'driven'],
+  break: ['broke', 'broken'],
+  eat: ['ate', 'eaten'],
+  drink: ['drank', 'drunk'],
+  sing: ['sang', 'sung'],
+  swim: ['swam', 'swum'],
+  ring: ['rang', 'rung'],
+  fly: ['flew', 'flown'],
+  throw: ['threw', 'thrown'],
+  wear: ['wore', 'worn'],
+  tear: ['tore', 'torn'],
+  sell: ['sold'],
+  hold: ['held'],
+  feed: ['fed'],
+  fight: ['fought'],
+  hide: ['hid', 'hidden'],
+  ride: ['rode', 'ridden'],
+  rise: ['rose', 'risen'],
+  shake: ['shook', 'shaken'],
+  steal: ['stole', 'stolen'],
+  freeze: ['froze', 'frozen'],
+  sleep: ['slept'],
+  bite: ['bit', 'bitten'],
+  hang: ['hung'],
+  shoot: ['shot'],
+  sink: ['sank', 'sunk'],
+  forget: ['forgot', 'forgotten'],
+  forgive: ['forgave', 'forgiven'],
+  lay: ['laid'],
+  deal: ['dealt'],
+  dig: ['dug'],
+  shine: ['shone'],
+  bend: ['bent'],
+  lend: ['lent'],
+  blow: ['blew', 'blown'],
+  beat: ['beaten'],
+  arise: ['arose', 'arisen'],
+  awake: ['awoke', 'awoken'],
+  // Irregular plurals.
+  man: ['men'],
+  woman: ['women'],
+  child: ['children'],
+  mouse: ['mice'],
+  louse: ['lice'],
+  goose: ['geese'],
+  foot: ['feet'],
+  tooth: ['teeth'],
+  person: ['people'],
+  ox: ['oxen'],
+  die: ['dice'],
+  criterion: ['criteria'],
+  phenomenon: ['phenomena'],
+  cactus: ['cacti'],
+  fungus: ['fungi'],
+  nucleus: ['nuclei'],
+  radius: ['radii'],
+  alumnus: ['alumni'],
+  index: ['indices'],
+  matrix: ['matrices'],
+  vertex: ['vertices'],
+  appendix: ['appendices'],
+  // Irregular comparatives / superlatives (adjective & adverb).
+  good: ['better', 'best', 'well'],
+  bad: ['worse', 'worst'],
+  far: ['further', 'furthest', 'farther', 'farthest'],
+  little: ['less', 'least'],
+};
+
+const IRREGULARS: Record<string, string> = {};
+for (const [base, forms] of Object.entries(IRREGULAR_GROUPS)) {
+  for (const form of forms) IRREGULARS[form] = base;
+}
+
+const DOUBLED_CONSONANT = /[bcdfgklmnprstvz]/;
+
+// "runn" -> "run", "bigg" -> "big"; only collapses a doubled final consonant.
+const undouble = (stem: string): string | null => {
+  const last = stem[stem.length - 1];
+  if (stem.length >= 2 && last === stem[stem.length - 2] && last && DOUBLED_CONSONANT.test(last)) {
+    return stem.slice(0, -1);
+  }
+  return null;
+};
+
+const applySuffixRules = (word: string, push: (candidate: string) => void): void => {
+  // --- plural / third-person present ---
+  if (word.endsWith('ies') && word.length > 4) push(word.slice(0, -3) + 'y'); // cities -> city
+  if (word.endsWith('ves') && word.length > 3) {
+    push(word.slice(0, -3) + 'f'); // wolves -> wolf
+    push(word.slice(0, -3) + 'fe'); // knives -> knife
+  }
+  // Greek/Latin -ses plural; tried before generic -es so the noun wins.
+  if (word.endsWith('ses') && word.length > 3) push(word.slice(0, -3) + 'sis'); // analyses -> analysis
+  if (/(s|x|z|ch|sh)es$/.test(word)) push(word.slice(0, -2)); // boxes -> box, dishes -> dish
+  if (word.endsWith('es') && word.length > 2) push(word.slice(0, -1)); // houses -> house
+  if (word.endsWith('s') && !word.endsWith('ss') && word.length > 1) push(word.slice(0, -1)); // cats -> cat
+
+  // --- past tense / past participle ---
+  if (word.endsWith('ied') && word.length > 3) push(word.slice(0, -3) + 'y'); // studied -> study
+  if (word.endsWith('ed') && word.length > 2) {
+    push(word.slice(0, -2)); // walked -> walk
+    push(word.slice(0, -1)); // realised -> realise, used -> use
+    const undoubled = undouble(word.slice(0, -2));
+    if (undoubled) push(undoubled); // stopped -> stop
+  }
+
+  // --- present participle / gerund ---
+  if (word.endsWith('ying') && word.length > 4) push(word.slice(0, -4) + 'ie'); // lying -> lie
+  if (word.endsWith('ing') && word.length > 3) {
+    push(word.slice(0, -3)); // walking -> walk
+    push(word.slice(0, -3) + 'e'); // making -> make
+    const undoubled = undouble(word.slice(0, -3));
+    if (undoubled) push(undoubled); // running -> run
+  }
+
+  // --- comparative / superlative ---
+  if (word.endsWith('iest') && word.length > 4) push(word.slice(0, -4) + 'y'); // happiest -> happy
+  if (word.endsWith('ier') && word.length > 3) push(word.slice(0, -3) + 'y'); // happier -> happy
+  if (word.endsWith('est') && word.length > 3) {
+    push(word.slice(0, -3)); // fastest -> fast
+    push(word.slice(0, -2)); // largest -> large
+    const undoubled = undouble(word.slice(0, -3));
+    if (undoubled) push(undoubled); // biggest -> big
+  }
+  if (word.endsWith('er') && word.length > 2) {
+    push(word.slice(0, -2)); // faster -> fast
+    push(word.slice(0, -1)); // larger -> large
+    const undoubled = undouble(word.slice(0, -2));
+    if (undoubled) push(undoubled); // bigger -> big
+  }
+
+  // --- adverb ---
+  if (word.endsWith('ly') && word.length > 2) push(word.slice(0, -2)); // quickly -> quick
+};
+
+export const lemmatizeEnglish = (word: string): string[] => {
+  const lower = word.toLowerCase();
+  if (!/^[a-z][a-z'’-]*$/.test(lower)) return [];
+
+  const out: string[] = [];
+  const push = (candidate: string): void => {
+    // Skip empties, single letters, the input itself, and duplicates.
+    if (candidate.length > 1 && candidate !== lower && !out.includes(candidate)) {
+      out.push(candidate);
+    }
+  };
+
+  // Possessive: "cat's" / "dogs'" -> drop the clitic and lemmatize the noun.
+  const stripped = lower.replace(/['’]s?$/, '');
+  const root = stripped !== lower ? stripped : lower;
+  if (stripped !== lower) push(stripped);
+
+  if (IRREGULARS[root]) push(IRREGULARS[root]);
+  applySuffixRules(root, push);
+
+  return out;
+};

apps/readest-app/src/services/dictionaries/lemmatize/index.ts

@@ -0,0 +1,34 @@
+/**
+ * Pluggable, language-aware lemmatizer registry for dictionary lookup.
+ *
+ * Dictionaries like the Oxford Dictionary of English store only base headwords,
+ * so an exact match on an inflected selection (`ran`, `mice`, `analyses`) misses
+ * even though the lemma (`run`, `mouse`, `analysis`) is present. Lookup callers
+ * append these lemma candidates after the exact/case variants so the lemma is
+ * only tried once an exact match fails — exact match always wins.
+ *
+ * To add another language, write its lemmatizer and register it under the
+ * primary subtag below; no caller changes are needed.
+ */
+import { normalizedLangCode } from '@/utils/lang';
+import { lemmatizeEnglish } from './english';
+
+/** Maps a single inflected word to ordered, de-duplicated base-form candidates. */
+export type Lemmatizer = (word: string) => string[];
+
+const REGISTRY: Record<string, Lemmatizer> = {
+  en: lemmatizeEnglish,
+};
+
+/**
+ * Base-form candidates for `word` in the given language. The language is
+ * normalized to its primary subtag (`en-US` → `en`). When the language is
+ * missing or unknown we default to English — imported dictionaries are
+ * overwhelmingly English and the English lemmatizer is a no-op on non-ASCII
+ * text. An *explicit* language with no registered lemmatizer yields `[]`.
+ */
+export const getLemmaCandidates = (word: string, lang?: string | null): string[] => {
+  const code = normalizedLangCode(lang) || 'en';
+  const lemmatizer = REGISTRY[code];
+  return lemmatizer ? lemmatizer(word) : [];
+};

apps/readest-app/src/services/dictionaries/lookupCandidates.ts

@@ -1,3 +1,5 @@
+import { getLemmaCandidates } from './lemmatize';
+
 /**
  * Ordered, de-duplicated query variants for a dictionary lookup.
  *
@@ -9,14 +11,18 @@
  * not. Callers try each candidate in order and keep the first hit.
  *
  * Variants, in priority order: the trimmed selection as-is, all-lowercase,
- * title-case, all-uppercase (for acronym headwords). Returns `[]` for a
- * blank input.
+ * title-case, all-uppercase (for acronym headwords), then language-aware
+ * lemma candidates (`ran → run`, `mice → mouse`). The lemmas sit at the tail
+ * so an exact/case match always wins; they are only reached once the lookup
+ * loop exhausts the exact forms with empty results. Returns `[]` for a blank
+ * input.
  */
-export const buildLookupCandidates = (word: string): string[] => {
+export const buildLookupCandidates = (word: string, lang?: string | null): string[] => {
   const trimmed = word.trim();
   if (!trimmed) return [];
   const lower = trimmed.toLowerCase();
   const title = trimmed.charAt(0).toUpperCase() + lower.slice(1);
   const upper = trimmed.toUpperCase();
-  return [...new Set([trimmed, lower, title, upper])];
+  const lemmas = getLemmaCandidates(lower, lang);
+  return [...new Set([trimmed, lower, title, upper, ...lemmas])];
 };