diff --git a/apps/readest-app/src/__tests__/services/dictionaries/lemmatize/english.test.ts b/apps/readest-app/src/__tests__/services/dictionaries/lemmatize/english.test.ts new file mode 100644 index 00000000..ec73f1e4 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/dictionaries/lemmatize/english.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect } from 'vitest'; + +import { lemmatizeEnglish } from '@/services/dictionaries/lemmatize/english'; + +describe('lemmatizeEnglish', () => { + describe('irregular forms (issue test cases)', () => { + it('maps irregular verb forms to their base', () => { + expect(lemmatizeEnglish('ran')).toContain('run'); + expect(lemmatizeEnglish('went')).toContain('go'); + expect(lemmatizeEnglish('gone')).toContain('go'); + }); + + it('maps irregular plurals to their singular', () => { + expect(lemmatizeEnglish('mice')).toContain('mouse'); + expect(lemmatizeEnglish('children')).toContain('child'); + }); + + it('maps irregular comparatives/superlatives to their base adjective', () => { + expect(lemmatizeEnglish('better')).toContain('good'); + expect(lemmatizeEnglish('best')).toContain('good'); + expect(lemmatizeEnglish('worse')).toContain('bad'); + }); + }); + + describe('regular suffix rules (issue test cases)', () => { + it('reduces Greek/Latin -ses plurals to -sis', () => { + expect(lemmatizeEnglish('analyses')).toContain('analysis'); + }); + + it('prefers the -sis noun ahead of the -se verb for -ses words', () => { + const candidates = lemmatizeEnglish('analyses'); + const analysis = candidates.indexOf('analysis'); + const analyse = candidates.indexOf('analyse'); + expect(analysis).toBeGreaterThanOrEqual(0); + // When both surface, the noun (issue's expected lookup) comes first. + if (analyse >= 0) expect(analysis).toBeLessThan(analyse); + }); + + it('drops a trailing -d from -ed words built on an e-stem', () => { + expect(lemmatizeEnglish('realised')).toContain('realise'); + }); + }); + + describe('regular suffix rule families', () => { + it('handles regular plurals and third-person -s/-es', () => { + expect(lemmatizeEnglish('cats')).toContain('cat'); + expect(lemmatizeEnglish('boxes')).toContain('box'); + expect(lemmatizeEnglish('dishes')).toContain('dish'); + expect(lemmatizeEnglish('cities')).toContain('city'); + expect(lemmatizeEnglish('wolves')).toContain('wolf'); + expect(lemmatizeEnglish('knives')).toContain('knife'); + }); + + it('handles regular past tense, including doubled consonants', () => { + expect(lemmatizeEnglish('walked')).toContain('walk'); + expect(lemmatizeEnglish('studied')).toContain('study'); + expect(lemmatizeEnglish('stopped')).toContain('stop'); + expect(lemmatizeEnglish('baked')).toContain('bake'); + }); + + it('handles -ing forms, including e-restoration and doubled consonants', () => { + expect(lemmatizeEnglish('walking')).toContain('walk'); + expect(lemmatizeEnglish('running')).toContain('run'); + expect(lemmatizeEnglish('making')).toContain('make'); + expect(lemmatizeEnglish('lying')).toContain('lie'); + }); + + it('handles comparatives and superlatives', () => { + expect(lemmatizeEnglish('faster')).toContain('fast'); + expect(lemmatizeEnglish('fastest')).toContain('fast'); + expect(lemmatizeEnglish('larger')).toContain('large'); + expect(lemmatizeEnglish('happier')).toContain('happy'); + expect(lemmatizeEnglish('happiest')).toContain('happy'); + expect(lemmatizeEnglish('bigger')).toContain('big'); + }); + + it('strips possessive clitics', () => { + expect(lemmatizeEnglish("cat's")).toContain('cat'); + expect(lemmatizeEnglish("dogs'")).toContain('dog'); + }); + }); + + describe('guards', () => { + it('returns an empty list for multi-word, numeric, or non-ASCII input', () => { + expect(lemmatizeEnglish('hello world')).toEqual([]); + expect(lemmatizeEnglish('123')).toEqual([]); + expect(lemmatizeEnglish('café')).toEqual([]); + expect(lemmatizeEnglish('')).toEqual([]); + expect(lemmatizeEnglish(' ')).toEqual([]); + }); + + it('is case-insensitive on the input', () => { + expect(lemmatizeEnglish('Ran')).toContain('run'); + expect(lemmatizeEnglish('MICE')).toContain('mouse'); + }); + + it('never returns the input word itself or single letters', () => { + expect(lemmatizeEnglish('run')).not.toContain('run'); + expect(lemmatizeEnglish('cat')).not.toContain('cat'); + expect(lemmatizeEnglish('best')).not.toContain('b'); + }); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/dictionaries/lemmatize/index.test.ts b/apps/readest-app/src/__tests__/services/dictionaries/lemmatize/index.test.ts new file mode 100644 index 00000000..c79474f7 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/dictionaries/lemmatize/index.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect } from 'vitest'; + +import { getLemmaCandidates } from '@/services/dictionaries/lemmatize'; + +describe('getLemmaCandidates', () => { + it('lemmatizes words for an English language code', () => { + expect(getLemmaCandidates('ran', 'en')).toContain('run'); + }); + + it('normalizes regional/script English subtags to the en lemmatizer', () => { + expect(getLemmaCandidates('mice', 'en-US')).toContain('mouse'); + expect(getLemmaCandidates('mice', 'en-GB')).toContain('mouse'); + }); + + it('defaults to English when the language is missing or empty', () => { + expect(getLemmaCandidates('ran')).toContain('run'); + expect(getLemmaCandidates('ran', undefined)).toContain('run'); + expect(getLemmaCandidates('ran', '')).toContain('run'); + expect(getLemmaCandidates('ran', null)).toContain('run'); + }); + + it('returns [] for an explicit language with no registered lemmatizer', () => { + expect(getLemmaCandidates('mange', 'fr')).toEqual([]); + expect(getLemmaCandidates('rennt', 'de')).toEqual([]); + expect(getLemmaCandidates('mice', 'zh')).toEqual([]); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/dictionaries/lookupCandidates.test.ts b/apps/readest-app/src/__tests__/services/dictionaries/lookupCandidates.test.ts index feaea3e5..c94778ce 100644 --- a/apps/readest-app/src/__tests__/services/dictionaries/lookupCandidates.test.ts +++ b/apps/readest-app/src/__tests__/services/dictionaries/lookupCandidates.test.ts @@ -8,8 +8,9 @@ describe('buildLookupCandidates', () => { }); it('trims leading/trailing whitespace from a double-click selection', () => { + // Non-inflecting words keep this focused on trimming + case folding. expect(buildLookupCandidates('world ')).toEqual(['world', 'World', 'WORLD']); - expect(buildLookupCandidates(' spaced ')).toEqual(['spaced', 'Spaced', 'SPACED']); + expect(buildLookupCandidates(' planet ')).toEqual(['planet', 'Planet', 'PLANET']); }); it('offers a lowercase variant for a sentence-initial capitalized word', () => { @@ -29,4 +30,47 @@ describe('buildLookupCandidates', () => { expect(buildLookupCandidates('')).toEqual([]); expect(buildLookupCandidates(' ')).toEqual([]); }); + + describe('lemmatization fallback', () => { + it('appends lemma candidates after the exact/case variants', () => { + const candidates = buildLookupCandidates('ran', 'en'); + expect(candidates[0]).toBe('ran'); // exact selection tried first + expect(candidates).toContain('run'); + expect(candidates.indexOf('run')).toBeGreaterThan(candidates.indexOf('ran')); + }); + + it('keeps every exact/case variant ahead of any lemma', () => { + const candidates = buildLookupCandidates('Mice', 'en'); + const lastCaseVariant = Math.max( + candidates.indexOf('Mice'), + candidates.indexOf('mice'), + candidates.indexOf('MICE'), + ); + expect(candidates.indexOf('mouse')).toBeGreaterThan(lastCaseVariant); + }); + + it('resolves the issue test cases to their expected lemma', () => { + const cases: Array<[string, string]> = [ + ['ran', 'run'], + ['went', 'go'], + ['gone', 'go'], + ['mice', 'mouse'], + ['children', 'child'], + ['better', 'good'], + ['analyses', 'analysis'], + ['realised', 'realise'], + ]; + for (const [selection, lemma] of cases) { + expect(buildLookupCandidates(selection, 'en')).toContain(lemma); + } + }); + + it('defaults to English lemmatization when no language is given', () => { + expect(buildLookupCandidates('ran')).toContain('run'); + }); + + it('does not lemmatize for an explicit non-English language', () => { + expect(buildLookupCandidates('ran', 'fr')).toEqual(['ran', 'Ran', 'RAN']); + }); + }); }); diff --git a/apps/readest-app/src/app/reader/components/annotator/DictionaryResultsView.tsx b/apps/readest-app/src/app/reader/components/annotator/DictionaryResultsView.tsx index 02f670e3..4e29c436 100644 --- a/apps/readest-app/src/app/reader/components/annotator/DictionaryResultsView.tsx +++ b/apps/readest-app/src/app/reader/components/annotator/DictionaryResultsView.tsx @@ -211,12 +211,15 @@ export function useDictionaryResults({ if (!container) { outcome = { ok: false, reason: 'error', message: 'no container' }; } else { - // Try normalized query variants (trimmed, case-folded) in - // priority order and keep the first hit. Case-sensitive - // formats (mdict) otherwise miss `Hello` / `world ` style - // selections whose headword is stored lowercased. + // Try normalized query variants (trimmed, case-folded) then + // language-aware lemma candidates in priority order, keeping the + // first hit. Case-sensitive formats (mdict) otherwise miss + // `Hello` / `world ` style selections whose headword is stored + // lowercased, and dictionaries that store only base headwords + // (e.g. Oxford Dictionary of English) miss inflected selections + // like `ran` / `mice` / `analyses`. outcome = { ok: false, reason: 'empty' }; - for (const candidate of buildLookupCandidates(currentWord)) { + for (const candidate of buildLookupCandidates(currentWord, langCode)) { container.replaceChildren(); outcome = await provider.lookup(candidate, { lang: langCode, diff --git a/apps/readest-app/src/services/dictionaries/lemmatize/english.ts b/apps/readest-app/src/services/dictionaries/lemmatize/english.ts new file mode 100644 index 00000000..ba4598ab --- /dev/null +++ b/apps/readest-app/src/services/dictionaries/lemmatize/english.ts @@ -0,0 +1,226 @@ +/** + * English lemmatizer for dictionary lookup fallback. + * + * Given an inflected English word it returns an ordered, de-duplicated list of + * candidate base forms (e.g. `ran → run`, `mice → mouse`, `analyses → + * analysis`). The list is intentionally *over-generated*: the dictionary + * lookup itself is the validator, so a bogus stem simply misses and the caller + * moves on. The rules therefore only need to *include* the correct base, not + * be linguistically precise. + * + * Two layers, in priority order: + * 1. an irregular-form table (common suppletive verbs, irregular plurals, + * and irregular comparatives) — these can't be derived by rule; + * 2. regular suffix rules (plural / past / gerund / comparative / possessive). + * + * Only single ASCII-alphabetic tokens are lemmatized; phrases, numbers, and + * accented/CJK text return `[]`. + */ + +// Base form -> the irregular inflected forms that should resolve to it. +// Readable as groups; flattened to an inflected->base map at module load. +const IRREGULAR_GROUPS: Record = { + // Suppletive / highly irregular verbs. + be: ['is', 'am', 'are', 'was', 'were', 'been', 'being'], + have: ['has', 'had', 'having'], + do: ['does', 'did', 'done', 'doing'], + go: ['goes', 'went', 'gone', 'going'], + say: ['said'], + get: ['got', 'gotten'], + make: ['made'], + know: ['knew', 'known'], + think: ['thought'], + take: ['took', 'taken'], + see: ['saw', 'seen'], + come: ['came'], + find: ['found'], + give: ['gave', 'given'], + tell: ['told'], + feel: ['felt'], + become: ['became'], + leave: ['left'], + mean: ['meant'], + keep: ['kept'], + begin: ['began', 'begun'], + show: ['showed', 'shown'], + hear: ['heard'], + run: ['ran'], + bring: ['brought'], + write: ['wrote', 'written'], + sit: ['sat'], + stand: ['stood'], + lose: ['lost'], + pay: ['paid'], + meet: ['met'], + learn: ['learnt'], + lead: ['led'], + understand: ['understood'], + speak: ['spoke', 'spoken'], + spend: ['spent'], + grow: ['grew', 'grown'], + win: ['won'], + teach: ['taught'], + buy: ['bought'], + send: ['sent'], + build: ['built'], + fall: ['fell', 'fallen'], + catch: ['caught'], + draw: ['drew', 'drawn'], + choose: ['chose', 'chosen'], + drive: ['drove', 'driven'], + break: ['broke', 'broken'], + eat: ['ate', 'eaten'], + drink: ['drank', 'drunk'], + sing: ['sang', 'sung'], + swim: ['swam', 'swum'], + ring: ['rang', 'rung'], + fly: ['flew', 'flown'], + throw: ['threw', 'thrown'], + wear: ['wore', 'worn'], + tear: ['tore', 'torn'], + sell: ['sold'], + hold: ['held'], + feed: ['fed'], + fight: ['fought'], + hide: ['hid', 'hidden'], + ride: ['rode', 'ridden'], + rise: ['rose', 'risen'], + shake: ['shook', 'shaken'], + steal: ['stole', 'stolen'], + freeze: ['froze', 'frozen'], + sleep: ['slept'], + bite: ['bit', 'bitten'], + hang: ['hung'], + shoot: ['shot'], + sink: ['sank', 'sunk'], + forget: ['forgot', 'forgotten'], + forgive: ['forgave', 'forgiven'], + lay: ['laid'], + deal: ['dealt'], + dig: ['dug'], + shine: ['shone'], + bend: ['bent'], + lend: ['lent'], + blow: ['blew', 'blown'], + beat: ['beaten'], + arise: ['arose', 'arisen'], + awake: ['awoke', 'awoken'], + // Irregular plurals. + man: ['men'], + woman: ['women'], + child: ['children'], + mouse: ['mice'], + louse: ['lice'], + goose: ['geese'], + foot: ['feet'], + tooth: ['teeth'], + person: ['people'], + ox: ['oxen'], + die: ['dice'], + criterion: ['criteria'], + phenomenon: ['phenomena'], + cactus: ['cacti'], + fungus: ['fungi'], + nucleus: ['nuclei'], + radius: ['radii'], + alumnus: ['alumni'], + index: ['indices'], + matrix: ['matrices'], + vertex: ['vertices'], + appendix: ['appendices'], + // Irregular comparatives / superlatives (adjective & adverb). + good: ['better', 'best', 'well'], + bad: ['worse', 'worst'], + far: ['further', 'furthest', 'farther', 'farthest'], + little: ['less', 'least'], +}; + +const IRREGULARS: Record = {}; +for (const [base, forms] of Object.entries(IRREGULAR_GROUPS)) { + for (const form of forms) IRREGULARS[form] = base; +} + +const DOUBLED_CONSONANT = /[bcdfgklmnprstvz]/; + +// "runn" -> "run", "bigg" -> "big"; only collapses a doubled final consonant. +const undouble = (stem: string): string | null => { + const last = stem[stem.length - 1]; + if (stem.length >= 2 && last === stem[stem.length - 2] && last && DOUBLED_CONSONANT.test(last)) { + return stem.slice(0, -1); + } + return null; +}; + +const applySuffixRules = (word: string, push: (candidate: string) => void): void => { + // --- plural / third-person present --- + if (word.endsWith('ies') && word.length > 4) push(word.slice(0, -3) + 'y'); // cities -> city + if (word.endsWith('ves') && word.length > 3) { + push(word.slice(0, -3) + 'f'); // wolves -> wolf + push(word.slice(0, -3) + 'fe'); // knives -> knife + } + // Greek/Latin -ses plural; tried before generic -es so the noun wins. + if (word.endsWith('ses') && word.length > 3) push(word.slice(0, -3) + 'sis'); // analyses -> analysis + if (/(s|x|z|ch|sh)es$/.test(word)) push(word.slice(0, -2)); // boxes -> box, dishes -> dish + if (word.endsWith('es') && word.length > 2) push(word.slice(0, -1)); // houses -> house + if (word.endsWith('s') && !word.endsWith('ss') && word.length > 1) push(word.slice(0, -1)); // cats -> cat + + // --- past tense / past participle --- + if (word.endsWith('ied') && word.length > 3) push(word.slice(0, -3) + 'y'); // studied -> study + if (word.endsWith('ed') && word.length > 2) { + push(word.slice(0, -2)); // walked -> walk + push(word.slice(0, -1)); // realised -> realise, used -> use + const undoubled = undouble(word.slice(0, -2)); + if (undoubled) push(undoubled); // stopped -> stop + } + + // --- present participle / gerund --- + if (word.endsWith('ying') && word.length > 4) push(word.slice(0, -4) + 'ie'); // lying -> lie + if (word.endsWith('ing') && word.length > 3) { + push(word.slice(0, -3)); // walking -> walk + push(word.slice(0, -3) + 'e'); // making -> make + const undoubled = undouble(word.slice(0, -3)); + if (undoubled) push(undoubled); // running -> run + } + + // --- comparative / superlative --- + if (word.endsWith('iest') && word.length > 4) push(word.slice(0, -4) + 'y'); // happiest -> happy + if (word.endsWith('ier') && word.length > 3) push(word.slice(0, -3) + 'y'); // happier -> happy + if (word.endsWith('est') && word.length > 3) { + push(word.slice(0, -3)); // fastest -> fast + push(word.slice(0, -2)); // largest -> large + const undoubled = undouble(word.slice(0, -3)); + if (undoubled) push(undoubled); // biggest -> big + } + if (word.endsWith('er') && word.length > 2) { + push(word.slice(0, -2)); // faster -> fast + push(word.slice(0, -1)); // larger -> large + const undoubled = undouble(word.slice(0, -2)); + if (undoubled) push(undoubled); // bigger -> big + } + + // --- adverb --- + if (word.endsWith('ly') && word.length > 2) push(word.slice(0, -2)); // quickly -> quick +}; + +export const lemmatizeEnglish = (word: string): string[] => { + const lower = word.toLowerCase(); + if (!/^[a-z][a-z'’-]*$/.test(lower)) return []; + + const out: string[] = []; + const push = (candidate: string): void => { + // Skip empties, single letters, the input itself, and duplicates. + if (candidate.length > 1 && candidate !== lower && !out.includes(candidate)) { + out.push(candidate); + } + }; + + // Possessive: "cat's" / "dogs'" -> drop the clitic and lemmatize the noun. + const stripped = lower.replace(/['’]s?$/, ''); + const root = stripped !== lower ? stripped : lower; + if (stripped !== lower) push(stripped); + + if (IRREGULARS[root]) push(IRREGULARS[root]); + applySuffixRules(root, push); + + return out; +}; diff --git a/apps/readest-app/src/services/dictionaries/lemmatize/index.ts b/apps/readest-app/src/services/dictionaries/lemmatize/index.ts new file mode 100644 index 00000000..e81d0801 --- /dev/null +++ b/apps/readest-app/src/services/dictionaries/lemmatize/index.ts @@ -0,0 +1,34 @@ +/** + * Pluggable, language-aware lemmatizer registry for dictionary lookup. + * + * Dictionaries like the Oxford Dictionary of English store only base headwords, + * so an exact match on an inflected selection (`ran`, `mice`, `analyses`) misses + * even though the lemma (`run`, `mouse`, `analysis`) is present. Lookup callers + * append these lemma candidates after the exact/case variants so the lemma is + * only tried once an exact match fails — exact match always wins. + * + * To add another language, write its lemmatizer and register it under the + * primary subtag below; no caller changes are needed. + */ +import { normalizedLangCode } from '@/utils/lang'; +import { lemmatizeEnglish } from './english'; + +/** Maps a single inflected word to ordered, de-duplicated base-form candidates. */ +export type Lemmatizer = (word: string) => string[]; + +const REGISTRY: Record = { + en: lemmatizeEnglish, +}; + +/** + * Base-form candidates for `word` in the given language. The language is + * normalized to its primary subtag (`en-US` → `en`). When the language is + * missing or unknown we default to English — imported dictionaries are + * overwhelmingly English and the English lemmatizer is a no-op on non-ASCII + * text. An *explicit* language with no registered lemmatizer yields `[]`. + */ +export const getLemmaCandidates = (word: string, lang?: string | null): string[] => { + const code = normalizedLangCode(lang) || 'en'; + const lemmatizer = REGISTRY[code]; + return lemmatizer ? lemmatizer(word) : []; +}; diff --git a/apps/readest-app/src/services/dictionaries/lookupCandidates.ts b/apps/readest-app/src/services/dictionaries/lookupCandidates.ts index 72278951..a8c4228d 100644 --- a/apps/readest-app/src/services/dictionaries/lookupCandidates.ts +++ b/apps/readest-app/src/services/dictionaries/lookupCandidates.ts @@ -1,3 +1,5 @@ +import { getLemmaCandidates } from './lemmatize'; + /** * Ordered, de-duplicated query variants for a dictionary lookup. * @@ -9,14 +11,18 @@ * not. Callers try each candidate in order and keep the first hit. * * Variants, in priority order: the trimmed selection as-is, all-lowercase, - * title-case, all-uppercase (for acronym headwords). Returns `[]` for a - * blank input. + * title-case, all-uppercase (for acronym headwords), then language-aware + * lemma candidates (`ran → run`, `mice → mouse`). The lemmas sit at the tail + * so an exact/case match always wins; they are only reached once the lookup + * loop exhausts the exact forms with empty results. Returns `[]` for a blank + * input. */ -export const buildLookupCandidates = (word: string): string[] => { +export const buildLookupCandidates = (word: string, lang?: string | null): string[] => { const trimmed = word.trim(); if (!trimmed) return []; const lower = trimmed.toLowerCase(); const title = trimmed.charAt(0).toUpperCase() + lower.slice(1); const upper = trimmed.toUpperCase(); - return [...new Set([trimmed, lower, title, upper])]; + const lemmas = getLemmaCandidates(lower, lang); + return [...new Set([trimmed, lower, title, upper, ...lemmas])]; };