feat(wordlens): en-en glosses, styling, derivation lemmas, display-time cap (#4744)
* feat(wordlens): support en-en monolingual glosses Gloss difficult English words with a short English definition for learners reading English with English hints. - build: buildEnEn + shortDefGloss read ECDICT's English `definition` column (first/primary WordNet sense, POS-stripped, drop ;-example, <=24 word-boundary with trailing-connector trim). New `en-en` CLI branch; buildEnZh/buildEnEn now share a buildEnPack core. - gating: drop the hardcoded `hint === source` rejections (wordlensSection, WordLensPanel) so same-language packs are allowed; availability is decided by the manifest (resolvePack returns null when no pack exists). - data: data/wordlens/en-en.json (26,578 entries) + regenerated manifest.json. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * feat(wordlens): gloss styling, derivation lemmas, display-time cap Builds on the en-en monolingual gloss support with refinements and regenerated packs. - settings: per-book gloss <rt> font size (em) and color in Settings > Language > Word Lens (getRubyStyles reads viewSettings). - en-en hints: WordNet hybrid (a simpler synonym, else a category hypernym, else the ECDICT definition) instead of raw verbose definitions. - lemmatization: gate difficulty by the lemma rank for every English source pair. enBaseFormCandidates now also covers -able/-ible suffixes and negative prefixes (un/in/im/ir/il), so insufferable resolves to suffer. A candidate is accepted when the English definition names the base OR the Chinese translations share a content character, which keeps true derivations (insufferable -> suffer) and rejects coincidental stems (capable -> cap). en-X packs inherit the en-en lemma table. - display cap: the max gloss length is applied at render time in cleanGloss (MAX_GLOSS_LEN), so the packs store the full hint and the cap can change without regenerating data. - tooling: pnpm wordlens:preview to sample pack entries; cache build corpora under data/wordlens/.sources (gitignored). - data: regenerate en-en, en-zh and en-de/es/fr/pt/ru plus the manifest. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Huang Xin
@@ -83,3 +83,7 @@ docs/superpowers
|
||||
/blob-report/
|
||||
/playwright/.cache/
|
||||
|
||||
|
||||
# Word Lens build-input corpora (ECDICT, WikDict, FrequencyWords, etc.) — large,
|
||||
# downloaded on demand by scripts/build-wordlens-data.mjs; cached locally, not committed.
|
||||
/data/wordlens/.sources/
|
||||
|
||||
@@ -17,7 +17,7 @@ See `ATTRIBUTION.md` for the data sources + licenses.
|
||||
```bash
|
||||
mkdir -p /tmp/ww-data && cd /tmp/ww-data
|
||||
|
||||
# en→中文: ECDICT (MIT) — ~66 MB
|
||||
# en→中文 + en→en: ECDICT (MIT) — ~66 MB
|
||||
curl -sL -o ecdict.csv https://raw.githubusercontent.com/skywind3000/ECDICT/master/ecdict.csv
|
||||
|
||||
# 中文→en: CC-CEDICT (CC-BY-SA) + HSK levels (drkameleon)
|
||||
@@ -34,8 +34,20 @@ for c in es fr de pt it ru; do curl -sL -o lemmatization-$c.txt https://raw.gith
|
||||
```
|
||||
|
||||
## 2. Generate packs (run from `apps/readest-app`)
|
||||
> **Order matters:** build `en-zh` **first** — the en→X WikDict builds reuse its English
|
||||
> inflection table to lemmatize (`kept`→`keep`).
|
||||
> **Order matters:** build `en-en` **first** — every en→X build reuses its English
|
||||
> inflection + derivation table to lemmatize (enforced: en→X throws without `en-en.json`).
|
||||
>
|
||||
> **Lemmatization rule (all English-source pairs):** the gloss difficulty is gated by
|
||||
> the LEMMA's frequency rank. `en-en`/`en-zh` (built directly via `buildEnPack`) resolve
|
||||
> both inflected (`kept`→`keep`, via ECDICT `exchange`) AND transparently-derived forms
|
||||
> (`thickly`→`thick`, `kindness`→`kind`, `insufferable`→`suffer`, via
|
||||
> `enBaseFormCandidates` — suffixes `-ly/-ful/-ness/-less/-ward/-able/-ible` and negative
|
||||
> prefixes `un-/in-/im-/ir-/il-`) to their lemma. A candidate is accepted when the ECDICT
|
||||
> `definition` names the base OR the Chinese `translation` shares a content character with
|
||||
> it (for meaning-shifting families whose def never names the base). Drift (`hardly`⇏
|
||||
> `hard`) and coincidental stems (`ally`⇏`ale`, `capable`⇏`cap`) are rejected by both
|
||||
> checks. en→X packs inherit this via the `en-en` table — `en-en` is the canonical
|
||||
> English lemma source.
|
||||
```bash
|
||||
cd apps/readest-app
|
||||
|
||||
@@ -43,12 +55,19 @@ cd apps/readest-app
|
||||
node scripts/build-wordlens-data.mjs en-zh /tmp/ww-data/ecdict.csv 30000
|
||||
node scripts/build-wordlens-data.mjs zh-en /tmp/ww-data/cedict.txt /tmp/ww-data/hsk.json 12000
|
||||
|
||||
# en→en (monolingual), short English hints for learners: a simpler SYNONYM → else a
|
||||
# category (WordNet HYPERNYM) → else the ECDICT `definition` (first ≤2 senses). Packs
|
||||
# store the FULL hint; the display-time length cap lives in `cleanGloss` (runtime), so
|
||||
# changing it needs no regeneration. Needs WordNet (synsets + hypernyms):
|
||||
# `npm pack wordnet-db && tar xzf wordnet-db-*.tgz` gives package/dict. Reuses ecdict.csv.
|
||||
node scripts/build-wordlens-data.mjs en-en /tmp/ww-data/ecdict.csv /tmp/ww-data/package/dict 30000
|
||||
|
||||
# X→en (foreign source): pass the source-language lemmatization list (6th arg) so
|
||||
# inflected source words ("corriendo" -> "correr") resolve to their lemma's gloss.
|
||||
for src in es fr de pt it ru; do
|
||||
node scripts/build-wordlens-data.mjs build-wikdict "$src" en "/tmp/ww-data/${src}_50k.txt" "/tmp/ww-data/$src-en.sqlite3" 20000 "/tmp/ww-data/lemmatization-$src.txt"
|
||||
done
|
||||
# en→X (English source): lemmatized automatically via en-zh.json (build it first)
|
||||
# en→X (English source): lemmatized automatically via en-en.json (built above)
|
||||
for tgt in es fr de pt ru; do
|
||||
node scripts/build-wordlens-data.mjs build-wikdict en "$tgt" /tmp/ww-data/en_50k.txt "/tmp/ww-data/en-$tgt.sqlite3" 20000
|
||||
done
|
||||
@@ -84,4 +103,5 @@ On next load the app fetches `manifest.json` (≤5-min CDN cache), compares each
|
||||
| `topN` per pack | the build CLI's last arg |
|
||||
| commonest-N words skipped (`skipTop`) + per-chapter render cap (`DEFAULT_CAP`) | `src/services/wordlens/{planner,…}` and `buildPack` in the build script |
|
||||
| difficulty cutoffs per slider level | `src/services/wordlens/difficulty.ts` |
|
||||
| gloss cleaning (POS/`[…]`/`CL:` strip, 24-char cap) | `shortGloss` in `scripts/build-wordlens-data.mjs` |
|
||||
| gloss shaping in the pack (POS/`[…]`/`CL:` strip, ≤2 senses) | `shortGloss`/`shortDefGloss` in `scripts/build-wordlens-data.mjs` |
|
||||
| display length cap (`MAX_GLOSS_LEN`, applied at render, no regen) | `cleanGloss` in `src/services/wordlens/gloss.ts` |
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -15,54 +15,63 @@
|
||||
"source": "en",
|
||||
"target": "de",
|
||||
"file": "en-de.json",
|
||||
"bytes": 962032,
|
||||
"sha256": "e7475f241743006278a3d771b08570f4bc9a49faa32007837e7a74461ee188e9",
|
||||
"entries": 14487
|
||||
"bytes": 1013755,
|
||||
"sha256": "a2919c2a0ceddca92d6cdcbcdc361b17c68f7264485ebf98a52af20533139264",
|
||||
"entries": 14926
|
||||
},
|
||||
{
|
||||
"pair": "en-en",
|
||||
"source": "en",
|
||||
"target": "en",
|
||||
"file": "en-en.json",
|
||||
"bytes": 1777292,
|
||||
"sha256": "534bf6c6d3ad23027066693f558fe1a30f01b14024ed4dda0bcd202e31cbf22e",
|
||||
"entries": 24246
|
||||
},
|
||||
{
|
||||
"pair": "en-es",
|
||||
"source": "en",
|
||||
"target": "es",
|
||||
"file": "en-es.json",
|
||||
"bytes": 946854,
|
||||
"sha256": "9b91c73c693bdade1458eb234eaa606a6490a6ec851deb229556fa4cbac21081",
|
||||
"entries": 14322
|
||||
"bytes": 997231,
|
||||
"sha256": "407a48af3067139b2ad0cf29d9d93b4e06a56b3764544cde918255f5eec2e90a",
|
||||
"entries": 14769
|
||||
},
|
||||
{
|
||||
"pair": "en-fr",
|
||||
"source": "en",
|
||||
"target": "fr",
|
||||
"file": "en-fr.json",
|
||||
"bytes": 933430,
|
||||
"sha256": "ff7dbd0a8b6a063db4ee69bcf595b718cd8f23542860a6f8377b43e0c093abdc",
|
||||
"entries": 14653
|
||||
"bytes": 982646,
|
||||
"sha256": "d067f9b4d0a9d23f08c35d3546124cdc91e7e0fed47d48c02fb812dcab10a418",
|
||||
"entries": 15070
|
||||
},
|
||||
{
|
||||
"pair": "en-pt",
|
||||
"source": "en",
|
||||
"target": "pt",
|
||||
"file": "en-pt.json",
|
||||
"bytes": 924391,
|
||||
"sha256": "776ce8b30aca99856927ddc767c432d987c79efe0024729418ada9f1c261bce8",
|
||||
"entries": 14209
|
||||
"bytes": 972984,
|
||||
"sha256": "f29b738eacb4851cb2a20e95ee418d0a17e33e8aecfbafe3bbc705d9a2db6230",
|
||||
"entries": 14656
|
||||
},
|
||||
{
|
||||
"pair": "en-ru",
|
||||
"source": "en",
|
||||
"target": "ru",
|
||||
"file": "en-ru.json",
|
||||
"bytes": 1197025,
|
||||
"sha256": "1d8e7c044d2412c6608dce7844dd0a90f40f041c95692879a9649284a55e6d6a",
|
||||
"entries": 14491
|
||||
"bytes": 1270231,
|
||||
"sha256": "3cb82f8c167f6e5302d0164dcdff27c40b92424a09ab5fc920c3ec418565395d",
|
||||
"entries": 14925
|
||||
},
|
||||
{
|
||||
"pair": "en-zh",
|
||||
"source": "en",
|
||||
"target": "zh",
|
||||
"file": "en-zh.json",
|
||||
"bytes": 2343610,
|
||||
"sha256": "693a94940e07e64c332366a9033dd3f3bd86ce18b6d439fdb7074383668bb5c6",
|
||||
"entries": 26729
|
||||
"bytes": 1803715,
|
||||
"sha256": "4132a5b9262d6aaf933022be4a060fb41a7ac3fe71f2ccc82618c3215e35f60c",
|
||||
"entries": 24446
|
||||
},
|
||||
{
|
||||
"pair": "es-en",
|
||||
|
||||
@@ -83,6 +83,7 @@
|
||||
"restore-build-original": "if [ \"$(uname)\" = \"Darwin\" ]; then sed -i '' 's/next build --webpack\"/next build\"/' package.json; else sed -i 's/next build --webpack\"/next build\"/' package.json; fi",
|
||||
"update-metadata": "bash ./scripts/sync-release-notes.sh release-notes.json ../../data/metainfo/appdata.xml",
|
||||
"wordlens:manifest": "node scripts/build-wordlens-data.mjs manifest",
|
||||
"wordlens:preview": "node scripts/preview-wordlens.mjs",
|
||||
"wordlens:sync": "node scripts/sync-wordlens-r2.mjs",
|
||||
"check:optional-chaining": "count=$(grep -rno '\\?\\.[a-zA-Z_$]' .next/static/chunks/* out/_next/static/chunks/* | wc -l); if [ \"$count\" -gt 0 ]; then echo '❌ Optional chaining found in output!'; exit 1; else echo '✅ No optional chaining found.'; fi",
|
||||
"check:translations": "count=$(grep -rno '__STRING_NOT_TRANSLATED__' public/locales/* | wc -l); if [ \"$count\" -gt 0 ]; then echo '❌ Untranslated strings found!'; exit 1; else echo '✅ All strings translated.'; fi",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Build trimmed Word Lens gloss indices from open datasets.
|
||||
//
|
||||
// node scripts/build-wordlens-data.mjs en-zh path/to/ecdict.csv [topN]
|
||||
// node scripts/build-wordlens-data.mjs en-en path/to/ecdict.csv path/to/wordnet/dict [topN]
|
||||
// node scripts/build-wordlens-data.mjs zh-en path/to/cedict.txt path/to/hsk.json [topN]
|
||||
// node scripts/build-wordlens-data.mjs build <src> <tgt> <freq.txt> <gloss.jsonl> [topN]
|
||||
//
|
||||
@@ -36,11 +37,14 @@ import { execFileSync } from 'node:child_process';
|
||||
const OUT_DIR = resolve('data/wordlens');
|
||||
const TOP_DEFAULT = 30000;
|
||||
|
||||
// Keep a hint short + clean: drop bracket annotations ([医], [网络], [ge4]),
|
||||
// leading part-of-speech tags (ECDICT "a." / "vt." / "n."), and CC-CEDICT
|
||||
// classifier clauses (CL:...); then keep the first 1–2 senses.
|
||||
// Keep a hint short + clean. ";"/"/" separate SENSES; within a sense, ","/"、"
|
||||
// separate near-synonyms — keep only the first. Drop bracket annotations ([医],
|
||||
// [网络], [ge4]), leading part-of-speech tags (ECDICT "a." / "vt." / "n."), and
|
||||
// CC-CEDICT classifier clauses (CL:...); then keep at most the first two senses.
|
||||
// NOTE: no length truncation here — the display length cap lives in cleanGloss
|
||||
// (src/services/wordlens/gloss.ts), so it can change without regenerating packs.
|
||||
export function shortGloss(s) {
|
||||
return s
|
||||
const senses = s
|
||||
.split(/[;；/]/)
|
||||
.map((x) =>
|
||||
x
|
||||
@@ -48,12 +52,157 @@ export function shortGloss(s) {
|
||||
.replace(/^\s*(?:[a-zA-Z]{1,5}\.\s*)+/, '') // leading POS: "a. " "vt. "
|
||||
.replace(/\bCL:[^;；/]*/g, '') // CC-CEDICT classifier clause
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.split(/[，,、]/)[0] // first synonym within the sense
|
||||
.trim(),
|
||||
)
|
||||
.filter(Boolean)
|
||||
.filter(Boolean);
|
||||
return [...new Set(senses)] // dedupe: two senses can share a first synonym
|
||||
.slice(0, 2)
|
||||
.join('；')
|
||||
.slice(0, 24);
|
||||
.join('；');
|
||||
}
|
||||
|
||||
// Clean one ECDICT definition line: strip the leading POS code — a period-
|
||||
// terminated abbreviation ("n." / "adv." / "interj.") OR a bare single-letter
|
||||
// WordNet code without a period ("v" / "s"; the period is inconsistent in the
|
||||
// data). A BARE leading "a" is the article and is kept — only "a." is the
|
||||
// adjective code. Then drop [..] annotations, collapse whitespace, and drop a
|
||||
// Webster-style trailing dot. Keeps any ";" — those separate senses (see below).
|
||||
function cleanDefSense(s) {
|
||||
return s
|
||||
.replace(/^\s*(?:(?:[a-zA-Z]{1,6}\.|[nvsr])\s+)+/, '')
|
||||
.replace(/\[[^\]]*\]/g, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.replace(/\.+$/, '');
|
||||
}
|
||||
|
||||
// Build the en-en gloss from an ECDICT English `definition` (lines separated by a
|
||||
// literal "\n"). At most TWO senses: when ";"-joined, each ";" segment is a sense and
|
||||
// the first two are kept; when there is no ";", the first two "\n" lines are kept;
|
||||
// joined with "; ". No length truncation here — the display cap lives in cleanGloss.
|
||||
export function shortDefGloss(def) {
|
||||
const lines = String(def || '')
|
||||
.split(/\\n/)
|
||||
.map(cleanDefSense)
|
||||
.filter(Boolean);
|
||||
if (!lines.length) return '';
|
||||
const senses = (
|
||||
lines.some((l) => l.includes(';'))
|
||||
? lines.flatMap((l) => l.split(';').map((s) => s.trim())).filter(Boolean)
|
||||
: lines
|
||||
).slice(0, 2);
|
||||
return senses.join('; ');
|
||||
}
|
||||
|
||||
// --- WordNet hybrid: short en-en hints (simpler synonym → category → definition) ---
|
||||
|
||||
const cleanWnWord = (w) => w.replace(/\([^)]*\)$/, '').replace(/_/g, ' ').toLowerCase();
|
||||
// WordNet pointer POS code → file. "s" (adjective satellite) lives in data.adj.
|
||||
const PTR_POS = { n: 'noun', v: 'verb', a: 'adj', s: 'adj', r: 'adv' };
|
||||
|
||||
// Profane/vulgar synset members WordNet carries (jack/dump share a synset with
|
||||
// "shit", rooster with "cock"); never surface them in a reading aid.
|
||||
const PROFANE_GLOSS_WORDS = new Set(
|
||||
(
|
||||
'shit shite crap piss fuck fucking fucker motherfucker cunt cock cocksucker dick dickhead ' +
|
||||
'prick pussy bitch bastard asshole arsehole ass arse bugger bollocks slut whore turd wanker ' +
|
||||
'twat horseshit bullshit nigger faggot fag retard'
|
||||
).split(' '),
|
||||
);
|
||||
|
||||
// Over-generic WordNet hypernyms that don't explain anything — skip them so the hint
|
||||
// falls through to a more specific category or the definition.
|
||||
const GENERIC_HYPERNYMS = new Set([
|
||||
'entity', 'physical entity', 'abstract entity', 'object', 'thing', 'whole', 'unit', 'part',
|
||||
'portion', 'matter', 'substance', 'abstraction', 'group', 'grouping', 'collection',
|
||||
'arrangement', 'act', 'action', 'activity', 'event', 'state', 'condition', 'attribute',
|
||||
'property', 'quality', 'relation', 'magnitude', 'amount', 'measure', 'quantity', 'person',
|
||||
'individual', 'being', 'content', 'cognition', 'feeling', 'psychological feature',
|
||||
'phenomenon', 'artifact', 'instrumentality', 'device', 'structure', 'form',
|
||||
]);
|
||||
|
||||
// Parse a WordNet data.<pos> file into byKey: `${pos}:${offset}` -> { members, hyper }.
|
||||
// Synset line: "<offset> <lexfile> <ss_type> <w_cnt(hex)> <word> <lex_id> ... <p_cnt>
|
||||
// [<ptr_symbol> <offset> <pos> <src/tgt>]... | gloss". "@"/"@i" pointers are the
|
||||
// (instance) hypernym; we keep the first. Words use "_" for spaces, may carry a marker.
|
||||
export function parseWordNetData(text, pos, byKey = new Map()) {
|
||||
for (const line of text.split('\n')) {
|
||||
if (!line || line[0] === ' ') continue;
|
||||
const t = line.split(' ');
|
||||
const wcnt = parseInt(t[3], 16);
|
||||
if (!Number.isFinite(wcnt) || wcnt < 1) continue;
|
||||
const members = [];
|
||||
for (let j = 0; j < wcnt; j++) {
|
||||
const raw = t[4 + 2 * j];
|
||||
if (raw) members.push(cleanWnWord(raw));
|
||||
}
|
||||
const hm = line.match(/ @i? (\d{8}) ([nvasr]) /);
|
||||
byKey.set(`${pos}:${t[0]}`, { members, hyper: hm ? `${PTR_POS[hm[2]]}:${hm[1]}` : null });
|
||||
}
|
||||
return byKey;
|
||||
}
|
||||
|
||||
// Parse a WordNet index.<pos> file into lemma -> primary (first/most-frequent) synset
|
||||
// offset for that POS.
|
||||
export function parseWordNetIndex(text, map = new Map()) {
|
||||
for (const line of text.split('\n')) {
|
||||
if (!line || line[0] === ' ') continue;
|
||||
const t = line.split(' ');
|
||||
const lemma = cleanWnWord(t[0] || '');
|
||||
const offset = t.find((x) => /^\d{8}$/.test(x));
|
||||
if (lemma && offset && !map.has(lemma)) map.set(lemma, offset);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// Simplest synonym in a synset: the single, non-profane co-member with the lowest
|
||||
// (most common) frequency that is STRICTLY more common than `word`. '' if none.
|
||||
function simplestWnSynonym(word, key, byKey, frqMap) {
|
||||
const syn = byKey.get(key);
|
||||
if (!syn) return '';
|
||||
const own = frqMap.get(word) ?? Number.MAX_SAFE_INTEGER;
|
||||
let best = '',
|
||||
bestFrq = own;
|
||||
for (const m of syn.members) {
|
||||
if (m === word || m.includes(' ') || PROFANE_GLOSS_WORDS.has(m)) continue;
|
||||
const f = frqMap.get(m);
|
||||
if (f == null || f >= bestFrq) continue;
|
||||
bestFrq = f;
|
||||
best = m;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
// The synset's hypernym category — first non-generic, non-self member. '' if none.
|
||||
function wnHypernym(key, byKey, word) {
|
||||
const syn = byKey.get(key);
|
||||
if (!syn || !syn.hyper) return '';
|
||||
const h = byKey.get(syn.hyper);
|
||||
if (!h) return '';
|
||||
for (const m of h.members) {
|
||||
if (m === word || GENERIC_HYPERNYMS.has(m)) continue;
|
||||
return m;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
// EN→EN hint: a simpler SYNONYM, else a category (HYPERNYM), else a short DEFINITION.
|
||||
// POS priority (noun→verb→adj→adv) picks the dominant sense's synset without needing
|
||||
// sense-frequency data, avoiding wrong-POS picks (the noun "ship" → "vessel", not the
|
||||
// verb sense "send").
|
||||
export function resolveEnEnGloss(word, rawDef, { byKey, primary, frqMap }) {
|
||||
const w = word.toLowerCase();
|
||||
for (const pos of ['noun', 'verb', 'adj', 'adv']) {
|
||||
const offset = primary[pos]?.get(w);
|
||||
if (!offset) continue;
|
||||
const key = `${pos}:${offset}`;
|
||||
const syn = simplestWnSynonym(w, key, byKey, frqMap);
|
||||
if (syn) return syn;
|
||||
const hyper = wnHypernym(key, byKey, w);
|
||||
if (hyper) return hyper;
|
||||
}
|
||||
return shortDefGloss(rawDef);
|
||||
}
|
||||
|
||||
// Minimal CSV line parser (ECDICT quotes fields containing commas/newlines).
|
||||
@@ -93,58 +242,220 @@ export function parseExchange(exchange, word) {
|
||||
return [...forms];
|
||||
}
|
||||
|
||||
// Build the EN→中文 index from ECDICT CSV *text* (so it's unit-testable).
|
||||
export function buildEnZh(csvText, topN) {
|
||||
// Over-generate base forms for a transparent English derivation (lazily→lazy,
|
||||
// thickly→thick, kindness→kind, harmless→harm). Mirrors baseFormCandidates in
|
||||
// src/services/wordlens/gloss.ts. The caller validates each candidate against the
|
||||
// entry table + the word's definition, so wrong guesses are harmless.
|
||||
export function enBaseFormCandidates(word) {
|
||||
const w = word.toLowerCase();
|
||||
const out = new Set();
|
||||
const add = (x) => {
|
||||
if (x.length >= 2) out.add(x);
|
||||
};
|
||||
if (w.endsWith('ily') && w.length >= 5) add(w.slice(0, -3) + 'y'); // lazily → lazy
|
||||
if (w.endsWith('ly') && w.length >= 4) {
|
||||
add(w.slice(0, -2)); // shyly → shy, thickly → thick
|
||||
add(w.slice(0, -2) + 'e'); // nicely → nice
|
||||
}
|
||||
if (w.endsWith('ful') && w.length >= 5) {
|
||||
add(w.slice(0, -3)); // sorrowful → sorrow
|
||||
add(w.slice(0, -4) + 'y'); // beautiful → beauty
|
||||
}
|
||||
if (w.endsWith('wards') && w.length >= 6) {
|
||||
add(w.slice(0, -1)); // downwards → downward
|
||||
add(w.slice(0, -5)); // downwards → down
|
||||
} else if (w.endsWith('ward') && w.length >= 5) {
|
||||
add(w.slice(0, -4)); // inward → in
|
||||
}
|
||||
if (w.endsWith('ness') && w.length >= 5) {
|
||||
add(w.slice(0, -4)); // kindness → kind
|
||||
add(w.slice(0, -5) + 'y'); // happiness → happy
|
||||
}
|
||||
if (w.endsWith('less') && w.length >= 5) add(w.slice(0, -4)); // harmless → harm
|
||||
if ((w.endsWith('able') || w.endsWith('ible')) && w.length >= 6) {
|
||||
add(w.slice(0, -4)); // comfortable → comfort, sufferable → suffer
|
||||
add(w.slice(0, -4) + 'e'); // sensible → sense, lovable → love
|
||||
}
|
||||
// Negative/reversive prefixes (unhappy → happy, insufferable → suffer once -able is
|
||||
// stripped above): peel the prefix off the word and off each suffix candidate.
|
||||
for (const stem of [w, ...out]) {
|
||||
for (const p of ['un', 'in', 'im', 'ir', 'il']) {
|
||||
if (stem.startsWith(p) && stem.length - p.length >= 3) add(stem.slice(p.length));
|
||||
}
|
||||
}
|
||||
out.delete(w);
|
||||
return [...out];
|
||||
}
|
||||
|
||||
// Does an entry's definition mention `base` as a whole word? Confirms a derivation
|
||||
// is transparent (thickly's def names "thick") and rejects drift (hardly's def
|
||||
// never says "hard") + coincidental string matches (ally's def never says "ale").
|
||||
function defMentionsWord(def, base) {
|
||||
if (!def || base.length < 3) return false;
|
||||
return new RegExp(`\\b${base.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i').test(def);
|
||||
}
|
||||
|
||||
// Do a derived word and its candidate base share a content (Han) character in their
|
||||
// Chinese translations? Confirms a transparent derivation whose English def doesn't
|
||||
// name the base — meaning-shifting derivations like the negative -able/prefix family
|
||||
// (insufferable/忍受 ⇐ suffer/忍受), while rejecting coincidental stems (capable ⇏ cap,
|
||||
// 能力 vs 帽子). Particles carry no meaning and are stripped before the test.
|
||||
const HAN_PARTICLES = /[的地得了着之吗呢啊，,；;、。.\s/]/g;
|
||||
const HAN_CHAR = /\p{Script=Han}/u;
|
||||
function transShareMeaning(a, b) {
|
||||
const an = (a || '').replace(HAN_PARTICLES, '');
|
||||
const bn = (b || '').replace(HAN_PARTICLES, '');
|
||||
if (!an || !bn) return false;
|
||||
const setB = new Set([...bn].filter((ch) => HAN_CHAR.test(ch)));
|
||||
return [...an].some((ch) => HAN_CHAR.test(ch) && setB.has(ch));
|
||||
}
|
||||
|
||||
// Clean up a candidate form→lemma inflection map against the entry table, in place:
|
||||
// 1. Prune a wrong link where the FORM is a more-common standalone entry than its
|
||||
// lemma — a primary word coincidentally matching an inflection pattern, not an
|
||||
// inflection (the noun "number", rank 300, wrongly claimed as numb's comparative).
|
||||
// 2. Resolve chains to the terminal lemma ("paintings"→"painting"→"paint" becomes
|
||||
// "paintings"→"paint"), so a dropped middle form never leaves a dangling pointer.
|
||||
// 3. Drop a form-entry whose terminal lemma is a present entry (it resolves to the
|
||||
// lemma at lookup); drop the mapping entirely when the terminal isn't an entry.
|
||||
// Postcondition: every inflection's lemma is an entry, and no inflected form is also
|
||||
// an entry.
|
||||
export function finalizeInflections(entries, inflections) {
|
||||
for (const [form, lemma] of Object.entries(inflections)) {
|
||||
const fe = entries[form];
|
||||
const le = entries[lemma];
|
||||
if (fe && le && fe.r < le.r) delete inflections[form]; // form outranks its "lemma"
|
||||
}
|
||||
for (const form of Object.keys(inflections)) {
|
||||
let lemma = inflections[form];
|
||||
const seen = new Set([form]);
|
||||
while (inflections[lemma] && !seen.has(lemma)) {
|
||||
seen.add(lemma);
|
||||
lemma = inflections[lemma];
|
||||
}
|
||||
// Stopped with `lemma` still a key ⇒ a cycle (ambiguous mutual inflections,
|
||||
// e.g. "axes"↔"axis"); don't lemmatize this form — leave it a standalone entry.
|
||||
if (inflections[lemma]) delete inflections[form];
|
||||
else inflections[form] = lemma;
|
||||
}
|
||||
for (const [form, lemma] of Object.entries(inflections)) {
|
||||
if (!entries[lemma]) delete inflections[form]; // terminal not an entry → no dangling
|
||||
else if (entries[form]) delete entries[form]; // resolves to its lemma at lookup
|
||||
}
|
||||
}
|
||||
|
||||
// Build an English-source index from ECDICT CSV *text* (so it's unit-testable).
|
||||
// Shared by buildEnZh (gloss from the Chinese `translation`) and buildEnEn (gloss
|
||||
// from the English `definition`). `glossColumn` is the ECDICT column to read and
|
||||
// `makeGloss` turns that raw field into the trimmed hint; everything else — frq
|
||||
// rank, exchange-based inflection map, drop-inflected-when-lemma-present, and the
|
||||
// derivational lemmatization below — is shared. Lemmatizing both inflected AND
|
||||
// transparently-derived forms to their base, so the difficulty check runs against
|
||||
// the LEMMA's frequency rank, is a workflow rule for every English-source pair.
|
||||
function buildEnPack(csvText, topN, { target, glossColumn, makeGloss }) {
|
||||
const rows = csvText.split('\n');
|
||||
const header = parseCsvLine(rows[0]) ?? [];
|
||||
const col = (name) => header.indexOf(name);
|
||||
const iWord = col('word'),
|
||||
iTr = col('translation'),
|
||||
iGloss = col(glossColumn),
|
||||
iDef = col('definition'),
|
||||
iTrans = col('translation'),
|
||||
iFrq = col('frq'),
|
||||
iEx = col('exchange');
|
||||
const entries = {};
|
||||
const inflections = {};
|
||||
const parsed = [];
|
||||
// Pass 1: parse rows + build a word→frq map (a gloss builder may pick the simplest
|
||||
// synonym by frequency, e.g. en-en's WordNet hybrid).
|
||||
const raw = [];
|
||||
const frqMap = new Map();
|
||||
for (let i = 1; i < rows.length; i++) {
|
||||
const c = parseCsvLine(rows[i]);
|
||||
if (!c || !c[iWord]) continue;
|
||||
const word = c[iWord];
|
||||
const frq = parseInt(c[iFrq] || '0', 10) || Number.MAX_SAFE_INTEGER;
|
||||
const g = shortGloss((c[iTr] || '').replace(/\\n/g, '；'));
|
||||
const key = word.toLowerCase();
|
||||
const prev = frqMap.get(key);
|
||||
if (prev == null || frq < prev) frqMap.set(key, frq);
|
||||
raw.push({
|
||||
word,
|
||||
frq,
|
||||
gloss: c[iGloss] || '',
|
||||
def: c[iDef] || '',
|
||||
trans: c[iTrans] || '',
|
||||
exchange: c[iEx] || '',
|
||||
});
|
||||
}
|
||||
// Pass 2: build the final gloss for each row.
|
||||
const entries = {};
|
||||
const inflections = {};
|
||||
const defByWord = new Map(); // lowercased word -> raw English definition (for lemmatization)
|
||||
const transByWord = new Map(); // lowercased word -> Chinese translation (for lemmatization)
|
||||
const parsed = [];
|
||||
for (const r of raw) {
|
||||
const g = makeGloss(r.gloss, { word: r.word, frqMap });
|
||||
if (!g) continue;
|
||||
parsed.push({ word: c[iWord], frq, g, exchange: c[iEx] || '' });
|
||||
parsed.push({ word: r.word, frq: r.frq, g, exchange: r.exchange, def: r.def, trans: r.trans });
|
||||
}
|
||||
parsed.sort((a, b) => a.frq - b.frq);
|
||||
for (const e of parsed.slice(0, topN)) {
|
||||
entries[e.word.toLowerCase()] = { r: e.frq, g: e.g };
|
||||
const key = e.word.toLowerCase();
|
||||
entries[key] = { r: e.frq, g: e.g };
|
||||
defByWord.set(key, e.def);
|
||||
transByWord.set(key, e.trans);
|
||||
// exchange: "p:past/i:ing/3:thirdperson/..." — map every form back to the lemma.
|
||||
// `parsed` is sorted most-common-first, so the FIRST lemma to claim a surface
|
||||
// form wins: ambiguous forms resolve to the most frequent lemma (e.g. "does"
|
||||
// -> "do", not "doe"; "putting" -> "put", not "putt").
|
||||
for (const form of parseExchange(e.exchange, e.word)) {
|
||||
const key = form.toLowerCase();
|
||||
if (!inflections[key]) inflections[key] = e.word.toLowerCase();
|
||||
const f = form.toLowerCase();
|
||||
if (!inflections[f]) inflections[f] = key;
|
||||
}
|
||||
}
|
||||
// Drop standalone inflected-form entries (e.g. "kept", "children") when their
|
||||
// lemma is also present: at lookup the surface form resolves to the lemma via
|
||||
// `inflections`, so it inherits the lemma's difficulty rank + real gloss instead
|
||||
// of being judged on its own + showing a cross-reference ("keep的过去式…").
|
||||
for (const [form, lemma] of Object.entries(inflections)) {
|
||||
if (entries[form] && entries[lemma]) delete entries[form];
|
||||
// Candidate transparent DERIVATIONS (thickly⇐thick, kindness⇐kind, insufferable⇐
|
||||
// suffer): a base that is also an entry AND either named by this word's English
|
||||
// definition OR sharing a Han character in its Chinese translation (for meaning-
|
||||
// shifting families like negative -able/prefix, whose def never names the base).
|
||||
// Both checks reject drift (hardly⇏hard) and coincidental stems (ally⇏ale, capable⇏
|
||||
// cap). Recorded alongside the exchange inflections; finalizeInflections does the drop.
|
||||
for (const word of Object.keys(entries)) {
|
||||
if (inflections[word]) continue; // an exchange inflection already claimed it
|
||||
for (const base of enBaseFormCandidates(word)) {
|
||||
if (
|
||||
entries[base] &&
|
||||
(defMentionsWord(defByWord.get(word), base) ||
|
||||
transShareMeaning(transByWord.get(word), transByWord.get(base)))
|
||||
) {
|
||||
inflections[word] = base;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
finalizeInflections(entries, inflections);
|
||||
return {
|
||||
meta: {
|
||||
source: 'en',
|
||||
target: 'zh',
|
||||
metric: 'frq',
|
||||
version: 1,
|
||||
count: Object.keys(entries).length,
|
||||
},
|
||||
meta: { source: 'en', target, metric: 'frq', version: 1, count: Object.keys(entries).length },
|
||||
entries,
|
||||
inflections,
|
||||
};
|
||||
}
|
||||
|
||||
// EN→中文: gloss from ECDICT's `translation` column (senses split by literal "\n").
|
||||
export const buildEnZh = (csvText, topN) =>
|
||||
buildEnPack(csvText, topN, {
|
||||
target: 'zh',
|
||||
glossColumn: 'translation',
|
||||
makeGloss: (s) => shortGloss(s.replace(/\\n/g, '；')),
|
||||
});
|
||||
|
||||
// EN→EN (monolingual): short hint = a simpler synonym, else a category (hypernym),
|
||||
// else a trimmed definition (see resolveEnEnGloss). `wordnet` is { byKey, primary }
|
||||
// from parseWordNetData/parseWordNetIndex. ECDICT supplies the frequency rank +
|
||||
// inflections, so difficulty is identical to en-zh.
|
||||
export const buildEnEn = (csvText, wordnet, topN) =>
|
||||
buildEnPack(csvText, topN, {
|
||||
target: 'en',
|
||||
glossColumn: 'definition',
|
||||
makeGloss: (rawDef, { word, frqMap }) =>
|
||||
resolveEnEnGloss(word, rawDef, { byKey: wordnet.byKey, primary: wordnet.primary, frqMap }),
|
||||
});
|
||||
|
||||
// Parse one CC-CEDICT line: `傳統 传统 [chuan2 tong3] /tradition/traditional/`.
|
||||
// Returns { simp, senses: [...] } or null for comments / malformed lines.
|
||||
export function parseCedictLine(line) {
|
||||
@@ -442,16 +753,13 @@ export function buildPack({ freqList, glossMap, meta, topN = 30000, skipTop = 10
|
||||
entries[word] = { r: rank, g };
|
||||
count++;
|
||||
}
|
||||
// Inflected surface forms resolve to their lemma's entry via `inflections`, so
|
||||
// drop any standalone inflected-form entry whose lemma is present (it then
|
||||
// inherits the lemma's difficulty rank + gloss instead of being glossed itself).
|
||||
// Inflected/derived surface forms resolve to their lemma's entry via `inflections`,
|
||||
// so drop any standalone form-entry whose lemma is present. finalizeInflections
|
||||
// prunes lemmas not present here, resolves chains, and rejects wrong collisions.
|
||||
const inflections = {};
|
||||
if (lemmaMap) {
|
||||
for (const [form, lemma] of lemmaMap) {
|
||||
if (!entries[lemma]) continue;
|
||||
delete entries[form];
|
||||
inflections[form] = lemma;
|
||||
}
|
||||
for (const [form, lemma] of lemmaMap) inflections[form] = lemma;
|
||||
finalizeInflections(entries, inflections);
|
||||
}
|
||||
return {
|
||||
meta: { ...meta, metric: 'frequency', version: 1, count: Object.keys(entries).length },
|
||||
@@ -497,6 +805,23 @@ function writeManifest() {
|
||||
return manifest;
|
||||
}
|
||||
|
||||
// Every English-SOURCE pack must lemmatize via the en-en inflection table — the
|
||||
// canonical English inflection + derivation map. en-en is the English-native pack,
|
||||
// and its definition-present entry set is exactly what the derivational lemmatizer
|
||||
// validates against, so it's the proper source of truth. Enforced: en-en must exist
|
||||
// first, or this throws instead of silently shipping an un-lemmatized en-X pack.
|
||||
// (en-en / en-zh build the table directly via buildEnPack.)
|
||||
function requireEnLemmaMap() {
|
||||
const enEnPath = resolve(OUT_DIR, 'en-en.json');
|
||||
if (!existsSync(enEnPath)) {
|
||||
throw new Error(
|
||||
'English-source builds require data/wordlens/en-en.json for lemmatization ' +
|
||||
'(inflections + derivations). Build en-en first.',
|
||||
);
|
||||
}
|
||||
return inflectionMapFromPack(readFileSync(enEnPath, 'utf8'));
|
||||
}
|
||||
|
||||
// CLI entry point — skipped when imported by tests.
|
||||
async function main() {
|
||||
const [pair, ...rest] = process.argv.slice(2);
|
||||
@@ -523,7 +848,9 @@ async function main() {
|
||||
attribution:
|
||||
'Glosses: Wiktionary (CC-BY-SA-4.0) via kaikki.org. Frequency: hermitdave/FrequencyWords from OpenSubtitles/OPUS (CC-BY-SA-4.0).',
|
||||
};
|
||||
const data = buildPack({ freqList, glossMap, meta, topN: Number(topN) || TOP_DEFAULT });
|
||||
// English-source packs must lemmatize via the en-zh table (enforced).
|
||||
const lemmaMap = src === 'en' ? requireEnLemmaMap() : null;
|
||||
const data = buildPack({ freqList, glossMap, meta, topN: Number(topN) || TOP_DEFAULT, lemmaMap });
|
||||
const file = `${src}-${tgt}.json`;
|
||||
writeFileSync(resolve(OUT_DIR, file), JSON.stringify(data));
|
||||
console.log(`${file}: ${data.meta.count} entries`);
|
||||
@@ -550,13 +877,13 @@ async function main() {
|
||||
const freqList = parseFrequencyWords(readFileSync(freqPath, 'utf8'));
|
||||
const glossMap = extractWikDict(rows);
|
||||
// Lemmatize the SOURCE language so inflected words are glossed via their lemma.
|
||||
// English source reuses the en-zh pack's inflection table ("kept"->"keep"); a
|
||||
// non-English source uses an optional michmech lemmatization list ("perros"->
|
||||
// "perro"). No-op if neither is available.
|
||||
// English source MUST reuse the en-en inflection + derivation table ("kept"->
|
||||
// "keep", "thickly"->"thick") — enforced via requireEnLemmaMap (throws if en-en
|
||||
// is missing). A non-English source uses an optional michmech lemmatization list
|
||||
// ("perros"->"perro").
|
||||
let lemmaMap = null;
|
||||
const enZhPath = resolve(OUT_DIR, 'en-zh.json');
|
||||
if (src === 'en' && existsSync(enZhPath)) {
|
||||
lemmaMap = inflectionMapFromPack(readFileSync(enZhPath, 'utf8'));
|
||||
if (src === 'en') {
|
||||
lemmaMap = requireEnLemmaMap();
|
||||
} else if (tgt === 'en' && lemmaFile && existsSync(lemmaFile)) {
|
||||
lemmaMap = parseLemmatizationList(readFileSync(lemmaFile, 'utf8'));
|
||||
}
|
||||
@@ -581,6 +908,22 @@ async function main() {
|
||||
`en-zh.json: ${data.meta.count} entries, ${Object.keys(data.inflections).length} inflections`,
|
||||
);
|
||||
writeManifest();
|
||||
} else if (pair === 'en-en') {
|
||||
const [csv, wnDir, topN] = rest;
|
||||
if (!csv || !wnDir)
|
||||
throw new Error('usage: build-wordlens-data.mjs en-en <ecdict.csv> <wordnet-dict-dir> [topN]');
|
||||
const byKey = new Map();
|
||||
const primary = { noun: new Map(), verb: new Map(), adj: new Map(), adv: new Map() };
|
||||
for (const pos of ['noun', 'verb', 'adj', 'adv']) {
|
||||
parseWordNetData(readFileSync(resolve(wnDir, `data.${pos}`), 'utf8'), pos, byKey);
|
||||
parseWordNetIndex(readFileSync(resolve(wnDir, `index.${pos}`), 'utf8'), primary[pos]);
|
||||
}
|
||||
const data = buildEnEn(readFileSync(csv, 'utf8'), { byKey, primary }, Number(topN) || TOP_DEFAULT);
|
||||
writeFileSync(resolve(OUT_DIR, 'en-en.json'), JSON.stringify(data));
|
||||
console.log(
|
||||
`en-en.json: ${data.meta.count} entries, ${Object.keys(data.inflections).length} inflections`,
|
||||
);
|
||||
writeManifest();
|
||||
} else if (pair === 'zh-en') {
|
||||
const [cedict, hsk, topN] = rest;
|
||||
if (!cedict || !hsk)
|
||||
@@ -598,7 +941,7 @@ async function main() {
|
||||
console.log('manifest.json:', m.packs.length, 'packs');
|
||||
} else {
|
||||
throw new Error(
|
||||
'usage: build-wordlens-data.mjs <en-zh|zh-en|build|build-wikdict|manifest> <sources...> [topN]',
|
||||
'usage: build-wordlens-data.mjs <en-zh|en-en|zh-en|build|build-wikdict|manifest> <sources...> [topN]',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
// Preview sample entries from the generated Word Lens gloss packs, to eyeball gloss
|
||||
// quality per language pair. Read-only; does not modify any pack.
|
||||
//
|
||||
// node scripts/preview-wordlens.mjs # a sample from every pack
|
||||
// node scripts/preview-wordlens.mjs en-en # a larger sample from one pair
|
||||
// node scripts/preview-wordlens.mjs en-en happy commence thickly # specific words
|
||||
//
|
||||
// (pnpm wordlens:preview [pair] [...words])
|
||||
import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
||||
import { resolve, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const DATA_DIR = resolve(dirname(fileURLToPath(import.meta.url)), '../data/wordlens');
|
||||
|
||||
// Pick `count` entries spread evenly across the frequency-rank range (commonest
|
||||
// first, rarest last) so a preview shows both easy and hard words. Deterministic.
|
||||
export function sampleEntries(entries, count = 12) {
|
||||
const sorted = Object.entries(entries).sort((a, b) => a[1].r - b[1].r);
|
||||
if (sorted.length <= count) return sorted;
|
||||
const step = (sorted.length - 1) / (count - 1);
|
||||
return Array.from({ length: count }, (_, i) => sorted[Math.round(i * step)]);
|
||||
}
|
||||
|
||||
function previewPack(pair, words, count) {
|
||||
const path = resolve(DATA_DIR, `${pair}.json`);
|
||||
if (!existsSync(path)) {
|
||||
console.log(`\n${pair}: (no pack file)`);
|
||||
return;
|
||||
}
|
||||
const { meta, entries, inflections } = JSON.parse(readFileSync(path, 'utf8'));
|
||||
const n = Object.keys(entries).length;
|
||||
console.log(`\n${pair} — ${meta.source}→${meta.target}, ${n} entries, metric=${meta.metric}`);
|
||||
const rows = words.length
|
||||
? words.map((w) => {
|
||||
const key = w.toLowerCase();
|
||||
const lemma = inflections[key];
|
||||
const entry = entries[key] ?? (lemma ? entries[lemma] : null);
|
||||
return [lemma ? `${key} → ${lemma}` : key, entry];
|
||||
})
|
||||
: sampleEntries(entries, count);
|
||||
for (const [word, entry] of rows) {
|
||||
if (!entry) console.log(` ${'—'.padStart(6)} ${String(word).padEnd(22)} (no entry)`);
|
||||
else console.log(` ${String(entry.r).padStart(6)} ${String(word).padEnd(22)} ${entry.g}`);
|
||||
}
|
||||
if (!words.length) {
|
||||
const infl = Object.entries(inflections)
|
||||
.slice(0, 6)
|
||||
.map(([f, l]) => `${f}→${l}`);
|
||||
if (infl.length) console.log(` inflections: ${infl.join(', ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
function listPairs() {
|
||||
return readdirSync(DATA_DIR)
|
||||
.filter((f) => f.endsWith('.json') && f !== 'manifest.json')
|
||||
.map((f) => f.replace(/\.json$/, ''))
|
||||
.sort();
|
||||
}
|
||||
|
||||
function main() {
|
||||
const [pair, ...words] = process.argv.slice(2);
|
||||
if (pair) previewPack(pair, words, words.length ? words.length : 25);
|
||||
else for (const p of listPairs()) previewPack(p, [], 12);
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) main();
|
||||
@@ -0,0 +1,61 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import type { ViewSettings } from '@/types/book';
|
||||
import type { AppService } from '@/types/system';
|
||||
|
||||
// Mock the pack loader: it's the boundary we assert the gate reaches. Resolving
|
||||
// null lets refreshSectionGlosses bail right after the await (no DOM work).
|
||||
vi.mock('@/services/wordlens/glossPacks', () => ({
|
||||
loadGlossIndex: vi.fn().mockResolvedValue(null),
|
||||
}));
|
||||
|
||||
import { refreshSectionGlosses } from '@/app/reader/utils/wordlensSection';
|
||||
import { loadGlossIndex } from '@/services/wordlens/glossPacks';
|
||||
|
||||
const mockedLoad = vi.mocked(loadGlossIndex);
|
||||
|
||||
const viewSettings = (overrides: Partial<ViewSettings> = {}): ViewSettings =>
|
||||
({
|
||||
wordLensEnabled: true,
|
||||
wordLensLevel: 3,
|
||||
wordLensHintLang: '',
|
||||
...overrides,
|
||||
}) as unknown as ViewSettings;
|
||||
|
||||
const ctx = (overrides: Record<string, unknown> = {}) => ({
|
||||
appService: {} as unknown as AppService,
|
||||
bookLang: 'en',
|
||||
appLang: 'en',
|
||||
allowDownload: false,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
mockedLoad.mockClear();
|
||||
mockedLoad.mockResolvedValue(null);
|
||||
});
|
||||
|
||||
describe('refreshSectionGlosses gating', () => {
|
||||
it('loads an en-en index when book and hint are both English (same-language allowed)', async () => {
|
||||
const doc = document.implementation.createHTMLDocument('t');
|
||||
await refreshSectionGlosses(doc, viewSettings({ wordLensHintLang: 'en' }), ctx());
|
||||
expect(mockedLoad).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
'en',
|
||||
'en',
|
||||
expect.objectContaining({ allowDownload: false }),
|
||||
);
|
||||
});
|
||||
|
||||
it('resolves a same-language hint from the app locale when no hint is set', async () => {
|
||||
const doc = document.implementation.createHTMLDocument('t');
|
||||
// hint falls back to appLang ('en'); source is also 'en' → still allowed.
|
||||
await refreshSectionGlosses(doc, viewSettings(), ctx({ appLang: 'en' }));
|
||||
expect(mockedLoad).toHaveBeenCalledWith(expect.anything(), 'en', 'en', expect.anything());
|
||||
});
|
||||
|
||||
it('does not load when no hint can be resolved (no app locale, no selection)', async () => {
|
||||
const doc = document.implementation.createHTMLDocument('t');
|
||||
await refreshSectionGlosses(doc, viewSettings(), ctx({ appLang: '' }));
|
||||
expect(mockedLoad).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -5,10 +5,16 @@ import {
|
||||
parseCsvLine,
|
||||
parseExchange,
|
||||
shortGloss,
|
||||
shortDefGloss,
|
||||
finalizeInflections as finalizeInflectionsUntyped,
|
||||
parseWordNetData as parseWordNetDataUntyped,
|
||||
parseWordNetIndex as parseWordNetIndexUntyped,
|
||||
resolveEnEnGloss as resolveEnEnGlossUntyped,
|
||||
sha256Hex,
|
||||
packEntry,
|
||||
buildManifest,
|
||||
buildEnZh as buildEnZhUntyped,
|
||||
buildEnEn as buildEnEnUntyped,
|
||||
buildZhEn as buildZhEnUntyped,
|
||||
parseFrequencyWords as parseFrequencyWordsUntyped,
|
||||
extractXToEn as extractXToEnUntyped,
|
||||
@@ -23,6 +29,45 @@ import type { GlossIndexData } from '@/services/wordlens/types';
|
||||
// The .mjs script has no type annotations; pin the builders' returns to the
|
||||
// real GlossIndexData shape so the assertions are type-checked.
|
||||
const buildEnZh = buildEnZhUntyped as (csvText: string, topN: number) => GlossIndexData;
|
||||
|
||||
type WnSynset = { members: string[]; hyper: string | null };
|
||||
type WnPrimary = {
|
||||
noun: Map<string, string>;
|
||||
verb: Map<string, string>;
|
||||
adj: Map<string, string>;
|
||||
adv: Map<string, string>;
|
||||
};
|
||||
interface WordNet {
|
||||
byKey: Map<string, WnSynset>;
|
||||
primary: WnPrimary;
|
||||
}
|
||||
const emptyWordNet = (): WordNet => ({
|
||||
byKey: new Map(),
|
||||
primary: { noun: new Map(), verb: new Map(), adj: new Map(), adv: new Map() },
|
||||
});
|
||||
const buildEnEn = buildEnEnUntyped as (
|
||||
csvText: string,
|
||||
wordnet: WordNet,
|
||||
topN: number,
|
||||
) => GlossIndexData;
|
||||
const finalizeInflections = finalizeInflectionsUntyped as (
|
||||
entries: Record<string, { r: number; g: string }>,
|
||||
inflections: Record<string, string>,
|
||||
) => void;
|
||||
const parseWordNetData = parseWordNetDataUntyped as (
|
||||
text: string,
|
||||
pos: string,
|
||||
byKey?: Map<string, WnSynset>,
|
||||
) => Map<string, WnSynset>;
|
||||
const parseWordNetIndex = parseWordNetIndexUntyped as (
|
||||
text: string,
|
||||
map?: Map<string, string>,
|
||||
) => Map<string, string>;
|
||||
const resolveEnEnGloss = resolveEnEnGlossUntyped as (
|
||||
word: string,
|
||||
rawDef: string,
|
||||
ctx: { byKey: Map<string, WnSynset>; primary: WnPrimary; frqMap: Map<string, number> },
|
||||
) => string;
|
||||
const buildZhEn = buildZhEnUntyped as (
|
||||
cedictText: string,
|
||||
hskJson: unknown,
|
||||
@@ -87,15 +132,14 @@ describe('parseExchange', () => {
|
||||
});
|
||||
|
||||
describe('shortGloss', () => {
|
||||
it('truncates to the first 1-2 senses joined by ；and ≤24 chars', () => {
|
||||
it('keeps the first 1-2 senses joined by ；', () => {
|
||||
const g = shortGloss('to run; to operate; to manage; foo');
|
||||
expect(g).toBe('to run；to operate');
|
||||
expect(g.length).toBeLessThanOrEqual(24);
|
||||
});
|
||||
|
||||
it('caps overly long senses at 24 chars', () => {
|
||||
it('does not length-cap a long sense (the cap is applied at display, not in the pack)', () => {
|
||||
const g = shortGloss('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa');
|
||||
expect(g.length).toBe(24);
|
||||
expect(g).toBe('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'); // 32 chars stored as-is
|
||||
});
|
||||
|
||||
it('strips leading POS tags, bracket annotations, and CL: clauses', () => {
|
||||
@@ -104,6 +148,281 @@ describe('shortGloss', () => {
|
||||
expect(shortGloss('[网络] 隐；晦涩的')).toBe('隐；晦涩的');
|
||||
expect(shortGloss('government/CL:個|个[ge4]')).toBe('government');
|
||||
});
|
||||
|
||||
it('keeps only the first synonym within each sense (","/"、"-separated)', () => {
|
||||
// ";" separates senses; "," within a sense separates near-synonyms → keep first.
|
||||
expect(shortGloss('阻止, 监禁, 拘留；隔离, 拘留, 滞留, 停')).toBe('阻止；隔离');
|
||||
expect(shortGloss('house, home')).toBe('house');
|
||||
});
|
||||
|
||||
it('dedupes when two senses share the same first synonym', () => {
|
||||
expect(shortGloss('天的, 天国的, 天空的；天的, 天空的, 天国')).toBe('天的');
|
||||
});
|
||||
});
|
||||
|
||||
describe('shortDefGloss', () => {
|
||||
it('shows at most two senses when there is no ";" (\\n-separated)', () => {
|
||||
// ECDICT separates sense lines with a literal "\n" (backslash-n).
|
||||
const def = String.raw`to begin\nto start\nto set in motion`;
|
||||
expect(shortDefGloss(def)).toBe('to begin; to start');
|
||||
});
|
||||
|
||||
it('treats each ";" part as a sense and keeps at most two of them', () => {
|
||||
expect(shortDefGloss('quick; fast')).toBe('quick; fast');
|
||||
expect(shortDefGloss('a; b; c; d; e')).toBe('a; b'); // at most two ";" parts
|
||||
// a ";" within a line is a sense even when the line came from a \n split.
|
||||
expect(shortDefGloss(String.raw`of an obscure nature; puzzling\nhidden`)).toBe(
|
||||
'of an obscure nature; puzzling',
|
||||
);
|
||||
});
|
||||
|
||||
it('stores the full ≤2-sense definition without length-capping (the cap is applied at display)', () => {
|
||||
expect(shortDefGloss('enjoying or showing or marked by joy or pleasure')).toBe(
|
||||
'enjoying or showing or marked by joy or pleasure',
|
||||
);
|
||||
});
|
||||
|
||||
it('strips POS codes — period-terminated (n./adv.) or bare WordNet (v/s)', () => {
|
||||
expect(shortDefGloss('v take the first step')).toBe('take the first step');
|
||||
expect(shortDefGloss('s being present')).toBe('being present');
|
||||
expect(shortDefGloss('adv. in a quick way')).toBe('in a quick way');
|
||||
// a bare leading "a" is the article, not the adjective POS code → kept.
|
||||
expect(shortDefGloss('n. a small house')).toBe('a small house');
|
||||
});
|
||||
|
||||
it('strips bracket annotations and a Webster-style trailing period', () => {
|
||||
expect(shortDefGloss('a happy state [psychology]')).toBe('a happy state');
|
||||
expect(shortDefGloss('A magician.')).toBe('A magician');
|
||||
});
|
||||
|
||||
it('returns empty string for empty / whitespace-only / all-empty senses', () => {
|
||||
expect(shortDefGloss('')).toBe('');
|
||||
expect(shortDefGloss(' ')).toBe('');
|
||||
expect(shortDefGloss(String.raw`\n \n`)).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildEnEn', () => {
|
||||
const header =
|
||||
'word,phonetic,definition,translation,pos,collins,oxford,tag,bnc,frq,exchange,detail,audio';
|
||||
const csv = [
|
||||
header,
|
||||
'Run,/rʌn/,move fast on foot,跑；经营,v,3,,,100,312,p:ran/i:running/3:runs/0:run,,',
|
||||
String.raw`cryptic,/ˈkrɪptɪk/,having a hidden meaning\nmysterious,晦涩的,adj,2,,,500,18000,,,`,
|
||||
].join('\n');
|
||||
|
||||
it('falls back to the full ECDICT definition (≤2 senses) when no WordNet hint applies', () => {
|
||||
const data = buildEnEn(csv, emptyWordNet(), 100);
|
||||
expect(data.meta).toEqual({
|
||||
source: 'en',
|
||||
target: 'en',
|
||||
metric: 'frq',
|
||||
version: 1,
|
||||
count: 2,
|
||||
});
|
||||
expect(data.entries['run']).toEqual({ r: 312, g: 'move fast on foot' });
|
||||
// stored uncapped; the display-time cap (cleanGloss) shortens it when rendered.
|
||||
expect(data.entries['cryptic']).toEqual({ r: 18000, g: 'having a hidden meaning; mysterious' });
|
||||
// exchange forms map back to the lemma; the lemma itself is not an inflection.
|
||||
expect(data.inflections['ran']).toBe('run');
|
||||
expect(data.inflections['runs']).toBe('run');
|
||||
expect(data.inflections['run']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('prefers a simpler synonym / category from WordNet over the definition', () => {
|
||||
const csv2 = [
|
||||
header,
|
||||
'ship,/ʃ/,a large seagoing vessel,船,n,,,,100,4000,,,',
|
||||
'vessel,/v/,a craft for traveling on water,船,n,,,,100,2500,,,',
|
||||
'prejudice,/p/,an adverse judgment formed beforehand,偏见,n,,,,100,6000,,,',
|
||||
'bias,/b/,a partiality,偏向,n,,,,100,3000,,,',
|
||||
].join('\n');
|
||||
const wn = emptyWordNet();
|
||||
wn.byKey.set('noun:1', { members: ['ship'], hyper: 'noun:2' });
|
||||
wn.byKey.set('noun:2', { members: ['vessel'], hyper: null });
|
||||
wn.byKey.set('noun:3', { members: ['prejudice', 'bias'], hyper: null });
|
||||
wn.primary.noun.set('ship', '1');
|
||||
wn.primary.noun.set('prejudice', '3');
|
||||
const data = buildEnEn(csv2, wn, 100);
|
||||
expect(data.entries['ship']!.g).toBe('vessel'); // no simpler synonym → hypernym
|
||||
expect(data.entries['prejudice']!.g).toBe('bias'); // simpler synonym wins
|
||||
});
|
||||
|
||||
it('skips a row whose definition is empty', () => {
|
||||
const data = buildEnEn([header, 'word,/x/,,无,n,1,,,1,10,,,'].join('\n'), emptyWordNet(), 100);
|
||||
expect(data.entries['word']).toBeUndefined();
|
||||
expect(data.meta.count).toBe(0);
|
||||
});
|
||||
|
||||
it('drops an inflected-form entry when its lemma is present (lemmatize)', () => {
|
||||
const csv2 = [
|
||||
header,
|
||||
'keep,/kiːp/,hold on to,保持,v,5,,,50,120,p:kept/i:keeping/3:keeps,,',
|
||||
'kept,/kept/,past tense of keep,过去式,v,,,,800,2500,,,',
|
||||
].join('\n');
|
||||
const data = buildEnEn(csv2, emptyWordNet(), 100);
|
||||
expect(data.entries['kept']).toBeUndefined();
|
||||
expect(data.entries['keep']).toEqual({ r: 120, g: 'hold on to' });
|
||||
expect(data.inflections['kept']).toBe('keep');
|
||||
expect(data.meta.count).toBe(1);
|
||||
});
|
||||
|
||||
it('lemmatizes a transparent derivation to its base when the definition names it (thickly ⇐ thick)', () => {
|
||||
const csv2 = [
|
||||
header,
|
||||
'thick,/θ/,of great width,厚的,adj,,,,100,1000,,,',
|
||||
'thickly,/θ/,in a thick manner,浓密地,adv,,,,500,15000,,,',
|
||||
].join('\n');
|
||||
const data = buildEnEn(csv2, emptyWordNet(), 100);
|
||||
// "thickly" resolves to "thick" (so the difficulty check uses thick's rank).
|
||||
expect(data.entries['thickly']).toBeUndefined();
|
||||
expect(data.inflections['thickly']).toBe('thick');
|
||||
expect(data.entries['thick']).toEqual({ r: 1000, g: 'of great width' });
|
||||
});
|
||||
|
||||
it('lemmatizes a negative -able derivation via translation overlap (insufferable ⇐ suffer)', () => {
|
||||
// The English def never names "suffer", so def-mention alone can't validate it;
|
||||
// the Chinese translations overlap (忍受) → transparent derivation.
|
||||
const csv2 = [
|
||||
header,
|
||||
'suffer,/s/,undergo pain,"遭受, 经历, 忍受",v,,,,100,1099,d:suffered/p:suffered/i:suffering/3:suffers,,',
|
||||
'insufferable,/i/,used of persons or their behavior,"不可忍受的, 忍耐不住的",adj,,,,500,24043,,,',
|
||||
].join('\n');
|
||||
const data = buildEnEn(csv2, emptyWordNet(), 100);
|
||||
expect(data.entries['insufferable']).toBeUndefined();
|
||||
expect(data.inflections['insufferable']).toBe('suffer');
|
||||
expect(data.entries['suffer']).toBeDefined();
|
||||
});
|
||||
|
||||
it('keeps an -able word whose stem is a coincidence (capable ⇏ cap)', () => {
|
||||
// morphologically capable→cap, but the translations share nothing → not a derivation.
|
||||
const csv2 = [
|
||||
header,
|
||||
'cap,/k/,a head covering,"帽子, 盖",n,,,,100,2000,,,',
|
||||
'capable,/k/,having the ability,"有能力的, 能干的",adj,,,,500,9000,,,',
|
||||
].join('\n');
|
||||
const data = buildEnEn(csv2, emptyWordNet(), 100);
|
||||
expect(data.inflections['capable']).toBeUndefined();
|
||||
expect(data.entries['capable']).toBeDefined();
|
||||
});
|
||||
|
||||
it('keeps a drifted derivation (hardly⇐hard) and a false base match (ally⇐ale)', () => {
|
||||
const csv2 = [
|
||||
header,
|
||||
'hard,/h/,not easy,坚硬的,adj,,,,100,500,,,',
|
||||
'hardly,/h/,almost not,几乎不,adv,,,,500,16000,,,',
|
||||
'ale,/eɪl/,a kind of beer,啤酒,n,,,,100,3000,,,',
|
||||
'ally,/ə/,a friendly nation,盟友,n,,,,500,8000,,,',
|
||||
].join('\n');
|
||||
const data = buildEnEn(csv2, emptyWordNet(), 100);
|
||||
// definition doesn't name the base → not a transparent derivation, kept as-is.
|
||||
expect(data.inflections['hardly']).toBeUndefined();
|
||||
expect(data.entries['hardly']).toBeDefined();
|
||||
expect(data.inflections['ally']).toBeUndefined();
|
||||
expect(data.entries['ally']).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('WordNet hybrid (parse + resolve)', () => {
|
||||
it('parseWordNetData: members + first hypernym pointer, keyed pos:offset', () => {
|
||||
const line = '01382086 00 s 02 huge 0 immense 0 002 @ 01382033 a 0000 & 0 a 0000 | great';
|
||||
const byKey = parseWordNetData(line, 'adj');
|
||||
expect(byKey.get('adj:01382086')).toEqual({
|
||||
members: ['huge', 'immense'],
|
||||
hyper: 'adj:01382033',
|
||||
});
|
||||
});
|
||||
|
||||
it('parseWordNetIndex: lemma → primary (first) synset offset', () => {
|
||||
const idx = ['huge a 2 1 & 2 0 01382086 09573200', 'enormous a 1 1 & 1 0 00109510'].join('\n');
|
||||
const m = parseWordNetIndex(idx);
|
||||
expect(m.get('huge')).toBe('01382086'); // first/primary offset, secondary dropped
|
||||
expect(m.get('enormous')).toBe('00109510');
|
||||
});
|
||||
|
||||
it('resolveEnEnGloss: synonym → hypernym → definition (POS-priority)', () => {
|
||||
const byKey = new Map<string, WnSynset>([
|
||||
['noun:1', { members: ['ship'], hyper: 'noun:2' }],
|
||||
['noun:2', { members: ['vessel'], hyper: null }],
|
||||
['noun:3', { members: ['prejudice', 'bias'], hyper: null }],
|
||||
['noun:4', { members: ['thing'], hyper: 'noun:5' }],
|
||||
['noun:5', { members: ['entity'], hyper: null }], // generic hypernym
|
||||
]);
|
||||
const primary: WnPrimary = {
|
||||
noun: new Map([
|
||||
['ship', '1'],
|
||||
['prejudice', '3'],
|
||||
['thing', '4'],
|
||||
]),
|
||||
verb: new Map(),
|
||||
adj: new Map(),
|
||||
adv: new Map(),
|
||||
};
|
||||
const frqMap = new Map<string, number>([
|
||||
['ship', 4000],
|
||||
['vessel', 2500],
|
||||
['prejudice', 6000],
|
||||
['bias', 3000],
|
||||
['thing', 50],
|
||||
]);
|
||||
const ctx = { byKey, primary, frqMap };
|
||||
expect(resolveEnEnGloss('prejudice', 'an adverse judgment', ctx)).toBe('bias'); // simpler synonym
|
||||
expect(resolveEnEnGloss('ship', 'a large vessel', ctx)).toBe('vessel'); // hypernym (no synonym)
|
||||
expect(resolveEnEnGloss('thing', 'a generic item', ctx)).toBe('a generic item'); // generic hyper → def
|
||||
expect(resolveEnEnGloss('zzz', 'made up word', ctx)).toBe('made up word'); // no WordNet entry → def
|
||||
});
|
||||
});
|
||||
|
||||
describe('finalizeInflections', () => {
|
||||
it('resolves an inflection chain to the terminal lemma (paintings→painting→paint)', () => {
|
||||
const entries: Record<string, { r: number; g: string }> = {
|
||||
paint: { r: 500, g: 'x' },
|
||||
painting: { r: 800, g: 'y' },
|
||||
paintings: { r: 1200, g: 'z' },
|
||||
};
|
||||
const inflections: Record<string, string> = { painting: 'paint', paintings: 'painting' };
|
||||
finalizeInflections(entries, inflections);
|
||||
expect(inflections['paintings']).toBe('paint'); // chain resolved to terminal
|
||||
expect(inflections['painting']).toBe('paint');
|
||||
expect(entries['painting']).toBeUndefined(); // both dropped (paint is more common)
|
||||
expect(entries['paintings']).toBeUndefined();
|
||||
expect(entries['paint']).toBeDefined();
|
||||
});
|
||||
|
||||
it('prunes a wrong collision where the form outranks its lemma (number⇏numb)', () => {
|
||||
const entries: Record<string, { r: number; g: string }> = {
|
||||
number: { r: 300, g: 'a count' },
|
||||
numb: { r: 9000, g: 'without feeling' },
|
||||
numbers: { r: 1500, g: 'counts' },
|
||||
};
|
||||
const inflections: Record<string, string> = { number: 'numb', numbers: 'number' };
|
||||
finalizeInflections(entries, inflections);
|
||||
expect(inflections['number']).toBeUndefined(); // wrong link pruned
|
||||
expect(entries['number']).toBeDefined(); // the common noun is kept as an entry
|
||||
expect(inflections['numbers']).toBe('number'); // valid inflection of the kept entry
|
||||
expect(entries['numbers']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('drops a mapping whose terminal lemma is not an entry (no dangling)', () => {
|
||||
const entries: Record<string, { r: number; g: string }> = { run: { r: 100, g: 'go fast' } };
|
||||
const inflections: Record<string, string> = { jogging: 'jog' }; // jog is not an entry
|
||||
finalizeInflections(entries, inflections);
|
||||
expect(inflections['jogging']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('breaks an inflection cycle instead of self-looping (axes↔axis)', () => {
|
||||
const entries: Record<string, { r: number; g: string }> = {
|
||||
axis: { r: 4000, g: 'a line' },
|
||||
axes: { r: 5000, g: 'plural' },
|
||||
};
|
||||
const inflections: Record<string, string> = { axes: 'axis', axis: 'axes' }; // mutual
|
||||
finalizeInflections(entries, inflections);
|
||||
// No self-loop, no dangling: every surviving inflection points at a present entry.
|
||||
for (const [form, lemma] of Object.entries(inflections)) {
|
||||
expect(form).not.toBe(lemma);
|
||||
expect(entries[lemma]).toBeDefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildEnZh', () => {
|
||||
@@ -142,6 +461,18 @@ describe('buildEnZh', () => {
|
||||
expect(data.meta.count).toBe(1);
|
||||
});
|
||||
|
||||
it('lemmatizes a derivation via the English definition column even though the gloss is Chinese', () => {
|
||||
const csv2 = [
|
||||
header,
|
||||
'thick,/θ/,of great width,厚的,adj,,,,100,1000,,,',
|
||||
'thickly,/θ/,in a thick manner,浓密地,adv,,,,500,15000,,,',
|
||||
].join('\n');
|
||||
const data = buildEnZh(csv2, 100);
|
||||
expect(data.entries['thickly']).toBeUndefined();
|
||||
expect(data.inflections['thickly']).toBe('thick');
|
||||
expect(data.entries['thick']).toEqual({ r: 1000, g: '厚的' }); // gloss still from translation
|
||||
});
|
||||
|
||||
it('resolves an ambiguous inflected form to the most frequent lemma (first-wins)', () => {
|
||||
// Both "do" (common) and "doe" (rare) claim the form "does"; the more frequent
|
||||
// lemma must win so "does" -> "do", not "doe".
|
||||
@@ -410,6 +741,20 @@ describe('buildPack', () => {
|
||||
expect(data.entries['perros']).toBeUndefined();
|
||||
expect(data.inflections['perros']).toBe('perro');
|
||||
});
|
||||
|
||||
it('lemmatizes an English DERIVATION in an en-X pack via the reused en-zh table (thickly⇐thick)', () => {
|
||||
// en-X (e.g. en-de) builds pass the en-zh inflection table as lemmaMap, which now
|
||||
// carries derivations (thickly→thick) — so the rule holds for every en-source pair.
|
||||
const freqList: FreqEntry[] = [
|
||||
{ word: 'thick', rank: 1 },
|
||||
{ word: 'thickly', rank: 2 },
|
||||
];
|
||||
const glossMap = new Map<string, string[]>([['thick', ['breit']]]); // en→de gloss
|
||||
const lemmaMap = new Map<string, string>([['thickly', 'thick']]); // from en-zh.json
|
||||
const data = buildPack({ freqList, glossMap, meta, topN: 30000, skipTop: 0, lemmaMap });
|
||||
expect(data.entries['thickly']).toBeUndefined();
|
||||
expect(data.inflections['thickly']).toBe('thick');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractWikDict', () => {
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { sampleEntries as sampleEntriesUntyped } from '../../../scripts/preview-wordlens.mjs';
|
||||
|
||||
type Entry = { r: number; g: string };
|
||||
const sampleEntries = sampleEntriesUntyped as (
|
||||
entries: Record<string, Entry>,
|
||||
count?: number,
|
||||
) => [string, Entry][];
|
||||
|
||||
describe('sampleEntries', () => {
|
||||
it('returns all entries rank-sorted (ascending) when count >= size', () => {
|
||||
const e = { b: { r: 20, g: 'B' }, a: { r: 10, g: 'A' } };
|
||||
expect(sampleEntries(e, 5).map(([w]) => w)).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
it('spreads samples across the rank range — commonest first, rarest last', () => {
|
||||
const e: Record<string, Entry> = {};
|
||||
for (let i = 1; i <= 100; i++) e[`w${i}`] = { r: i, g: `g${i}` };
|
||||
const s = sampleEntries(e, 5);
|
||||
expect(s.length).toBe(5);
|
||||
expect(s[0]![1].r).toBe(1); // most common
|
||||
expect(s[4]![1].r).toBe(100); // rarest
|
||||
const ranks = s.map(([, v]) => v.r);
|
||||
expect([...ranks].sort((a, b) => a - b)).toEqual(ranks); // non-decreasing
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { resolve, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import type { GlossIndexData } from '@/services/wordlens/types';
|
||||
|
||||
// Validates the lemmatization in the COMMITTED Word Lens pack data (not just the
|
||||
// build functions). Every English-source pack runs through build-time lemmatization
|
||||
// — buildEnPack for en-en/en-zh, buildPack reusing en-en's table for en-X — so the
|
||||
// shipped JSON must satisfy the inflection invariants below. Regenerate with
|
||||
// `node scripts/build-wordlens-data.mjs ...` if these fail after a data refresh.
|
||||
|
||||
const DATA_DIR = resolve(dirname(fileURLToPath(import.meta.url)), '../../../data/wordlens');
|
||||
const load = (pair: string): GlossIndexData =>
|
||||
JSON.parse(readFileSync(resolve(DATA_DIR, `${pair}.json`), 'utf8')) as GlossIndexData;
|
||||
|
||||
const EN_SOURCE_PAIRS = ['en-en', 'en-zh', 'en-de', 'en-es', 'en-fr', 'en-pt', 'en-ru'] as const;
|
||||
|
||||
describe('Word Lens pack data — lemmatization invariants', () => {
|
||||
for (const pair of EN_SOURCE_PAIRS) {
|
||||
describe(pair, () => {
|
||||
const { entries, inflections } = load(pair);
|
||||
|
||||
it('resolves every inflection to a present lemma entry (no dangling pointer)', () => {
|
||||
const dangling = Object.entries(inflections).filter(([, lemma]) => !entries[lemma]);
|
||||
expect(dangling).toEqual([]);
|
||||
});
|
||||
|
||||
it('never keeps an inflected form as a standalone entry (it resolves to its lemma)', () => {
|
||||
const leaked = Object.keys(inflections).filter((form) => entries[form]);
|
||||
expect(leaked).toEqual([]);
|
||||
});
|
||||
|
||||
it('has no inflection chains (every lemma is terminal)', () => {
|
||||
const chained = Object.keys(inflections).filter((form) => inflections[inflections[form]!]);
|
||||
expect(chained).toEqual([]);
|
||||
});
|
||||
|
||||
it('lemmatizes the transparent derivation thickly → thick (gated by lemma rank)', () => {
|
||||
expect(inflections['thickly']).toBe('thick');
|
||||
expect(entries['thickly']).toBeUndefined();
|
||||
expect(entries['thick']).toBeDefined();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
it('keeps the common noun "number" (not dropped as the comparative of "numb")', () => {
|
||||
for (const pair of ['en-en', 'en-zh'] as const) {
|
||||
const { entries, inflections } = load(pair);
|
||||
expect(entries['number']).toBeDefined();
|
||||
expect(inflections['number']).toBeUndefined(); // not treated as an inflection
|
||||
expect(inflections['numbers']).toBe('number'); // its plural resolves to it
|
||||
}
|
||||
});
|
||||
|
||||
it('lemmatizes the negative -able derivation insufferable → suffer (translation overlap)', () => {
|
||||
// Its English def never names "suffer" — validated by the shared Chinese 忍受.
|
||||
for (const pair of ['en-en', 'en-zh'] as const) {
|
||||
const { entries, inflections } = load(pair);
|
||||
expect(inflections['insufferable']).toBe('suffer');
|
||||
expect(entries['insufferable']).toBeUndefined();
|
||||
expect(entries['suffer']).toBeDefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -6,14 +6,17 @@ describe('cleanGloss', () => {
|
||||
expect(cleanGloss('interj. 呃哼')).toBe('呃哼');
|
||||
});
|
||||
|
||||
it('strips short leading POS tags', () => {
|
||||
it('strips short leading POS tags (per sense)', () => {
|
||||
expect(cleanGloss('a. 神秘的')).toBe('神秘的');
|
||||
expect(cleanGloss('vt. 做；vi. 看')).toBe('做');
|
||||
expect(cleanGloss('vt. 做；vi. 看')).toBe('做；看'); // two senses, POS stripped from each
|
||||
});
|
||||
|
||||
it('keeps only the first sense (comma- or semicolon-separated)', () => {
|
||||
expect(cleanGloss('内心的, 向内的, 本来的；向内的')).toBe('内心的');
|
||||
expect(cleanGloss('to run, to operate, to manage')).toBe('to run');
|
||||
it('keeps the first synonym of each of the first two senses', () => {
|
||||
// ";" separates senses; "," / "、" separate near-synonyms within a sense.
|
||||
expect(cleanGloss('内心的, 向内的, 本来的；向内的')).toBe('内心的；向内的');
|
||||
expect(cleanGloss('阻止, 监禁, 拘留；隔离, 拘留, 滞留, 停')).toBe('阻止；隔离');
|
||||
expect(cleanGloss('to run, to operate, to manage')).toBe('to run'); // single sense → first synonym
|
||||
expect(cleanGloss('天的, 天国的；天的, 天空的')).toBe('天的'); // dedupe shared first synonym
|
||||
});
|
||||
|
||||
it('leaves a single short sense unchanged, including a trailing 的', () => {
|
||||
@@ -24,6 +27,27 @@ describe('cleanGloss', () => {
|
||||
it('returns empty string for empty input', () => {
|
||||
expect(cleanGloss('')).toBe('');
|
||||
});
|
||||
|
||||
it('preserves the build-formatted gloss for monolingual (en-en) packs', () => {
|
||||
// en-en glosses keep their full shape (≤2 senses joined by "; "); monolingual
|
||||
// mode must NOT split on ";" or drop a synonym — only normalize whitespace.
|
||||
expect(cleanGloss('to begin; to start', true)).toBe('to begin; to start');
|
||||
// non-monolingual reprocesses into ≤2 senses joined by "；" (fullwidth)
|
||||
expect(cleanGloss('to begin; to start')).toBe('to begin；to start');
|
||||
});
|
||||
|
||||
it('applies the display length cap (the packs store the full hint)', () => {
|
||||
// monolingual: caps with an ellipsis instead of splitting/reprocessing.
|
||||
expect(cleanGloss('having a hidden meaning; mysterious', true)).toBe(
|
||||
'having a hidden meaning…',
|
||||
);
|
||||
// a hint already within the cap is unchanged.
|
||||
expect(cleanGloss('having a hidden meaning…', true)).toBe('having a hidden meaning…');
|
||||
// cross-lingual: after sense/synonym reduction the (single, long) result is also capped.
|
||||
const long = '啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊啊'; // 26 chars, no separators
|
||||
expect(cleanGloss(long).length).toBe(24);
|
||||
expect(cleanGloss(long).endsWith('…')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('baseFormCandidates', () => {
|
||||
@@ -35,6 +59,13 @@ describe('baseFormCandidates', () => {
|
||||
expect(baseFormCandidates('inwards')).toContain('inward');
|
||||
});
|
||||
|
||||
it('over-generates base forms for -able / -ible and negative-prefix derivations', () => {
|
||||
expect(baseFormCandidates('comfortable')).toContain('comfort');
|
||||
expect(baseFormCandidates('sensible')).toContain('sense');
|
||||
expect(baseFormCandidates('unhappy')).toContain('happy');
|
||||
expect(baseFormCandidates('insufferable')).toContain('suffer'); // strip -able then in-
|
||||
});
|
||||
|
||||
it('returns no candidates for a non-derived word', () => {
|
||||
expect(baseFormCandidates('cryptic')).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -83,6 +83,23 @@ describe('planGlosses derivational reduction (English source)', () => {
|
||||
const occ = planGlosses('ahem', derv, { sourceLang: 'en', rankCutoff: cutoff });
|
||||
expect(occ[0]?.gloss).toBe('呃哼');
|
||||
});
|
||||
|
||||
it('suppresses a derivation whose definition mentions the base word (thickly ⇐ thick)', () => {
|
||||
// en-en: glosses are definitions, so the base/derived glosses rarely share a word —
|
||||
// but "thickly"'s definition literally contains "thick", a transparent derivation.
|
||||
const enen: GlossSource = {
|
||||
lookup(word) {
|
||||
const table: Record<string, GlossEntry> = {
|
||||
thickly: { rank: 15000, gloss: 'in a thick manner' },
|
||||
thick: { rank: 1000, gloss: 'of great width' },
|
||||
};
|
||||
return table[word.toLowerCase()] ?? null;
|
||||
},
|
||||
};
|
||||
expect(
|
||||
planGlosses('thickly', enen, { sourceLang: 'en', rankCutoff: cutoff, monolingual: true }),
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('planGlosses against a zh-en fixture', () => {
|
||||
|
||||
@@ -744,6 +744,32 @@ describe('getTranslationStyles branches (via getStyles)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// getRubyStyles branches (Word Lens gloss <rt> size + color)
|
||||
// ---------------------------------------------------------------------------
|
||||
describe('getRubyStyles branches (via getStyles)', () => {
|
||||
const theme = makeThemeCode();
|
||||
const rtBlock = (css: string) => css.match(/ruby\.wl-gloss\s*>\s*rt\s*\{([^}]*)\}/)?.[1] ?? '';
|
||||
|
||||
it('defaults the gloss to 0.5em and muted (opacity 0.7, no color override)', () => {
|
||||
const block = rtBlock(getStyles(makeViewSettings(), theme));
|
||||
expect(block).toMatch(/font-size:\s*0\.5em/);
|
||||
expect(block).toMatch(/opacity:\s*0\.7/);
|
||||
expect(block).not.toContain('color:');
|
||||
});
|
||||
|
||||
it('applies a configured gloss font size', () => {
|
||||
const block = rtBlock(getStyles(makeViewSettings({ wordLensGlossFontSize: 0.8 }), theme));
|
||||
expect(block).toMatch(/font-size:\s*0\.8em/);
|
||||
});
|
||||
|
||||
it('applies a configured gloss color at full opacity', () => {
|
||||
const block = rtBlock(getStyles(makeViewSettings({ wordLensGlossColor: '#ff0000' }), theme));
|
||||
expect(block).toMatch(/color:\s*#ff0000/);
|
||||
expect(block).toMatch(/opacity:\s*1\b/);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// getStyles integration: userStylesheet appended
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -53,7 +53,9 @@ export const refreshSectionGlosses = async (
|
||||
const source = toWordLensSource(ctx.bookLang);
|
||||
if (!source || !canTokenizeSource(source)) return;
|
||||
const hint = (viewSettings.wordLensHintLang || ctx.appLang).toLowerCase().split('-')[0] || '';
|
||||
if (!hint || hint === source) return; // no self-gloss
|
||||
// Same-language packs (e.g. en-en monolingual) are allowed; availability is
|
||||
// decided by the manifest — loadGlossIndex returns null when no pack exists.
|
||||
if (!hint) return;
|
||||
const index = await loadGlossIndex(ctx.appService, source, hint, {
|
||||
onProgress: ctx.onProgress,
|
||||
allowDownload: ctx.allowDownload,
|
||||
@@ -69,6 +71,7 @@ export const refreshSectionGlosses = async (
|
||||
sourceLang: source,
|
||||
rankCutoff: getRankCutoff(source, viewSettings.wordLensLevel),
|
||||
cutZh: source === 'zh' ? cutZh : undefined,
|
||||
monolingual: hint === source, // en-en: gloss is a build-formatted definition
|
||||
});
|
||||
if (occ.length) applyGlosses(doc, model, occ);
|
||||
} catch (err) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import clsx from 'clsx';
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import { useEnv } from '@/context/EnvContext';
|
||||
import { useReaderStore } from '@/store/readerStore';
|
||||
import { useBookDataStore } from '@/store/bookDataStore';
|
||||
@@ -26,8 +26,13 @@ import {
|
||||
type WordLensPack,
|
||||
} from '@/services/wordlens/glossPacks';
|
||||
import SubPageHeader from './SubPageHeader';
|
||||
import ColorInput from './color/ColorInput';
|
||||
import { BoxedList, SettingsRow, SettingsSelect, SettingsSwitchRow } from './primitives';
|
||||
|
||||
// Swatch shown for the "default" (muted, theme-adaptive) gloss color, which has
|
||||
// no fixed hex of its own. Picking any color overrides; "Default" clears back.
|
||||
const DEFAULT_GLOSS_SWATCH = '#808080';
|
||||
|
||||
interface WordLensPanelProps {
|
||||
bookKey: string;
|
||||
onBack: () => void;
|
||||
@@ -50,6 +55,12 @@ const WordLensPanel: React.FC<WordLensPanelProps> = ({ bookKey, onBack }) => {
|
||||
const [wordLensEnabled, setWordLensEnabled] = useState(viewSettings.wordLensEnabled ?? false);
|
||||
const [wordLensLevel, setWordLensLevel] = useState(viewSettings.wordLensLevel ?? 3);
|
||||
const [hintLang, setHintLang] = useState(viewSettings.wordLensHintLang || appLang);
|
||||
const [glossFontSize, setGlossFontSize] = useState(viewSettings.wordLensGlossFontSize ?? 0.5);
|
||||
const [glossColor, setGlossColor] = useState(viewSettings.wordLensGlossColor ?? '');
|
||||
// Track the latest gloss color so ColorInput's onCommit (no args) persists the
|
||||
// value the user landed on after dragging, without re-injecting CSS per tick.
|
||||
const glossColorRef = useRef(glossColor);
|
||||
glossColorRef.current = glossColor;
|
||||
const [autoDownload, setAutoDownload] = useState(
|
||||
settings.globalReadSettings.wordLensAutoDownload ?? true,
|
||||
);
|
||||
@@ -95,7 +106,9 @@ const WordLensPanel: React.FC<WordLensPanelProps> = ({ bookKey, onBack }) => {
|
||||
return;
|
||||
}
|
||||
const hint = baseCode(hintLang) || appLang;
|
||||
if (!hint || hint === bookSource) {
|
||||
// Same-language pairs (e.g. en-en) are allowed; getPackStatus returns null
|
||||
// when the manifest has no pack for the pair, which renders the empty row.
|
||||
if (!hint) {
|
||||
setPackStatus(null);
|
||||
return;
|
||||
}
|
||||
@@ -153,6 +166,33 @@ const WordLensPanel: React.FC<WordLensPanelProps> = ({ bookKey, onBack }) => {
|
||||
setViewSettings(bookKey, { ...viewSettings });
|
||||
};
|
||||
|
||||
// Gloss appearance (font size + color) drives the <rt> CSS via getRubyStyles, so
|
||||
// saving with applyStyles (default) re-injects the stylesheet and restyles the
|
||||
// already-rendered glosses live — no DOM rebuild / re-gloss needed.
|
||||
const glossFontSizeOptions = [
|
||||
{ value: '0.4', label: _('Small') },
|
||||
{ value: '0.5', label: _('Default') },
|
||||
{ value: '0.65', label: _('Large') },
|
||||
{ value: '0.8', label: _('Extra Large') },
|
||||
];
|
||||
|
||||
const handleSelectGlossFontSize = (event: React.ChangeEvent<HTMLSelectElement>) => {
|
||||
const value = Number(event.target.value) || 0.5;
|
||||
setGlossFontSize(value);
|
||||
saveViewSettings(envConfig, bookKey, 'wordLensGlossFontSize', value);
|
||||
};
|
||||
|
||||
const handleGlossColorChange = (hex: string) => setGlossColor(hex);
|
||||
|
||||
const handleGlossColorCommit = () => {
|
||||
saveViewSettings(envConfig, bookKey, 'wordLensGlossColor', glossColorRef.current);
|
||||
};
|
||||
|
||||
const handleResetGlossColor = () => {
|
||||
setGlossColor('');
|
||||
saveViewSettings(envConfig, bookKey, 'wordLensGlossColor', '');
|
||||
};
|
||||
|
||||
const handleToggleAutoDownload = () => {
|
||||
const next = !autoDownload;
|
||||
setAutoDownload(next);
|
||||
@@ -318,6 +358,45 @@ const WordLensPanel: React.FC<WordLensPanelProps> = ({ bookKey, onBack }) => {
|
||||
options={hintLangOptions}
|
||||
/>
|
||||
</SettingsRow>
|
||||
<SettingsRow
|
||||
label={_('Hint size')}
|
||||
description={_('Gloss text size above the word')}
|
||||
disabled={!wordLensEnabled}
|
||||
>
|
||||
<SettingsSelect
|
||||
value={String(glossFontSize)}
|
||||
onChange={handleSelectGlossFontSize}
|
||||
ariaLabel={_('Hint size')}
|
||||
options={glossFontSizeOptions}
|
||||
disabled={!wordLensEnabled}
|
||||
/>
|
||||
</SettingsRow>
|
||||
<SettingsRow
|
||||
label={_('Hint color')}
|
||||
description={glossColor ? glossColor : _('Default')}
|
||||
disabled={!wordLensEnabled}
|
||||
>
|
||||
<div className='flex items-center gap-2'>
|
||||
{glossColor && (
|
||||
<button
|
||||
type='button'
|
||||
onClick={handleResetGlossColor}
|
||||
className='btn btn-ghost btn-xs eink-bordered shrink-0'
|
||||
>
|
||||
{_('Default')}
|
||||
</button>
|
||||
)}
|
||||
<ColorInput
|
||||
label={_('Hint color')}
|
||||
value={glossColor || DEFAULT_GLOSS_SWATCH}
|
||||
onChange={handleGlossColorChange}
|
||||
onCommit={handleGlossColorCommit}
|
||||
swatchOnly
|
||||
showPickerIcon
|
||||
pickerPosition='right'
|
||||
/>
|
||||
</div>
|
||||
</SettingsRow>
|
||||
</BoxedList>
|
||||
|
||||
<BoxedList title={_('Data')}>
|
||||
|
||||
@@ -411,6 +411,8 @@ export const DEFAULT_WORD_LENS_CONFIG: WordLensConfig = {
|
||||
wordLensEnabled: false,
|
||||
wordLensLevel: 3,
|
||||
wordLensHintLang: '',
|
||||
wordLensGlossFontSize: 0.5,
|
||||
wordLensGlossColor: '',
|
||||
};
|
||||
|
||||
export const DEFAULT_SCREEN_CONFIG: ScreenConfig = {
|
||||
|
||||
@@ -7,21 +7,49 @@
|
||||
|
||||
// Leading dictionary POS tags: "a." "vt." "interj." (≤6 letters so interj. is covered).
|
||||
const LEADING_POS = /^\s*(?:[a-zA-Z]{1,6}\.\s*)+/;
|
||||
// Sense separators in both the zh packs ("，；、") and the en-target packs (",;/").
|
||||
const SENSE_SEPARATORS = /[,，、;；/]/;
|
||||
// Sense separators: ";" "；" "/". Commas are NOT here — within a cross-lingual sense
|
||||
// "，"/","/"、" separate near-synonyms, of which only the first is kept.
|
||||
const SENSE_SEPARATORS = /[;；/]/;
|
||||
const SYNONYM_SEPARATORS = /[,，、]/;
|
||||
// DISPLAY length cap. Lives here (not in the build), so changing it does NOT require
|
||||
// regenerating the gloss packs — they store the full hint and this trims it.
|
||||
const MAX_GLOSS_LEN = 24;
|
||||
|
||||
/** First sense only, POS/bracket/classifier noise stripped, length-capped. */
|
||||
export function cleanGloss(gloss: string): string {
|
||||
const capForDisplay = (s: string): string =>
|
||||
s.length <= MAX_GLOSS_LEN ? s : s.slice(0, MAX_GLOSS_LEN - 1).trimEnd() + '…';
|
||||
|
||||
/**
|
||||
* Normalize a stored gloss for display, applying the length cap (the packs store the
|
||||
* full hint; the cap is applied here so it can change without regenerating them).
|
||||
*
|
||||
* Cross-lingual (word) packs: keep at most the first TWO senses; within each sense
|
||||
* keep only the first near-synonym (so "阻止, 监禁, 拘留；隔离, 拘留, 滞留" → "阻止；隔离"),
|
||||
* POS/bracket/classifier noise stripped, joined by "；".
|
||||
*
|
||||
* Monolingual (en-en) packs hold the final hint already (a synonym/category, or a
|
||||
* ≤2-sense definition where "," is intra-sense content) — so they must NOT be
|
||||
* sense/synonym-split here; just normalize whitespace, then cap.
|
||||
*/
|
||||
export function cleanGloss(gloss: string, monolingual = false): string {
|
||||
if (!gloss) return '';
|
||||
const firstSense = gloss.split(SENSE_SEPARATORS)[0] ?? '';
|
||||
return firstSense
|
||||
.replace(LEADING_POS, '')
|
||||
.replace(/\[[^\]]*\]/g, '') // [ge4] pinyin / [医] domain tags
|
||||
.replace(/\bCL:.*$/, '') // CC-CEDICT classifier clause
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.slice(0, MAX_GLOSS_LEN);
|
||||
if (monolingual) return capForDisplay(gloss.replace(/\s+/g, ' ').trim());
|
||||
const senses = gloss
|
||||
.split(SENSE_SEPARATORS)
|
||||
.map((sense) =>
|
||||
sense
|
||||
.replace(LEADING_POS, '')
|
||||
.replace(/\[[^\]]*\]/g, '') // [ge4] pinyin / [医] domain tags
|
||||
.replace(/\bCL:.*$/, '') // CC-CEDICT classifier clause
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.split(SYNONYM_SEPARATORS)[0]! // first near-synonym within the sense
|
||||
.trim(),
|
||||
)
|
||||
.filter(Boolean);
|
||||
const out = [...new Set(senses)] // dedupe: two senses can share a first synonym
|
||||
.slice(0, 2)
|
||||
.join('；');
|
||||
return capForDisplay(out);
|
||||
}
|
||||
|
||||
// Reverse-derivation rules. Over-generate candidates ("all possibilities"); the
|
||||
@@ -53,6 +81,17 @@ export function baseFormCandidates(word: string): string[] {
|
||||
add(w.slice(0, -5) + 'y'); // happiness → happy
|
||||
}
|
||||
if (w.endsWith('less') && w.length >= 5) add(w.slice(0, -4)); // harmless → harm
|
||||
if ((w.endsWith('able') || w.endsWith('ible')) && w.length >= 6) {
|
||||
add(w.slice(0, -4)); // comfortable → comfort, sufferable → suffer
|
||||
add(w.slice(0, -4) + 'e'); // sensible → sense, lovable → love
|
||||
}
|
||||
// Negative/reversive prefixes (unhappy → happy, insufferable → suffer once -able is
|
||||
// stripped above): peel the prefix off the word and off each suffix candidate.
|
||||
for (const stem of [w, ...out]) {
|
||||
for (const p of ['un', 'in', 'im', 'ir', 'il']) {
|
||||
if (stem.startsWith(p) && stem.length - p.length >= 3) add(stem.slice(p.length));
|
||||
}
|
||||
}
|
||||
out.delete(w);
|
||||
return [...out];
|
||||
}
|
||||
@@ -67,6 +106,17 @@ const HAN = /\p{Script=Han}/u;
|
||||
* (knowing the base really does mean knowing the derived form). CJK glosses match
|
||||
* on a shared Han character; Latin-script glosses match on a shared word (≥3 letters).
|
||||
*/
|
||||
/**
|
||||
* Does a derived word's gloss literally mention its base form as a whole word?
|
||||
* For en-en (definition) packs the base/derived glosses rarely share a word, but a
|
||||
* transparent derivation's definition often names the base ("thickly" → "in a thick
|
||||
* manner"), which is a strong signal it's transparent and should inherit the rank.
|
||||
*/
|
||||
export function glossMentionsWord(gloss: string, word: string): boolean {
|
||||
if (!gloss || word.length < 3) return false;
|
||||
return new RegExp(`\\b${word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i').test(gloss);
|
||||
}
|
||||
|
||||
export function glossesShareMeaning(a: string, b: string): boolean {
|
||||
const an = (a || '').replace(PARTICLES, '');
|
||||
const bn = (b || '').replace(PARTICLES, '');
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { GlossEntry, GlossOccurrence, GlossSource, WordLensSourceLang } from './types';
|
||||
import { isDifficult } from './difficulty';
|
||||
import { baseFormCandidates, cleanGloss, glossesShareMeaning } from './gloss';
|
||||
import { baseFormCandidates, cleanGloss, glossMentionsWord, glossesShareMeaning } from './gloss';
|
||||
|
||||
export interface PlanOptions {
|
||||
sourceLang: WordLensSourceLang;
|
||||
@@ -10,6 +10,11 @@ export interface PlanOptions {
|
||||
maxOccurrences?: number;
|
||||
/** Chinese segmenter; injected for tests. Required for sourceLang 'zh'. */
|
||||
cutZh?: (text: string) => string[];
|
||||
/**
|
||||
* Monolingual pack (source === hint, e.g. en-en): the gloss is a definition
|
||||
* formatted at build time, so cleanGloss must preserve it (no sense-split / cut).
|
||||
*/
|
||||
monolingual?: boolean;
|
||||
}
|
||||
|
||||
// A foliate "section" is usually a whole chapter, so this is a per-chapter bound:
|
||||
@@ -76,7 +81,12 @@ export const planGlosses = (
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
occurrences.push({ start: t.start, end: t.end, word: t.word, gloss: cleanGloss(entry.gloss) });
|
||||
occurrences.push({
|
||||
start: t.start,
|
||||
end: t.end,
|
||||
word: t.word,
|
||||
gloss: cleanGloss(entry.gloss, opts.monolingual),
|
||||
});
|
||||
if (occurrences.length >= cap) {
|
||||
console.warn(`[wordlens] occurrence cap (${cap}) hit; some hints omitted`);
|
||||
break;
|
||||
@@ -92,7 +102,9 @@ const effectiveRank = (word: string, entry: GlossEntry, source: GlossSource): nu
|
||||
let rank = entry.rank;
|
||||
for (const base of baseFormCandidates(word)) {
|
||||
const b = source.lookup(base);
|
||||
if (b && glossesShareMeaning(entry.gloss, b.gloss)) rank = Math.min(rank, b.rank);
|
||||
if (b && (glossesShareMeaning(entry.gloss, b.gloss) || glossMentionsWord(entry.gloss, base))) {
|
||||
rank = Math.min(rank, b.rank);
|
||||
}
|
||||
}
|
||||
return rank;
|
||||
};
|
||||
|
||||
@@ -355,6 +355,10 @@ export interface WordLensConfig {
|
||||
wordLensLevel: number;
|
||||
/** Hint (target) language; '' = auto (app UI language). */
|
||||
wordLensHintLang: string;
|
||||
/** Gloss (<rt>) font size relative to the word, in em (default 0.5). */
|
||||
wordLensGlossFontSize: number;
|
||||
/** Gloss (<rt>) color as a hex string; '' = default (muted, theme-adaptive). */
|
||||
wordLensGlossColor: string;
|
||||
}
|
||||
|
||||
export interface ScreenConfig {
|
||||
|
||||
@@ -752,7 +752,13 @@ const getWarichuStyles = () => `
|
||||
}
|
||||
`;
|
||||
|
||||
const getRubyStyles = () => `
|
||||
// Word Lens gloss <rt> styling is user-configurable: the font size (relative to
|
||||
// the word, in em) and the color. An empty color keeps the default muted,
|
||||
// theme-adaptive look (inherit + 0.7 opacity); a set color paints at full opacity.
|
||||
const getRubyStyles = (viewSettings: ViewSettings) => {
|
||||
const fontSize = viewSettings.wordLensGlossFontSize || 0.5;
|
||||
const color = viewSettings.wordLensGlossColor || '';
|
||||
return `
|
||||
rt {
|
||||
user-select: none;
|
||||
-webkit-user-select: none;
|
||||
@@ -764,13 +770,14 @@ const getRubyStyles = () => `
|
||||
cursor: help;
|
||||
}
|
||||
ruby.wl-gloss > rt {
|
||||
font-size: 0.5em;
|
||||
font-size: ${fontSize}em;
|
||||
line-height: 1.1;
|
||||
opacity: 0.7;
|
||||
${color ? `color: ${color};\n opacity: 1;` : 'opacity: 0.7;'}
|
||||
font-weight: normal;
|
||||
text-align: center;
|
||||
}
|
||||
`;
|
||||
};
|
||||
|
||||
export interface ThemeCode {
|
||||
bg: string;
|
||||
@@ -880,7 +887,7 @@ export const getStyles = (
|
||||
);
|
||||
const translationStyles = getTranslationStyles(viewSettings.showTranslateSource!);
|
||||
const warichuStyles = getWarichuStyles();
|
||||
const rubyStyles = getRubyStyles();
|
||||
const rubyStyles = getRubyStyles(viewSettings);
|
||||
const userStylesheet = viewSettings.userStylesheet!;
|
||||
// The `@namespace` declaration must lead the stylesheet: a `@namespace` rule
|
||||
// placed after any style or `@font-face` rule is invalid and silently ignored,
|
||||
|
||||
Reference in New Issue
Block a user