forked from akai/readest
refactor(wordlens): rename "Word Wise" to "Word Lens" (#4633)
"Word Wise" is a Kindle trademark, so rename the inline-gloss feature to
"Word Lens" throughout the product.
- User-facing strings → "Word Lens" across all 34 locales; brand translated
for Chinese (zh-CN 单词透镜, zh-TW 單詞透鏡) and German (Word-Lens-Daten).
- Code identifiers: WordWise→WordLens, wordWise→wordLens, WORD_WISE→WORD_LENS.
- Files/dirs: src/services/wordwise→wordlens, WordWisePanel→WordLensPanel,
wordwise{Ruby,Section}.ts, build/sync scripts, test dirs/fixtures,
data/wordwise→data/wordlens.
- Storage paths: CDN base, R2 key, on-device cache dir, WORDLENS_R2_BUCKET env,
pnpm wordlens:{manifest,sync}. manifest.json is path-agnostic so its
sha256/bytes stay valid (verified).
- biome.json: point the formatter-ignore at data/wordlens so the generated
one-line gloss packs aren't pretty-printed on commit.
Migration notes:
- Re-run `pnpm wordlens:sync` to upload packs to cdn.readest.com/wordlens/.
- Persisted view-settings keys renamed (wordWiseEnabled/Level/HintLang and
wordWiseAutoDownload) — saved values reset to defaults once on upgrade.
- Cached packs under the old Data/wordwise/ orphan (harmless re-download).
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+13
-13
@@ -1,8 +1,8 @@
|
||||
// Build trimmed Word Wise gloss indices from open datasets.
|
||||
// Build trimmed Word Lens gloss indices from open datasets.
|
||||
//
|
||||
// node scripts/build-wordwise-data.mjs en-zh path/to/ecdict.csv [topN]
|
||||
// node scripts/build-wordwise-data.mjs zh-en path/to/cedict.txt path/to/hsk.json [topN]
|
||||
// node scripts/build-wordwise-data.mjs build <src> <tgt> <freq.txt> <gloss.jsonl> [topN]
|
||||
// node scripts/build-wordlens-data.mjs en-zh path/to/ecdict.csv [topN]
|
||||
// node scripts/build-wordlens-data.mjs zh-en path/to/cedict.txt path/to/hsk.json [topN]
|
||||
// node scripts/build-wordlens-data.mjs build <src> <tgt> <freq.txt> <gloss.jsonl> [topN]
|
||||
//
|
||||
// The generalized `build` mode assembles a pack for any (src→tgt) pair where one
|
||||
// side is English, from two open datasets:
|
||||
@@ -11,9 +11,9 @@
|
||||
// tgt === 'en' → foreign headword → English glosses (extractXToEn).
|
||||
// src === 'en' → English headword → target-language words (extractEnToX).
|
||||
//
|
||||
// Outputs data/wordwise/<pair>.json in the GlossIndexData shape:
|
||||
// Outputs data/wordlens/<pair>.json in the GlossIndexData shape:
|
||||
// { meta, entries: { word: { r, g } }, inflections: { form: lemma } }
|
||||
// plus data/wordwise/manifest.json indexing the available packs.
|
||||
// plus data/wordlens/manifest.json indexing the available packs.
|
||||
//
|
||||
// ECDICT (MIT): columns word,phonetic,definition,translation,pos,collins,
|
||||
// oxford,tag,bnc,frq,exchange,detail,audio. We keep word, frq (rank),
|
||||
@@ -33,7 +33,7 @@ import { resolve } from 'node:path';
|
||||
import { createInterface } from 'node:readline';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
const OUT_DIR = resolve('data/wordwise');
|
||||
const OUT_DIR = resolve('data/wordlens');
|
||||
const TOP_DEFAULT = 30000;
|
||||
|
||||
// Keep a hint short + clean: drop bracket annotations ([医], [网络], [ge4]),
|
||||
@@ -505,7 +505,7 @@ async function main() {
|
||||
const [src, tgt, freqPath, glossPath, topN] = rest;
|
||||
if (!src || !tgt || !freqPath || !glossPath)
|
||||
throw new Error(
|
||||
'usage: build-wordwise-data.mjs build <src> <tgt> <freq.txt> <gloss.jsonl> [topN]',
|
||||
'usage: build-wordlens-data.mjs build <src> <tgt> <freq.txt> <gloss.jsonl> [topN]',
|
||||
);
|
||||
const freqList = parseFrequencyWords(readFileSync(freqPath, 'utf8'));
|
||||
let glossMap;
|
||||
@@ -532,7 +532,7 @@ async function main() {
|
||||
const [src, tgt, freqPath, dbPath, topN, lemmaFile] = rest;
|
||||
if (!src || !tgt || !freqPath || !dbPath)
|
||||
throw new Error(
|
||||
'usage: build-wordwise-data.mjs build-wikdict <src> <tgt> <freq.txt> <wikdict.sqlite3> [topN] [lemma.txt]',
|
||||
'usage: build-wordlens-data.mjs build-wikdict <src> <tgt> <freq.txt> <wikdict.sqlite3> [topN] [lemma.txt]',
|
||||
);
|
||||
let rows;
|
||||
try {
|
||||
@@ -574,7 +574,7 @@ async function main() {
|
||||
writeManifest();
|
||||
} else if (pair === 'en-zh') {
|
||||
const [csv, topN] = rest;
|
||||
if (!csv) throw new Error('usage: build-wordwise-data.mjs en-zh <ecdict.csv> [topN]');
|
||||
if (!csv) throw new Error('usage: build-wordlens-data.mjs en-zh <ecdict.csv> [topN]');
|
||||
const data = buildEnZh(readFileSync(csv, 'utf8'), Number(topN) || TOP_DEFAULT);
|
||||
writeFileSync(resolve(OUT_DIR, 'en-zh.json'), JSON.stringify(data));
|
||||
console.log(
|
||||
@@ -584,7 +584,7 @@ async function main() {
|
||||
} else if (pair === 'zh-en') {
|
||||
const [cedict, hsk, topN] = rest;
|
||||
if (!cedict || !hsk)
|
||||
throw new Error('usage: build-wordwise-data.mjs zh-en <cedict.txt> <hsk.json> [topN]');
|
||||
throw new Error('usage: build-wordlens-data.mjs zh-en <cedict.txt> <hsk.json> [topN]');
|
||||
const data = buildZhEn(
|
||||
readFileSync(cedict, 'utf8'),
|
||||
JSON.parse(readFileSync(hsk, 'utf8')),
|
||||
@@ -598,12 +598,12 @@ async function main() {
|
||||
console.log('manifest.json:', m.packs.length, 'packs');
|
||||
} else {
|
||||
throw new Error(
|
||||
'usage: build-wordwise-data.mjs <en-zh|zh-en|build|build-wikdict|manifest> <sources...> [topN]',
|
||||
'usage: build-wordlens-data.mjs <en-zh|zh-en|build|build-wikdict|manifest> <sources...> [topN]',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Only run the CLI when executed directly (`node build-wordwise-data.mjs ...`),
|
||||
// Only run the CLI when executed directly (`node build-wordlens-data.mjs ...`),
|
||||
// not when imported by the unit tests.
|
||||
if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) {
|
||||
main().catch((err) => {
|
||||
+9
-9
@@ -1,7 +1,7 @@
|
||||
// Upload the committed Word Wise gloss packs + manifest to the cdn.readest.com
|
||||
// R2 bucket under /wordwise/. Maintainer/CI tool — run after regenerating data.
|
||||
// Upload the committed Word Lens gloss packs + manifest to the cdn.readest.com
|
||||
// R2 bucket under /wordlens/. Maintainer/CI tool — run after regenerating data.
|
||||
//
|
||||
// WORDWISE_R2_BUCKET=<bucket> node scripts/sync-wordwise-r2.mjs
|
||||
// WORDLENS_R2_BUCKET=<bucket> node scripts/sync-wordlens-r2.mjs
|
||||
//
|
||||
// Pack files get a one-year immutable cache (the app cache-busts via a ?v=<sha8>
|
||||
// query); manifest.json gets a short max-age so new packs surface quickly. Packs
|
||||
@@ -18,7 +18,7 @@ import { readdirSync } from 'node:fs';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
const SRC_DIR = resolve('data/wordwise');
|
||||
const SRC_DIR = resolve('data/wordlens');
|
||||
const PACK_CACHE = 'public, max-age=31536000, immutable';
|
||||
const MANIFEST_CACHE = 'public, max-age=300';
|
||||
const SUCCESS_RE = /Upload complete/i;
|
||||
@@ -28,7 +28,7 @@ const PER_FILE_TIMEOUT_MS = 120_000; // hard backstop per file
|
||||
const uploadOne = (bucket, file) =>
|
||||
new Promise((resolveP) => {
|
||||
const cacheControl = file === 'manifest.json' ? MANIFEST_CACHE : PACK_CACHE;
|
||||
const key = `${bucket}/wordwise/${file}`;
|
||||
const key = `${bucket}/wordlens/${file}`;
|
||||
console.log(`Uploading ${file} -> ${key}`);
|
||||
const child = spawn(
|
||||
'wrangler',
|
||||
@@ -82,13 +82,13 @@ const uploadOne = (bucket, file) =>
|
||||
});
|
||||
|
||||
async function main() {
|
||||
const bucket = process.env.WORDWISE_R2_BUCKET;
|
||||
const bucket = process.env.WORDLENS_R2_BUCKET;
|
||||
if (!bucket) {
|
||||
throw new Error('WORDWISE_R2_BUCKET env var is required (the cdn.readest.com R2 bucket name)');
|
||||
throw new Error('WORDLENS_R2_BUCKET env var is required (the cdn.readest.com R2 bucket name)');
|
||||
}
|
||||
const all = readdirSync(SRC_DIR).filter((f) => f.endsWith('.json'));
|
||||
if (!all.includes('manifest.json')) {
|
||||
throw new Error('manifest.json missing — run `pnpm wordwise:manifest` first');
|
||||
throw new Error('manifest.json missing — run `pnpm wordlens:manifest` first');
|
||||
}
|
||||
const ordered = [...all.filter((f) => f !== 'manifest.json').sort(), 'manifest.json'];
|
||||
|
||||
@@ -101,7 +101,7 @@ async function main() {
|
||||
else failed.push(file);
|
||||
}
|
||||
|
||||
console.log(`\nSynced ${ok}/${ordered.length} files to ${bucket}/wordwise/`);
|
||||
console.log(`\nSynced ${ok}/${ordered.length} files to ${bucket}/wordlens/`);
|
||||
if (failed.length) {
|
||||
console.error(`Failed: ${failed.join(', ')}`);
|
||||
process.exit(1);
|
||||
Reference in New Issue
Block a user