fix(dictionary): add Chinese dictionary lookup with pinyin support (#3784)

The Wiktionary REST API does not support Chinese entries — simplified
characters return 404, traditional characters omit Chinese results, and
no pronunciation data is ever included. Switch Chinese lookups to the
Wiktionary Action API which returns full wikitext with pinyin and
definitions. Also add encodeURIComponent to the REST API URL for all
other languages.

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
pythontyphon
2026-04-07 00:16:26 -04:00
committed by GitHub
parent cf21a752c6
commit 017a9338b3
3 changed files with 502 additions and 80 deletions
@@ -0,0 +1,260 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import {
parseRedirect,
parsePinyin,
cleanWikiMarkup,
parseDefinitions,
fetchChineseDefinition,
} from '@/services/dictionaries/chineseDict';
// Sample wikitext for 書 (traditional, full entry)
const WIKITEXT_SHU_TRADITIONAL = `
==Chinese==
===Glyph origin===
Some origin text.
===Pronunciation===
{{zh-pron
|m=shū
|c=syu1
|cat=n,v,pn
}}
===Definitions===
{{head|zh|hanzi}}
# [[book]]; [[codex]]
# [[letter]]; [[document]]
# form of a written or printed [[Chinese character]]; [[style]]
# {{lb|zh|literary}} [[Chinese character]]; [[writing]]; [[script]]
# {{lb|zh|historical}} ancient government [[post]]
# [[storytelling]]
# to [[write]]
# {{surname|zh}}
===Compounds===
* {{zh-l|書法}}
`;
// Sample wikitext for 书 (simplified, redirect)
const WIKITEXT_SHU_SIMPLIFIED = `
{{also|書}}
{{character info}}
==Translingual==
===Han character===
{{Han char|rn=5|rad=乙|as=3|sn=4}}
==Chinese==
===Glyph origin===
{{Han simp|書}}
===Definitions===
{{zh-see|書}}
`;
// Sample wikitext for 你好 (multi-character word)
const WIKITEXT_NI_HAO = `
==Chinese==
===Pronunciation===
{{zh-pron
|m=nǐ hǎo
|c=nei5 hou2
|cat=intj
}}
===Definitions===
{{head|zh|interjection}}
# [[hello]]; [[hi]]
====Usage notes====
Often used as a polite greeting.
===See also===
* {{zh-l|您好}}
`;
describe('parseRedirect', () => {
it('detects zh-see redirect', () => {
expect(parseRedirect(WIKITEXT_SHU_SIMPLIFIED)).toBe('書');
});
it('returns null when no redirect', () => {
expect(parseRedirect(WIKITEXT_SHU_TRADITIONAL)).toBeNull();
});
it('returns null for empty string', () => {
expect(parseRedirect('')).toBeNull();
});
});
describe('parsePinyin', () => {
it('extracts pinyin from zh-pron template', () => {
expect(parsePinyin(WIKITEXT_SHU_TRADITIONAL)).toBe('shū');
});
it('extracts multi-syllable pinyin', () => {
expect(parsePinyin(WIKITEXT_NI_HAO)).toBe('nǐ hǎo');
});
it('returns null when no zh-pron template', () => {
expect(parsePinyin('no pronunciation here')).toBeNull();
});
it('returns null when zh-pron has no mandarin entry', () => {
const wikitext = '{{zh-pron\n|c=syu1\n}}';
expect(parsePinyin(wikitext)).toBeNull();
});
});
describe('cleanWikiMarkup', () => {
it('cleans simple wiki links', () => {
expect(cleanWikiMarkup('[[book]]')).toBe('book');
});
it('cleans piped wiki links', () => {
expect(cleanWikiMarkup('[[Chinese character|character]]')).toBe('character');
});
it('cleans language label templates', () => {
expect(cleanWikiMarkup('{{lb|zh|literary}} text')).toBe('(literary) text');
});
it('cleans multiple labels', () => {
expect(cleanWikiMarkup('{{lb|zh|historical|archaic}} text')).toBe('(historical, archaic) text');
});
it('cleans surname template', () => {
expect(cleanWikiMarkup('{{surname|zh}}')).toBe('A surname');
});
it('cleans zh-abbrev template', () => {
expect(cleanWikiMarkup('{{zh-abbrev|书经}}')).toBe('abbreviation of 书经');
});
it('cleans l template', () => {
expect(cleanWikiMarkup('{{l|zh|書}}')).toBe('書');
});
it('removes unknown templates', () => {
expect(cleanWikiMarkup('{{unknown|template}}')).toBe('');
});
it('handles mixed content', () => {
const input = '{{lb|zh|literary}} [[Chinese character]]; [[writing]]';
expect(cleanWikiMarkup(input)).toBe('(literary) Chinese character; writing');
});
});
describe('parseDefinitions', () => {
it('parses definitions from traditional character entry', () => {
const result = parseDefinitions(WIKITEXT_SHU_TRADITIONAL);
expect(result).toHaveLength(1);
expect(result[0]!.meanings).toContain('book; codex');
expect(result[0]!.meanings).toContain('letter; document');
expect(result[0]!.meanings).toContain('to write');
expect(result[0]!.meanings).toContain('A surname');
});
it('parses definitions from multi-character word', () => {
const result = parseDefinitions(WIKITEXT_NI_HAO);
expect(result).toHaveLength(1);
expect(result[0]!.meanings).toContain('hello; hi');
});
it('returns empty for redirect entries', () => {
const result = parseDefinitions(WIKITEXT_SHU_SIMPLIFIED);
expect(result).toHaveLength(0);
});
it('returns empty for text without definitions', () => {
expect(parseDefinitions('no definitions here')).toHaveLength(0);
});
it('skips sub-definition lines (## lines)', () => {
const wikitext = `===Definitions===
{{head|zh|hanzi}}
# main definition
## sub definition
# another main
===Other===`;
const result = parseDefinitions(wikitext);
expect(result[0]!.meanings).toHaveLength(2);
expect(result[0]!.meanings).toContain('main definition');
expect(result[0]!.meanings).toContain('another main');
});
});
describe('fetchChineseDefinition', () => {
beforeEach(() => {
vi.restoreAllMocks();
});
it('fetches and parses a traditional character', async () => {
vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
new Response(
JSON.stringify({
parse: { wikitext: { '*': WIKITEXT_SHU_TRADITIONAL } },
}),
),
);
const result = await fetchChineseDefinition('書');
expect(result).not.toBeNull();
expect(result!.pinyin).toBe('shū');
expect(result!.word).toBe('書');
expect(result!.definitions[0]!.meanings).toContain('book; codex');
});
it('follows simplified → traditional redirect', async () => {
const fetchSpy = vi.spyOn(globalThis, 'fetch');
// First call returns simplified (redirect)
fetchSpy.mockResolvedValueOnce(
new Response(
JSON.stringify({
parse: { wikitext: { '*': WIKITEXT_SHU_SIMPLIFIED } },
}),
),
);
// Second call returns traditional (full entry)
fetchSpy.mockResolvedValueOnce(
new Response(
JSON.stringify({
parse: { wikitext: { '*': WIKITEXT_SHU_TRADITIONAL } },
}),
),
);
const result = await fetchChineseDefinition('书');
expect(result).not.toBeNull();
expect(result!.word).toBe('書');
expect(result!.pinyin).toBe('shū');
expect(fetchSpy).toHaveBeenCalledTimes(2);
});
it('returns null on network error', async () => {
vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(new Response(null, { status: 404 }));
const result = await fetchChineseDefinition('nonexistent');
expect(result).toBeNull();
});
it('returns null when wikitext has no useful data', async () => {
vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
new Response(
JSON.stringify({
parse: { wikitext: { '*': '==Translingual==\nSome unrelated content.' } },
}),
),
);
const result = await fetchChineseDefinition('test');
expect(result).toBeNull();
});
});
@@ -2,6 +2,8 @@ import React, { useEffect, useRef, useState } from 'react';
import { MdArrowBack } from 'react-icons/md';
import { Position } from '@/utils/sel';
import { useTranslation } from '@/hooks/useTranslation';
import { normalizedLangCode } from '@/utils/lang';
import { fetchChineseDefinition } from '@/services/dictionaries/chineseDict';
import Popup from '@/components/Popup';
type Definition = {
@@ -178,94 +180,151 @@ const WiktionaryPopup: React.FC<WiktionaryPopupProps> = ({
const footer = footerRef.current;
if (!main || !footer) return;
const renderError = (word: string) => {
footer.dataset['state'] = 'error';
const div = document.createElement('div');
div.className =
'flex flex-col items-center justify-center w-full h-full text-center absolute inset-0';
const h1 = document.createElement('h1');
h1.innerText = _('Error');
h1.className = 'text-lg font-bold';
const p = document.createElement('p');
p.innerHTML = _('Unable to load the word. Try searching directly on {{link}}.', {
link: `<a href="https://en.wiktionary.org/w/index.php?search=${encodeURIComponent(
word,
)}" target="_blank" rel="noopener noreferrer" class="not-eink:text-primary underline">Wiktionary</a>`,
});
div.append(h1, p);
main.append(div);
};
const fetchChineseDefs = async (word: string) => {
const entry = await fetchChineseDefinition(word);
if (!entry) throw new Error('No Chinese entry found');
const hgroup = document.createElement('hgroup');
const h1 = document.createElement('h1');
h1.innerText = entry.word;
h1.className = 'text-lg font-bold';
hgroup.append(h1);
if (entry.pinyin) {
const pinyinEl = document.createElement('p');
pinyinEl.innerText = entry.pinyin;
pinyinEl.className = 'text-base italic not-eink:opacity-85';
hgroup.append(pinyinEl);
}
const langEl = document.createElement('p');
langEl.innerText = 'Chinese';
langEl.className = 'text-sm italic not-eink:opacity-75';
hgroup.append(langEl);
main.append(hgroup);
entry.definitions.forEach(({ partOfSpeech, meanings }) => {
const h2 = document.createElement('h2');
h2.innerText = partOfSpeech;
h2.className = 'text-base font-semibold mt-4';
const ol = document.createElement('ol');
ol.className = 'pl-8 list-decimal';
meanings.forEach((meaning) => {
const li = document.createElement('li');
li.innerText = meaning;
ol.appendChild(li);
});
main.appendChild(h2);
main.appendChild(ol);
});
footer.dataset['state'] = 'loaded';
};
const fetchWiktionaryDefs = async (word: string, language?: string) => {
const response = await fetch(
`https://en.wiktionary.org/api/rest_v1/page/definition/${encodeURIComponent(word)}`,
);
if (!response.ok) {
throw new Error('Failed to fetch definitions');
}
const json = await response.json();
const results: Result[] | undefined = language
? json[language] || json['en']
: json[Object.keys(json)[0]!];
if (!results || results.length === 0) {
throw new Error('No results found');
}
const hgroup = document.createElement('hgroup');
const h1 = document.createElement('h1');
h1.innerText = word;
h1.className = 'text-lg font-bold';
const p = document.createElement('p');
p.innerText = results[0]!.language;
p.className = 'text-sm italic not-eink:opacity-75';
hgroup.append(h1, p);
main.append(hgroup);
results.forEach(({ partOfSpeech, definitions }: Result) => {
const h2 = document.createElement('h2');
h2.innerText = partOfSpeech;
h2.className = 'text-base font-semibold mt-4';
const ol = document.createElement('ol');
ol.className = 'pl-8 list-decimal';
definitions.forEach(({ definition, examples }: Definition) => {
if (!definition) return;
const li = document.createElement('li');
const processedContent = interceptDictLinks(definition);
li.append(...processedContent);
if (examples) {
const ul = document.createElement('ul');
ul.className = 'pl-8 list-disc text-sm italic not-eink:opacity-75';
examples.forEach((example) => {
const exampleLi = document.createElement('li');
exampleLi.innerHTML = example;
ul.appendChild(exampleLi);
});
li.appendChild(ul);
}
ol.appendChild(li);
});
main.appendChild(h2);
main.appendChild(ol);
});
footer.dataset['state'] = 'loaded';
};
const fetchDefinitions = async (word: string, language?: string) => {
main.innerHTML = '';
footer.dataset['state'] = 'loading';
try {
const response = await fetch(
`https://en.wiktionary.org/api/rest_v1/page/definition/${word}`,
);
if (!response.ok) {
throw new Error('Failed to fetch definitions');
const isChineseLookup = language && normalizedLangCode(language) === 'zh';
if (isChineseLookup) {
await fetchChineseDefs(word);
} else {
await fetchWiktionaryDefs(word, language);
}
const json = await response.json();
const results: Result[] | undefined = language
? json[language] || json['en']
: json[Object.keys(json)[0]!];
if (!results || results.length === 0) {
throw new Error('No results found');
}
const hgroup = document.createElement('hgroup');
const h1 = document.createElement('h1');
h1.innerText = word;
h1.className = 'text-lg font-bold';
const p = document.createElement('p');
p.innerText = results[0]!.language;
p.className = 'text-sm italic not-eink:opacity-75';
hgroup.append(h1, p);
main.append(hgroup);
results.forEach(({ partOfSpeech, definitions }: Result) => {
const h2 = document.createElement('h2');
h2.innerText = partOfSpeech;
h2.className = 'text-base font-semibold mt-4';
const ol = document.createElement('ol');
ol.className = 'pl-8 list-decimal';
definitions.forEach(({ definition, examples }: Definition) => {
if (!definition) return;
const li = document.createElement('li');
const processedContent = interceptDictLinks(definition);
li.append(...processedContent);
if (examples) {
const ul = document.createElement('ul');
ul.className = 'pl-8 list-disc text-sm italic not-eink:opacity-75';
examples.forEach((example) => {
const exampleLi = document.createElement('li');
exampleLi.innerHTML = example;
ul.appendChild(exampleLi);
});
li.appendChild(ul);
}
ol.appendChild(li);
});
main.appendChild(h2);
main.appendChild(ol);
});
footer.dataset['state'] = 'loaded';
} catch (error) {
console.error(error);
footer.dataset['state'] = 'error';
const div = document.createElement('div');
div.className =
'flex flex-col items-center justify-center w-full h-full text-center absolute inset-0';
const h1 = document.createElement('h1');
h1.innerText = _('Error');
h1.className = 'text-lg font-bold';
const p = document.createElement('p');
p.innerHTML = _('Unable to load the word. Try searching directly on {{link}}.', {
link: `<a href="https://en.wiktionary.org/w/index.php?search=${encodeURIComponent(
word,
)}" target="_blank" rel="noopener noreferrer" class="not-eink:text-primary underline">Wiktionary</a>`,
});
div.append(h1, p);
main.append(div);
renderError(word);
}
};
@@ -0,0 +1,103 @@
export type ChineseDefinition = {
partOfSpeech: string;
meanings: string[];
};
export type ChineseEntry = {
word: string;
pinyin: string | null;
definitions: ChineseDefinition[];
};
const WIKTIONARY_API = 'https://en.wiktionary.org/w/api.php';
async function fetchWikitext(word: string): Promise<string | null> {
const url = `${WIKTIONARY_API}?action=parse&page=${encodeURIComponent(word)}&prop=wikitext&format=json&origin=*`;
const response = await fetch(url);
if (!response.ok) return null;
const json = await response.json();
return json?.parse?.wikitext?.['*'] ?? null;
}
export function parseRedirect(wikitext: string): string | null {
const match = wikitext.match(/\{\{zh-see\|([^|}]+)/);
return match?.[1] ?? null;
}
export function parsePinyin(wikitext: string): string | null {
const pronMatch = wikitext.match(/\{\{zh-pron[\s\S]*?\}\}/);
if (!pronMatch) return null;
const mMatch = pronMatch[0].match(/\|m=([^|}\n]+)/);
if (!mMatch) return null;
return mMatch[1]!.trim();
}
export function cleanWikiMarkup(text: string): string {
let result = text;
// [[word|display]] → display
result = result.replace(/\[\[(?:[^|\]]*\|)?([^\]]*)\]\]/g, '$1');
// {{lb|zh|...|...}} → (label)
result = result.replace(/\{\{lb\|zh\|([^}]*)\}\}/g, (_match, params: string) => {
const labels = params
.split('|')
.filter((p: string) => p !== '_' && p !== 'zh')
.join(', ');
return labels ? `(${labels})` : '';
});
// {{surname|zh}} → A surname
result = result.replace(/\{\{surname\|zh\}\}/g, 'A surname');
// {{zh-abbrev|X}} → abbreviation of X
result = result.replace(/\{\{zh-abbrev\|([^|}]+)(?:\|[^}]*)?\}\}/g, 'abbreviation of $1');
// {{gloss|X}} → (X)
result = result.replace(/\{\{gloss\|([^}]+)\}\}/g, '($1)');
// {{qualifier|X}} → (X)
result = result.replace(/\{\{qualifier\|([^}]+)\}\}/g, '($1)');
// {{l|...|word}} or {{m|...|word}} → word
result = result.replace(/\{\{[lm]\|[^|]+\|([^|}]+)(?:\|[^}]*)?\}\}/g, '$1');
// Remove remaining templates
result = result.replace(/\{\{[^}]*\}\}/g, '');
// Clean up whitespace
result = result.replace(/\s{2,}/g, ' ').trim();
return result;
}
export function parseDefinitions(wikitext: string): ChineseDefinition[] {
// Find the Chinese ===Definitions=== section
const defSectionMatch = wikitext.match(
/===Definitions===\s*\n(?:\{\{[^}]*\}\}\s*\n)?([\s\S]*?)(?=\n===|\n==(?!=))/,
);
if (!defSectionMatch) return [];
const defLines = defSectionMatch[1]!
.split('\n')
.filter((line) => /^#[^#*:]/.test(line))
.map((line) => cleanWikiMarkup(line.replace(/^#\s*/, '')))
.filter((line) => line.length > 0);
if (defLines.length === 0) return [];
return [{ partOfSpeech: 'Definitions', meanings: defLines }];
}
export async function fetchChineseDefinition(word: string): Promise<ChineseEntry | null> {
let wikitext = await fetchWikitext(word);
if (!wikitext) return null;
// Handle simplified → traditional redirect
const redirect = parseRedirect(wikitext);
if (redirect) {
wikitext = await fetchWikitext(redirect);
if (!wikitext) return null;
}
const pinyin = parsePinyin(wikitext);
const definitions = parseDefinitions(wikitext);
if (!pinyin && definitions.length === 0) return null;
return {
word: redirect ?? word,
pinyin,
definitions,
};
}