feat(reader): add TTS speak button to dictionary popup (#4876) (#4957)

Add a speaker button to the dictionary popup/sheet header that pronounces
the current headword. Tapping it speaks via Edge TTS, falling back to the
platform speech engine (Web Speech on desktop/web, native on the mobile
app) when Edge is unavailable.

To speak as soon as possible, a dedicated wordPronouncer bypasses the
reader's TTSController entirely: it never runs EdgeTTSClient.init() (which
wastes a round trip synthesizing "test"), calls EdgeSpeechTTS directly
(whose static MP3 cache makes repeat words instant), and schedules one
chunk on a dedicated Web Audio context isolated from any active read-aloud
session. The context is warmed synchronously inside the click gesture so
playback is not blocked by autoplay policy after the network await.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Huang Xin
2026-07-06 13:56:51 +09:00
committed by GitHub
parent 6f3b401c24
commit 4527aa277a
6 changed files with 463 additions and 4 deletions
@@ -69,6 +69,14 @@ vi.mock('@/hooks/useTranslation', () => ({
useTranslation: () => (s: string) => s, useTranslation: () => (s: string) => s,
})); }));
// The word pronouncer talks to Edge/WebAudio; stub it so the sheet's speak
// button can be exercised without real audio. (#4876)
vi.mock('@/services/tts/wordPronouncer', () => ({
pronounceWord: vi.fn().mockResolvedValue(undefined),
warmWordAudio: vi.fn(),
cancelWordPronounce: vi.fn(),
}));
const mockOpenUrl = vi.fn().mockResolvedValue(undefined); const mockOpenUrl = vi.fn().mockResolvedValue(undefined);
vi.mock('@tauri-apps/plugin-opener', () => ({ vi.mock('@tauri-apps/plugin-opener', () => ({
openUrl: (...args: unknown[]) => mockOpenUrl(...args), openUrl: (...args: unknown[]) => mockOpenUrl(...args),
@@ -217,6 +225,7 @@ const buildExactProvider = (storedHeadword: string): DictionaryProvider => {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
import DictionarySheet from '@/app/reader/components/annotator/DictionarySheet'; import DictionarySheet from '@/app/reader/components/annotator/DictionarySheet';
import { pronounceWord, warmWordAudio } from '@/services/tts/wordPronouncer';
const renderSheet = ( const renderSheet = (
props: Partial<{ props: Partial<{
@@ -271,6 +280,24 @@ describe('DictionarySheet — header', () => {
}); });
}); });
describe('DictionarySheet — speak button', () => {
it('warms audio and pronounces the current word when the speaker is tapped', async () => {
providersForNextRender.push(buildRealStarDictProvider());
renderSheet({ word: 'hello', lang: 'en' });
const speak = await screen.findByLabelText('Speak');
fireEvent.click(speak);
expect(warmWordAudio).toHaveBeenCalledTimes(1);
expect(pronounceWord).toHaveBeenCalledWith(
'hello',
'en',
expect.any(Object),
expect.any(Function),
);
});
});
describe('DictionarySheet — concurrent lookup', () => { describe('DictionarySheet — concurrent lookup', () => {
it('fans out lookups across every enabled definition provider', async () => { it('fans out lookups across every enabled definition provider', async () => {
const stardict = buildRealStarDictProvider(); const stardict = buildRealStarDictProvider();
@@ -0,0 +1,184 @@
/**
* wordPronouncer — pronounces a single dictionary word.
*
* The pronouncer is deliberately independent of the reader's TTSController: it
* synthesizes via EdgeSpeechTTS directly (no throwaway init synth), plays on a
* dedicated Web Audio context, and drops to the platform speech client
* (Web Speech on desktop/web, native on the mobile app) when Edge is
* unavailable. These tests pin that Edge-first / fallback-on-failure contract
* and the language -> Edge voice selection.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { TTSUtils } from '@/services/tts/TTSUtils';
const h = vi.hoisted(() => {
const createAudioData = vi.fn();
let sessionOnEvent: ((e: { type: string; message?: string }) => void) | null = null;
const player = {
ensureContext: vi.fn().mockResolvedValue({}),
decode: vi.fn().mockResolvedValue({ duration: 0.5 }),
startSession: vi.fn((onEvent: (e: { type: string; message?: string }) => void) => {
sessionOnEvent = onEvent;
return 1;
}),
scheduleChunk: vi.fn(),
endSession: vi.fn(),
abortSession: vi.fn(),
shutdown: vi.fn().mockResolvedValue(undefined),
fireSessionEnd: () => sessionOnEvent?.({ type: 'session-end' }),
fireContextError: () => sessionOnEvent?.({ type: 'context-error', message: 'boom' }),
};
const makeClient = (speak: unknown) => ({
init: vi.fn().mockResolvedValue(true),
setPrimaryLang: vi.fn(),
speak,
shutdown: vi.fn().mockResolvedValue(undefined),
});
// eslint-disable-next-line require-yield
const webSpeak = vi.fn(async function* (_ssml: string, _signal: AbortSignal) {});
// eslint-disable-next-line require-yield
const nativeSpeak = vi.fn(async function* (_ssml: string, _signal: AbortSignal) {});
return {
createAudioData,
player,
webClient: makeClient(webSpeak),
nativeClient: makeClient(nativeSpeak),
webSpeak,
nativeSpeak,
};
});
vi.mock('@/libs/edgeTTS', () => {
class EdgeSpeechTTS {
static voices = [
{ name: 'Aria', id: 'en-US-AriaNeural', lang: 'en-US' },
{ name: 'Ryan', id: 'en-GB-RyanNeural', lang: 'en-GB' },
{ name: 'Denise', id: 'fr-FR-DeniseNeural', lang: 'fr-FR' },
];
createAudioData = h.createAudioData;
}
return { EdgeSpeechTTS };
});
vi.mock('@/services/tts/WebAudioPlayer', () => ({
WebAudioPlayer: class {
constructor() {
Object.assign(this, h.player);
}
},
}));
vi.mock('@/services/tts/WebSpeechClient', () => ({
WebSpeechClient: class {
constructor() {
Object.assign(this, h.webClient);
}
},
}));
vi.mock('@/services/tts/NativeTTSClient', () => ({
NativeTTSClient: class {
constructor() {
Object.assign(this, h.nativeClient);
}
},
}));
import { pronounceWord, pickEdgeVoiceId, cancelWordPronounce } from '@/services/tts/wordPronouncer';
const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
beforeEach(() => {
vi.clearAllMocks();
(globalThis as unknown as { AudioContext: unknown }).AudioContext = vi.fn();
h.createAudioData.mockReset();
});
describe('pickEdgeVoiceId', () => {
it('returns the first Edge voice whose locale matches the language', () => {
expect(pickEdgeVoiceId('en')).toBe('en-US-AriaNeural');
expect(pickEdgeVoiceId('fr')).toBe('fr-FR-DeniseNeural');
});
it('falls back to the default English voice for an unknown language', () => {
expect(pickEdgeVoiceId('xx')).toBe('en-US-AriaNeural');
});
it("respects the user's preferred Edge voice for the language when valid", () => {
const spy = vi.spyOn(TTSUtils, 'getPreferredVoice').mockReturnValue('en-GB-RyanNeural');
expect(pickEdgeVoiceId('en')).toBe('en-GB-RyanNeural');
spy.mockRestore();
});
});
describe('pronounceWord — Edge path', () => {
it('synthesizes with Edge and schedules playback without touching the fallback', async () => {
h.createAudioData.mockResolvedValue({ data: new ArrayBuffer(8), boundaries: [] });
const onStatus = vi.fn();
await pronounceWord('hello', 'en', {}, onStatus);
expect(h.createAudioData).toHaveBeenCalledWith(
expect.objectContaining({ text: 'hello', lang: 'en', voice: 'en-US-AriaNeural' }),
);
expect(h.player.scheduleChunk).toHaveBeenCalledTimes(1);
expect(h.webSpeak).not.toHaveBeenCalled();
expect(h.nativeSpeak).not.toHaveBeenCalled();
expect(onStatus).toHaveBeenLastCalledWith('playing');
h.player.fireSessionEnd();
expect(onStatus).toHaveBeenLastCalledWith('ended');
});
it('reports an error when the audio context surfaces one mid-playback', async () => {
h.createAudioData.mockResolvedValue({ data: new ArrayBuffer(8), boundaries: [] });
const onStatus = vi.fn();
await pronounceWord('hello', 'en', {}, onStatus);
h.player.fireContextError();
expect(onStatus).toHaveBeenLastCalledWith('error');
});
});
describe('pronounceWord — fallback path', () => {
it('drops to Web Speech on desktop/web when Edge fails', async () => {
h.createAudioData.mockRejectedValue(new Error('wss blocked'));
const onStatus = vi.fn();
await pronounceWord('hello', 'en', {}, onStatus);
await flush();
expect(h.webSpeak).toHaveBeenCalledTimes(1);
const ssml = h.webSpeak.mock.calls[0]![0];
expect(ssml).toContain('hello');
expect(h.nativeSpeak).not.toHaveBeenCalled();
expect(onStatus).toHaveBeenLastCalledWith('ended');
});
it('uses the native client on the mobile app when Edge fails', async () => {
h.createAudioData.mockRejectedValue(new Error('wss blocked'));
const onStatus = vi.fn();
await pronounceWord('hello', 'en', { appService: { isMobile: true } as never }, onStatus);
await flush();
expect(h.nativeSpeak).toHaveBeenCalledTimes(1);
expect(h.webSpeak).not.toHaveBeenCalled();
});
});
describe('pronounceWord — guards', () => {
it('does nothing for a blank word', async () => {
const onStatus = vi.fn();
await pronounceWord(' ', 'en', {}, onStatus);
expect(h.createAudioData).not.toHaveBeenCalled();
expect(onStatus).toHaveBeenLastCalledWith('ended');
});
it('aborts the active session on cancel', async () => {
cancelWordPronounce();
expect(h.player.abortSession).toHaveBeenCalled();
});
});
@@ -55,6 +55,8 @@ const DictionaryPopup: React.FC<DictionaryPopupProps> = ({
canGoBack={state.canGoBack} canGoBack={state.canGoBack}
goBack={state.goBack} goBack={state.goBack}
onManage={onManage} onManage={onManage}
onSpeak={state.speakWord}
speaking={state.isSpeaking}
/> />
<div className='min-h-0 flex-1'> <div className='min-h-0 flex-1'>
<DictionaryResultsBody {...state} /> <DictionaryResultsBody {...state} />
@@ -1,7 +1,7 @@
'use client'; 'use client';
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { MdArrowBack, MdChevronRight, MdSettings } from 'react-icons/md'; import { MdArrowBack, MdChevronRight, MdSettings, MdVolumeUp } from 'react-icons/md';
import clsx from 'clsx'; import clsx from 'clsx';
import { openUrl } from '@tauri-apps/plugin-opener'; import { openUrl } from '@tauri-apps/plugin-opener';
@@ -12,6 +12,7 @@ import { useCustomDictionaryStore } from '@/store/customDictionaryStore';
import { getEnabledProviders } from '@/services/dictionaries/registry'; import { getEnabledProviders } from '@/services/dictionaries/registry';
import { buildLookupCandidates } from '@/services/dictionaries/lookupCandidates'; import { buildLookupCandidates } from '@/services/dictionaries/lookupCandidates';
import { isTauriAppPlatform } from '@/services/environment'; import { isTauriAppPlatform } from '@/services/environment';
import { cancelWordPronounce, pronounceWord, warmWordAudio } from '@/services/tts/wordPronouncer';
import { import {
getBuiltinWebSearch, getBuiltinWebSearch,
substituteUrlTemplate, substituteUrlTemplate,
@@ -51,6 +52,10 @@ export interface DictionaryResultsState {
noProvidersAtAll: boolean; noProvidersAtAll: boolean;
/** Dictionary popup font-size multiplier (#4443); `1` = default sizes. */ /** Dictionary popup font-size multiplier (#4443); `1` = default sizes. */
fontScale: number; fontScale: number;
/** Whether the current word is being fetched/spoken by the TTS engine (#4876). */
isSpeaking: boolean;
/** Pronounce the current word via Edge TTS (falling back to platform speech). */
speakWord: () => void;
} }
/** /**
@@ -124,6 +129,30 @@ export function useDictionaryResults({
setHistoryStack((prev) => (prev.length > 1 ? prev.slice(0, -1) : prev)); setHistoryStack((prev) => (prev.length > 1 ? prev.slice(0, -1) : prev));
}, []); }, []);
// Pronounce the current headword (#4876). `isSpeaking` covers both the
// Edge fetch and playback so the header button can show one active state.
const [isSpeaking, setIsSpeaking] = useState(false);
const langCode = typeof lang === 'string' ? lang : Array.isArray(lang) ? lang[0] : undefined;
const speakWord = useCallback(() => {
// Warm the audio context synchronously inside the click gesture; the
// synth/play happens after a network await, outside the gesture window.
warmWordAudio();
setIsSpeaking(true);
void pronounceWord(currentWord, langCode, { appService }, (status) => {
if (status !== 'playing') setIsSpeaking(false);
});
}, [currentWord, langCode, appService]);
// Stop any in-flight pronunciation when the word changes (in-content
// navigation / reopen) or the popup unmounts.
useEffect(() => {
return () => cancelWordPronounce();
}, []);
useEffect(() => {
cancelWordPronounce();
setIsSpeaking(false);
}, [currentWord]);
const toggleExpanded = useCallback((id: string) => { const toggleExpanded = useCallback((id: string) => {
setCards((prev) => { setCards((prev) => {
const old = prev[id]; const old = prev[id];
@@ -315,6 +344,8 @@ export function useDictionaryResults({
onWebSearchClickTauri, onWebSearchClickTauri,
noProvidersAtAll, noProvidersAtAll,
fontScale: settings.fontScale ?? 1, fontScale: settings.fontScale ?? 1,
isSpeaking,
speakWord,
}; };
} }
@@ -324,6 +355,10 @@ interface DictionaryResultsHeaderProps {
canGoBack: boolean; canGoBack: boolean;
goBack: () => void; goBack: () => void;
onManage?: () => void; onManage?: () => void;
/** Pronounce the current word (#4876); omit to hide the speaker button. */
onSpeak?: () => void;
/** Whether pronunciation is in progress, for the active button state. */
speaking?: boolean;
} }
export const DictionaryResultsHeader: React.FC<DictionaryResultsHeaderProps> = ({ export const DictionaryResultsHeader: React.FC<DictionaryResultsHeaderProps> = ({
@@ -332,6 +367,8 @@ export const DictionaryResultsHeader: React.FC<DictionaryResultsHeaderProps> = (
canGoBack, canGoBack,
goBack, goBack,
onManage, onManage,
onSpeak,
speaking,
}) => { }) => {
const _ = useTranslation(); const _ = useTranslation();
return ( return (
@@ -348,9 +385,28 @@ export const DictionaryResultsHeader: React.FC<DictionaryResultsHeaderProps> = (
</button> </button>
) : null} ) : null}
</div> </div>
<span data-testid='dict-title' className='line-clamp-1 flex-1 text-center font-bold'> <div className='flex min-w-0 flex-1 items-center justify-center gap-1'>
{currentWord} {onSpeak ? (
</span> <button
type='button'
aria-label={_('Speak')}
title={_('Speak')}
aria-pressed={!!speaking}
onClick={onSpeak}
className={clsx(
'btn btn-ghost btn-square btn-xs shrink-0',
speaking
? 'text-base-content not-eink:animate-pulse'
: 'text-base-content/60 hover:text-base-content not-eink:hover:bg-base-200/60',
)}
>
<MdVolumeUp size={18} />
</button>
) : null}
<span data-testid='dict-title' className='line-clamp-1 min-w-0 font-bold'>
{currentWord}
</span>
</div>
<div className='flex h-8 w-8 items-center justify-center'> <div className='flex h-8 w-8 items-center justify-center'>
{onManage ? ( {onManage ? (
<button <button
@@ -33,6 +33,8 @@ const DictionarySheet: React.FC<DictionarySheetProps> = ({ word, lang, onDismiss
canGoBack={state.canGoBack} canGoBack={state.canGoBack}
goBack={state.goBack} goBack={state.goBack}
onManage={onManage} onManage={onManage}
onSpeak={state.speakWord}
speaking={state.isSpeaking}
/> />
} }
contentClassName='!px-0 !mt-0' contentClassName='!px-0 !mt-0'
@@ -0,0 +1,188 @@
import { AppService } from '@/types/system';
import { EdgeSpeechTTS, EdgeTTSPayload } from '@/libs/edgeTTS';
import { isSameLang } from '@/utils/lang';
import { genSSMLRaw } from '@/utils/ssml';
import { TTSClient } from './TTSClient';
import { TTSUtils } from './TTSUtils';
import { NativeTTSClient } from './NativeTTSClient';
import { WebSpeechClient } from './WebSpeechClient';
import { WebAudioPlayer } from './WebAudioPlayer';
import type { TTSAudioContext } from './WebAudioPlayer';
// Speaks a single dictionary word as fast as possible. Unlike the reader's
// TTSController, this never runs EdgeTTSClient.init() (which wastes a round
// trip synthesizing "test") and never spins up a full speaking session — it
// calls EdgeSpeechTTS directly (whose static MP3 cache makes repeat words
// instant) and schedules one chunk on a dedicated Web Audio context. Edge is
// tried first (wss, then the authenticated https proxy); any failure drops to
// the platform speech client so a word still speaks offline. See issue #4876.
const EDGE_TTS_NAME = 'edge-tts';
const DEFAULT_EDGE_VOICE = 'en-US-AriaNeural';
export type PronounceStatus = 'playing' | 'ended' | 'error';
export interface PronounceWordOptions {
appService?: AppService | null;
}
// Choose an Edge voice for a language: the user's preferred Edge voice for that
// language (as picked in TTS settings) when it exists, else the first voice
// whose locale matches, else a safe English default.
export const pickEdgeVoiceId = (lang: string): string => {
const preferred = TTSUtils.getPreferredVoice(EDGE_TTS_NAME, lang);
const voices = EdgeSpeechTTS.voices;
if (preferred && voices.some((v) => v.id === preferred)) return preferred;
const match = voices.find((v) => isSameLang(v.lang, lang));
return match?.id ?? DEFAULT_EDGE_VOICE;
};
// A dedicated context, isolated from the reader's shared-context TTS so
// pronouncing a word can never resume/suspend or overlap an active read-aloud
// session. Created lazily on first use inside a user gesture (see warmWordAudio).
let dedicatedPlayer: WebAudioPlayer | null = null;
const getPlayer = (): WebAudioPlayer | null => {
if (typeof AudioContext === 'undefined') return null;
if (!dedicatedPlayer) {
dedicatedPlayer = new WebAudioPlayer(() => new AudioContext() as unknown as TTSAudioContext);
}
return dedicatedPlayer;
};
// Reused across calls; EdgeSpeechTTS keeps its MP3/boundary caches on static
// members, so this just avoids per-call allocation.
const edgeWss = new EdgeSpeechTTS('wss');
const edgeHttps = new EdgeSpeechTTS('https');
// Bumped on every new request so a slower in-flight synth/fetch can detect it
// has been superseded and bail before touching the player or status.
let requestToken = 0;
let fallbackAbort: AbortController | null = null;
let fallbackClient: TTSClient | null = null;
const stopFallback = (): void => {
fallbackAbort?.abort();
fallbackAbort = null;
const client = fallbackClient;
fallbackClient = null;
if (client) void client.shutdown().catch(() => {});
};
// Warm (create + resume) the dedicated audio context. MUST be called
// synchronously from the click handler: pronounceWord resumes the context only
// after a network await, outside WebKit's user-gesture window, where resume()
// is rejected by autoplay policy.
export const warmWordAudio = (): void => {
const player = getPlayer();
if (player) void player.ensureContext().catch(() => {});
};
export const cancelWordPronounce = (): void => {
requestToken++;
getPlayer()?.abortSession();
stopFallback();
};
// Edge audio bytes: direct wss first; on failure the authenticated https proxy
// (the reader's own fallback for networks that block Bing). The proxy throws
// "Not authenticated" when logged out, which propagates to the speech fallback.
const fetchEdgeAudio = async (payload: EdgeTTSPayload): Promise<ArrayBuffer> => {
try {
return (await edgeWss.createAudioData(payload)).data;
} catch {
return (await edgeHttps.createAudioData(payload)).data;
}
};
const speakViaFallback = async (
word: string,
lang: string,
options: PronounceWordOptions,
token: number,
emit: (status: PronounceStatus) => void,
): Promise<void> => {
// Web Speech is the reader's built-in engine on desktop/web; on the mobile
// app the native TTS plugin is what actually produces audio.
const client: TTSClient = options.appService?.isMobile
? new NativeTTSClient()
: new WebSpeechClient();
fallbackClient = client;
const controller = new AbortController();
fallbackAbort = controller;
try {
const ready = await client.init();
if (!ready || token !== requestToken) {
emit('error');
return;
}
client.setPrimaryLang(lang);
emit('playing');
for await (const ev of client.speak(genSSMLRaw(word), controller.signal)) {
if (ev.code === 'error') {
emit('error');
return;
}
}
emit('ended');
} catch {
emit('error');
} finally {
if (fallbackClient === client) fallbackClient = null;
if (fallbackAbort === controller) fallbackAbort = null;
void client.shutdown().catch(() => {});
}
};
export const pronounceWord = async (
word: string,
lang: string | undefined,
options: PronounceWordOptions,
onStatus?: (status: PronounceStatus) => void,
): Promise<void> => {
const token = ++requestToken;
const emit = (status: PronounceStatus) => {
if (token === requestToken) onStatus?.(status);
};
const trimmed = word.trim();
if (!trimmed) {
emit('ended');
return;
}
const voiceLang = lang && lang.length ? lang : 'en';
// Stop whatever is currently playing (Edge session or fallback client).
getPlayer()?.abortSession();
stopFallback();
const player = getPlayer();
if (player) {
try {
const voice = pickEdgeVoiceId(voiceLang);
const data = await fetchEdgeAudio({
lang: voiceLang,
text: trimmed,
voice,
rate: 1.0,
pitch: 1.0,
});
if (token !== requestToken) return;
const buffer = await player.decode(data);
if (token !== requestToken) return;
const generation = player.startSession((event) => {
if (event.type === 'session-end') emit('ended');
else if (event.type === 'context-error') emit('error');
});
player.scheduleChunk(generation, buffer, { trimStartSec: 0, mediaScale: 1, gapSec: 0 });
player.endSession(generation);
emit('playing');
return;
} catch (err) {
if (token !== requestToken) return;
console.warn('[dict-tts] Edge pronunciation failed, falling back', err);
}
}
if (token !== requestToken) return;
await speakViaFallback(trimmed, voiceLang, options, token, emit);
};