From 00003a9415b8dcd529e5517e70b9eaf43346bb6f Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Fri, 10 Jan 2025 14:11:09 +0100 Subject: [PATCH] Preloading audio for Edge TTS and other fixes (#138) * Blacklist some low-quality web tts voices * Dropdown layout tweaks * Handle no audio data received in Edge TTS * Preloading audio for Edge TTS --- apps/readest-app/src-tauri/tauri.conf.json | 2 +- .../components/settings/FontDropDown.tsx | 2 +- .../app/reader/components/tts/TTSControl.tsx | 16 ++++++ .../app/reader/components/tts/TTSPanel.tsx | 10 +++- apps/readest-app/src/libs/edgeTTS.ts | 35 +++++++++--- .../src/services/tts/EdgeTTSClient.ts | 49 ++++++++++++----- .../readest-app/src/services/tts/TTSClient.ts | 1 + .../src/services/tts/TTSController.ts | 17 ++++-- .../src/services/tts/WebSpeechClient.ts | 48 ++++++++++++++++- apps/readest-app/src/utils/lru.ts | 54 +++++++++++++++++++ apps/readest-app/src/utils/ssml.ts | 7 ++- 11 files changed, 210 insertions(+), 31 deletions(-) create mode 100644 apps/readest-app/src/utils/lru.ts diff --git a/apps/readest-app/src-tauri/tauri.conf.json b/apps/readest-app/src-tauri/tauri.conf.json index 6a17c3d8..6c6f05db 100644 --- a/apps/readest-app/src-tauri/tauri.conf.json +++ b/apps/readest-app/src-tauri/tauri.conf.json @@ -15,7 +15,7 @@ "security": { "csp": { "default-src": "'self' 'unsafe-inline' blob: customprotocol: asset: http://asset.localhost ipc: http://ipc.localhost https://fonts.gstatic.com https://db.onlinewebfonts.com", - "connect-src": "'self' blob: asset: http://asset.localhost ipc: http://ipc.localhost https://*.sentry.io https://*.posthog.com https://*.deepl.com https://*.wikipedia.org https://*.wiktionary.org https://*.supabase.co https://*.readest.com", + "connect-src": "'self' blob: asset: http://asset.localhost ipc: http://ipc.localhost https://*.sentry.io https://*.posthog.com https://*.deepl.com https://*.wikipedia.org https://*.wiktionary.org https://*.supabase.co https://*.readest.com wss://speech.platform.bing.com", "img-src": "'self' blob: data: asset: http://asset.localhost https://*", "style-src": "'self' 'unsafe-inline' blob: asset: http://asset.localhost", "frame-src": "'self' blob: asset: http://asset.localhost", diff --git a/apps/readest-app/src/app/reader/components/settings/FontDropDown.tsx b/apps/readest-app/src/app/reader/components/settings/FontDropDown.tsx index 947b21e3..849dd657 100644 --- a/apps/readest-app/src/app/reader/components/settings/FontDropDown.tsx +++ b/apps/readest-app/src/app/reader/components/settings/FontDropDown.tsx @@ -62,7 +62,7 @@ const FontDropdown: React.FC = ({ > {moreOptions.map((option) => (
  • onSelect(option)}> -
    +
    {selected === option && } diff --git a/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx b/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx index d99cf4e1..f443498f 100644 --- a/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx +++ b/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx @@ -54,6 +54,11 @@ const TTSControl = () => { setBookKey(bookKey); + if (ttsControllerRef.current) { + ttsControllerRef.current.stop(); + ttsControllerRef.current = null; + } + try { const ttsController = new TTSController(view); await ttsController.init(); @@ -94,6 +99,8 @@ const TTSControl = () => { setIsPlaying(false); setIsPaused(true); } else if (isPaused) { + // start for forward/backward/setvoice-paused + // set rate don't pause the tts if (ttsController.state === 'paused') { ttsController.resume(); } else { @@ -163,6 +170,14 @@ const TTSControl = () => { return []; }; + const handleGetVoiceId = () => { + const ttsController = ttsControllerRef.current; + if (ttsController) { + return ttsController.getVoiceId(); + } + return ''; + }; + const updatePanelPosition = () => { if (iconRef.current) { const rect = iconRef.current.getBoundingClientRect(); @@ -228,6 +243,7 @@ const TTSControl = () => { onSetRate={handleSetRate} onGetVoices={handleGetVoices} onSetVoice={handleSetVoice} + onGetVoiceId={handleGetVoiceId} /> )} diff --git a/apps/readest-app/src/app/reader/components/tts/TTSPanel.tsx b/apps/readest-app/src/app/reader/components/tts/TTSPanel.tsx index bd7eb859..c88c5042 100644 --- a/apps/readest-app/src/app/reader/components/tts/TTSPanel.tsx +++ b/apps/readest-app/src/app/reader/components/tts/TTSPanel.tsx @@ -18,6 +18,7 @@ type TTSPanelProps = { onSetRate: (rate: number) => void; onGetVoices: (lang: string) => Promise; onSetVoice: (voice: string) => void; + onGetVoiceId: () => string; }; const TTSPanel = ({ @@ -31,6 +32,7 @@ const TTSPanel = ({ onSetRate, onGetVoices, onSetVoice, + onGetVoiceId, }: TTSPanelProps) => { const _ = useTranslation(); const { getViewSettings, setViewSettings } = useReaderStore(); @@ -58,6 +60,12 @@ const TTSPanel = ({ setViewSettings(bookKey, viewSettings); }; + useEffect(() => { + const voiceId = onGetVoiceId(); + setSelectedVoice(voiceId); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + useEffect(() => { const fetchVoices = async () => { const voices = await onGetVoices(ttsLang); @@ -131,7 +139,7 @@ const TTSPanel = ({ key={`${index}-${voice.id}`} onClick={() => !voice.disabled && handleSelectVoice(voice.id)} > -
    +
    {selectedVoice === voice.id && ( diff --git a/apps/readest-app/src/libs/edgeTTS.ts b/apps/readest-app/src/libs/edgeTTS.ts index 9f1ac718..9d594b9a 100644 --- a/apps/readest-app/src/libs/edgeTTS.ts +++ b/apps/readest-app/src/libs/edgeTTS.ts @@ -1,4 +1,6 @@ +import { md5 } from 'js-md5'; import { randomMd5 } from '@/utils/misc'; +import { LRUCache } from '@/utils/lru'; const EDGE_SPEECH_URL = 'wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud/edge/v1'; @@ -46,18 +48,25 @@ const genVoiceList = (voices: Record) => { }; export interface EdgeTTSPayload { + lang: string; text: string; voice: string; rate: number; pitch: number; } +const hashPayload = (payload: EdgeTTSPayload): string => { + const base = JSON.stringify(payload); + return md5(base); +}; + export class EdgeSpeechTTS { static voices = genVoiceList(EDGE_TTS_VOICES); + private static audioCache = new LRUCache(200); constructor() {} - async #fetchEdgeSpeechWs({ text, voice, rate }: EdgeTTSPayload): Promise { + async #fetchEdgeSpeechWs({ lang, text, voice, rate }: EdgeTTSPayload): Promise { const connectId = randomMd5(); const url = `${EDGE_SPEECH_URL}?ConnectionId=${connectId}&TrustedClientToken=${EDGE_API_TOKEN}`; const date = new Date().toString(); @@ -83,9 +92,9 @@ export class EdgeSpeechTTS { }, }); - const genSSML = (text: string, voice: string, rate: number) => { + const genSSML = (lang: string, text: string, voice: string, rate: number) => { return ` - + ${text} @@ -126,7 +135,7 @@ export class EdgeSpeechTTS { return { headers, body }; }; - const ssml = genSSML(text, voice, rate); + const ssml = genSSML(lang, text, voice, rate); const content = genSendContent(contentHeaders, ssml); const config = genSendContent(configHeaders, configContent); @@ -177,9 +186,19 @@ export class EdgeSpeechTTS { } async createAudio(payload: EdgeTTSPayload): Promise { - const res = await this.create(payload); - const arrayBuffer = await res.arrayBuffer(); - const audioContext = new AudioContext(); - return await audioContext.decodeAudioData(arrayBuffer.slice(0)); + const cacheKey = hashPayload(payload); + if (EdgeSpeechTTS.audioCache.has(cacheKey)) { + return EdgeSpeechTTS.audioCache.get(cacheKey)!; + } + try { + const res = await this.create(payload); + const arrayBuffer = await res.arrayBuffer(); + const audioContext = new AudioContext(); + const audioBuffer = await audioContext.decodeAudioData(arrayBuffer.slice(0)); + EdgeSpeechTTS.audioCache.set(cacheKey, audioBuffer); + return audioBuffer; + } catch (error) { + throw error; + } } } diff --git a/apps/readest-app/src/services/tts/EdgeTTSClient.ts b/apps/readest-app/src/services/tts/EdgeTTSClient.ts index 0f7e59c4..b429c9b1 100644 --- a/apps/readest-app/src/services/tts/EdgeTTSClient.ts +++ b/apps/readest-app/src/services/tts/EdgeTTSClient.ts @@ -1,6 +1,6 @@ import { getUserLocale } from '@/utils/misc'; import { TTSClient, TTSMessageEvent, TTSVoice } from './TTSClient'; -import { EdgeSpeechTTS } from '@/libs/edgeTTS'; +import { EdgeSpeechTTS, EdgeTTSPayload } from '@/libs/edgeTTS'; import { parseSSMLLang, parseSSMLMarks } from '@/utils/ssml'; import { TTSGranularity } from '@/types/view'; @@ -27,6 +27,7 @@ export class EdgeTTSClient implements TTSClient { this.#voices = EdgeSpeechTTS.voices; try { await this.#edgeTTS.create({ + lang: 'en', text: 'test', voice: 'en-US-AriaNeural', rate: 1.0, @@ -39,29 +40,39 @@ export class EdgeTTSClient implements TTSClient { return this.available; } + getPayload = (lang: string, text: string, voiceId: string) => { + return { lang, text, voice: voiceId, rate: this.#rate, pitch: this.#pitch } as EdgeTTSPayload; + }; + async *speak(ssml: string): AsyncGenerator { const { marks } = parseSSMLMarks(ssml); const lang = parseSSMLLang(ssml) || 'en'; let voiceId = 'en-US-AriaNeural'; + if (!this.#voice) { + const voices = await this.getVoices(lang); + this.#voice = voices[0] ? voices[0] : this.#voices.find((v) => v.id === voiceId) || null; + } if (this.#voice) { voiceId = this.#voice.id; - } else { - const voices = await this.getVoices(lang); - voiceId = voices[0]?.id || voiceId; } this.stopInternal(); + // Preloading for longer ssml + if (marks.length > 1) { + for (const mark of marks.slice(1)) { + this.#edgeTTS.createAudio(this.getPayload(lang, mark.text, voiceId)).catch((error) => { + console.warn('Error preloading mark:', mark, error); + }); + } + } + for (const mark of marks) { try { - this.#audioBuffer = await this.#edgeTTS.createAudio({ - text: mark.text.replace(/\r?\n/g, ''), - voice: voiceId, - rate: this.#rate, - pitch: this.#pitch, - }); - + this.#audioBuffer = await this.#edgeTTS.createAudio( + this.getPayload(lang, mark.text, voiceId), + ); this.#audioContext = new AudioContext(); this.#sourceNode = this.#audioContext.createBufferSource(); this.#sourceNode.buffer = this.#audioBuffer; @@ -89,10 +100,16 @@ export class EdgeTTSClient implements TTSClient { this.#startedAt = this.#audioContext.currentTime; }); yield result; - if (result.code === 'error') { - break; - } } catch (error) { + if (error instanceof Error && error.message === 'No audio data received.') { + console.warn('No audio data received for:', mark.text); + yield { + code: 'end', + message: `Chunk finished: ${mark.name}`, + }; + continue; + } + console.log('Error:', error); yield { code: 'error', message: error instanceof Error ? error.message : String(error), @@ -175,4 +192,8 @@ export class EdgeTTSClient implements TTSClient { getGranularities(): TTSGranularity[] { return ['sentence']; } + + getVoiceId(): string { + return this.#voice?.id || ''; + } } diff --git a/apps/readest-app/src/services/tts/TTSClient.ts b/apps/readest-app/src/services/tts/TTSClient.ts index 4aa18b08..d20f4682 100644 --- a/apps/readest-app/src/services/tts/TTSClient.ts +++ b/apps/readest-app/src/services/tts/TTSClient.ts @@ -27,4 +27,5 @@ export interface TTSClient { getAllVoices(): Promise; getVoices(lang: string): Promise; getGranularities(): TTSGranularity[]; + getVoiceId(): string; } diff --git a/apps/readest-app/src/services/tts/TTSController.ts b/apps/readest-app/src/services/tts/TTSController.ts index da751571..eeec956d 100644 --- a/apps/readest-app/src/services/tts/TTSController.ts +++ b/apps/readest-app/src/services/tts/TTSController.ts @@ -3,7 +3,13 @@ import { TTSClient, TTSMessageCode, TTSVoice } from './TTSClient'; import { WebSpeechClient } from './WebSpeechClient'; import { EdgeTTSClient } from './EdgeTTSClient'; -type TTSState = 'stopped' | 'playing' | 'paused' | 'backward-paused' | 'forward-paused'; +type TTSState = + | 'stopped' + | 'playing' + | 'paused' + | 'backward-paused' + | 'forward-paused' + | 'setvoice-paused'; export class TTSController extends EventTarget { state: TTSState = 'stopped'; @@ -53,7 +59,7 @@ export class TTSController extends EventTarget { if (!ssml) { this.#nossmlCnt++; // FIXME: in case we are at the end of the book, need a better way to handle this - if (this.#nossmlCnt < 10) { + if (this.#nossmlCnt < 10 && this.state === 'playing') { await this.view.next(1); this.forward(); } @@ -71,7 +77,7 @@ export class TTSController extends EventTarget { lastCode = code; } - if (lastCode === 'end') { + if (lastCode === 'end' && this.state === 'playing') { this.forward(); } } @@ -147,6 +153,7 @@ export class TTSController extends EventTarget { } async setVoice(voiceId: string) { + this.state = 'setvoice-paused'; this.ttsClient.stop(); if (this.ttsEdgeVoices.find((voice) => voice.id === voiceId && !voice.disabled)) { this.ttsClient = this.ttsEdgeClient; @@ -156,6 +163,10 @@ export class TTSController extends EventTarget { await this.ttsClient.setVoice(voiceId); } + getVoiceId() { + return this.ttsClient.getVoiceId(); + } + error(e: unknown) { console.error(e); this.state = 'stopped'; diff --git a/apps/readest-app/src/services/tts/WebSpeechClient.ts b/apps/readest-app/src/services/tts/WebSpeechClient.ts index 11ce38d6..9b5e8a56 100644 --- a/apps/readest-app/src/services/tts/WebSpeechClient.ts +++ b/apps/readest-app/src/services/tts/WebSpeechClient.ts @@ -4,6 +4,36 @@ import { AsyncQueue } from '@/utils/queue'; import { findSSMLMark, parseSSMLLang, parseSSMLMarks } from '@/utils/ssml'; import { TTSGranularity } from '@/types/view'; +const BLACKLISTED_VOICES = [ + 'Albert', + 'Bad News', + 'Bahh', + 'Bells', + 'Boing', + 'Bubbles', + 'Cellos', + 'Eddy', + 'Flo', + 'Fred', + 'Good News', + 'Grandma', + 'Grandpa', + 'Jester', + 'Junior', + 'Kathy', + 'Organ', + 'Ralph', + 'Reed', + 'Rocko', + 'Sandy', + 'Shelley', + 'Superstar', + 'Trinoids', + 'Whisper', + 'Wobble', + 'Zarvox', +]; + interface TTSBoundaryEvent { type: 'boundary' | 'end' | 'error'; speaking: boolean; @@ -176,6 +206,12 @@ export class WebSpeechClient implements TTSClient { } async *speak(ssml: string): AsyncGenerator { + const lang = parseSSMLLang(ssml) || 'en'; + if (!this.#voice) { + const voices = await this.getVoices(lang); + const voiceId = voices[0]?.id ?? ''; + this.#voice = this.#voices.find((v) => v.voiceURI === voiceId) || null; + } for await (const ev of speakWithMarks( ssml, () => this.#rate, @@ -240,11 +276,15 @@ export class WebSpeechClient implements TTSClient { async getVoices(lang: string) { const locale = lang === 'en' ? getUserLocale(lang) || lang : lang; const isValidVoice = (id: string) => { - return !id.includes('com.apple') || id.includes('com.apple.voice'); + return !id.includes('com.apple') || id.includes('com.apple.voice.compact'); + }; + const isNotBlacklisted = (voice: SpeechSynthesisVoice) => { + return BLACKLISTED_VOICES.some((name) => voice.name.includes(name)) === false; }; const filteredVoices = this.#voices .filter((voice) => voice.lang.startsWith(locale)) - .filter((voice) => isValidVoice(voice.voiceURI || '')); + .filter((voice) => isValidVoice(voice.voiceURI || '')) + .filter(isNotBlacklisted); const voices = filteredVoices.map((voice) => { return { id: voice.voiceURI, name: voice.name, lang: voice.lang } as TTSVoice; }); @@ -259,4 +299,8 @@ export class WebSpeechClient implements TTSClient { // in the middle of speech is not possible for different granularities return ['sentence']; } + + getVoiceId(): string { + return this.#voice?.voiceURI ?? ''; + } } diff --git a/apps/readest-app/src/utils/lru.ts b/apps/readest-app/src/utils/lru.ts new file mode 100644 index 00000000..de0b1a2e --- /dev/null +++ b/apps/readest-app/src/utils/lru.ts @@ -0,0 +1,54 @@ +export class LRUCache { + private capacity: number; + private map: Map; + + constructor(capacity: number) { + if (capacity <= 0) { + throw new Error('LRUCache capacity must be greater than 0'); + } + this.capacity = capacity; + this.map = new Map(); + } + + get(key: K): V | undefined { + if (!this.map.has(key)) { + return undefined; + } + const value = this.map.get(key)!; + this.map.delete(key); + this.map.set(key, value); + return value; + } + + set(key: K, value: V): void { + if (this.map.has(key)) { + this.map.delete(key); + } else if (this.map.size === this.capacity) { + const oldestKey = this.map.keys().next().value; + if (oldestKey) { + this.map.delete(oldestKey); + } + } + this.map.set(key, value); + } + + has(key: K): boolean { + return this.map.has(key); + } + + delete(key: K): boolean { + return this.map.delete(key); + } + + clear(): void { + this.map.clear(); + } + + size(): number { + return this.map.size; + } + + entries(): Array<[K, V]> { + return Array.from(this.map).reverse(); + } +} diff --git a/apps/readest-app/src/utils/ssml.ts b/apps/readest-app/src/utils/ssml.ts index d22e1a2c..d8c97b1c 100644 --- a/apps/readest-app/src/utils/ssml.ts +++ b/apps/readest-app/src/utils/ssml.ts @@ -20,7 +20,12 @@ export const parseSSMLMarks = (ssml: string) => { markTagEndIndex, nextMarkIndex !== -1 ? nextMarkIndex : ssml.length, ); - const cleanedChunk = nextChunk.replace(/<[^>]+>/g, '').trimStart(); + const cleanedChunk = nextChunk + .replace(/<[^>]+>/g, '') + .replace(/\r\n/g, ' ') + .replace(/\r/g, ' ') + .replace(/\n/g, ' ') + .trimStart(); plainText += cleanedChunk; const offset = plainText.length - cleanedChunk.length;