From 8dc04f4095a756bfe94540c25c50e7bc6a011e77 Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Sat, 24 May 2025 00:18:24 +0800 Subject: [PATCH] feat: add bilingual TTS support for bilingual books, closes #1226 (#1230) --- apps/readest-app/src/libs/edgeTTS.ts | 9 +- .../src/services/tts/EdgeTTSClient.ts | 54 +++++--- .../src/services/tts/TTSController.ts | 5 +- .../src/services/tts/WebSpeechClient.ts | 130 ++++-------------- apps/readest-app/src/utils/ssml.ts | 74 +++++++--- 5 files changed, 122 insertions(+), 150 deletions(-) diff --git a/apps/readest-app/src/libs/edgeTTS.ts b/apps/readest-app/src/libs/edgeTTS.ts index 44a98499..7c809272 100644 --- a/apps/readest-app/src/libs/edgeTTS.ts +++ b/apps/readest-app/src/libs/edgeTTS.ts @@ -174,11 +174,12 @@ export class EdgeSpeechTTS { }); const genSSML = (lang: string, text: string, voice: string, rate: number) => { + const cleanedText = text.replace(/^]*>/i, ''); return ` - ${text} + ${cleanedText} @@ -255,6 +256,12 @@ export class EdgeSpeechTTS { } }); + ws.addEventListener('close', () => { + if (!audioData.byteLength) { + reject(new Error('No audio data received.')); + } + }); + ws.addEventListener('error', () => { ws.close(); reject(new Error('WebSocket error occurred.')); diff --git a/apps/readest-app/src/services/tts/EdgeTTSClient.ts b/apps/readest-app/src/services/tts/EdgeTTSClient.ts index 60234b37..b949d586 100644 --- a/apps/readest-app/src/services/tts/EdgeTTSClient.ts +++ b/apps/readest-app/src/services/tts/EdgeTTSClient.ts @@ -10,7 +10,6 @@ export class EdgeTTSClient implements TTSClient { #rate = 1.0; #pitch = 1.0; #voice: TTSVoice | null = null; - #currentVoiceLang = ''; #voices: TTSVoice[] = []; #edgeTTS: EdgeSpeechTTS; @@ -45,25 +44,26 @@ export class EdgeTTSClient implements TTSClient { return { lang, text, voice: voiceId, rate: this.#rate, pitch: this.#pitch } as EdgeTTSPayload; }; + getVoiceIdFromLang = async (lang: string) => { + let voiceId = 'en-US-AriaNeural'; + const preferredVoiceId = TTSUtils.getPreferredVoice('edge-tts', lang); + const preferredVoice = this.#voices.find((v) => v.id === preferredVoiceId); + this.#voice = preferredVoice ? preferredVoice : (await this.getVoices(lang))[0] || null; + if (this.#voice) { + voiceId = this.#voice.id; + } + return voiceId; + }; + async *speak( ssml: string, signal: AbortSignal, preload = false, ): AsyncGenerator { const { marks } = parseSSMLMarks(ssml); - let lang = parseSSMLLang(ssml) || 'en'; - if (lang === 'en' && this.#primaryLang && this.#primaryLang !== 'en') { - lang = this.#primaryLang; - } - let voiceId = 'en-US-AriaNeural'; - if (!this.#voice || this.#currentVoiceLang !== lang) { - const preferredVoiceId = TTSUtils.getPreferredVoice('edge-tts', lang); - const preferredVoice = this.#voices.find((v) => v.id === preferredVoiceId); - this.#voice = preferredVoice ? preferredVoice : (await this.getVoices(lang))[0] || null; - this.#currentVoiceLang = lang; - } - if (this.#voice) { - voiceId = this.#voice.id; + let defaultLang = parseSSMLLang(ssml) || 'en'; + if (defaultLang === 'en' && this.#primaryLang && this.#primaryLang !== 'en') { + defaultLang = this.#primaryLang; } if (preload) { @@ -71,16 +71,24 @@ export class EdgeTTSClient implements TTSClient { const maxImmediate = 2; for (let i = 0; i < Math.min(maxImmediate, marks.length); i++) { const mark = marks[i]!; - await this.#edgeTTS.createAudio(this.getPayload(lang, mark.text, voiceId)).catch((err) => { - console.warn('Error preloading mark', i, err); - }); + const { language } = mark; + const voiceLang = language || defaultLang; + const voiceId = await this.getVoiceIdFromLang(voiceLang); + await this.#edgeTTS + .createAudio(this.getPayload(voiceLang, mark.text, voiceId)) + .catch((err) => { + console.warn('Error preloading mark', i, err); + }); } if (marks.length > maxImmediate) { (async () => { for (let i = maxImmediate; i < marks.length; i++) { const mark = marks[i]!; try { - await this.#edgeTTS.createAudio(this.getPayload(lang, mark.text, voiceId)); + const { language } = mark; + const voiceLang = language || defaultLang; + const voiceId = await this.getVoiceIdFromLang(voiceLang); + await this.#edgeTTS.createAudio(this.getPayload(voiceLang, mark.text, voiceId)); } catch (err) { console.warn('Error preloading mark (bg)', i, err); } @@ -107,7 +115,12 @@ export class EdgeTTSClient implements TTSClient { break; } try { - const blob = await this.#edgeTTS.createAudio(this.getPayload(lang, mark.text, voiceId)); + const { language } = mark; + const voiceLang = language || defaultLang; + const voiceId = await this.getVoiceIdFromLang(voiceLang); + const blob = await this.#edgeTTS.createAudio( + this.getPayload(voiceLang, mark.text, voiceId), + ); const url = URL.createObjectURL(blob); this.#audioElement = new Audio(url); const audio = this.#audioElement; @@ -240,9 +253,6 @@ export class EdgeTTSClient implements TTSClient { } async getVoices(lang: string): Promise { - if (this.#currentVoiceLang) { - lang = this.#currentVoiceLang; - } const locale = lang === 'en' ? getUserLocale(lang) || lang : lang; const voices = await this.getAllVoices(); return voices diff --git a/apps/readest-app/src/services/tts/TTSController.ts b/apps/readest-app/src/services/tts/TTSController.ts index 2f6e33c7..c18c7991 100644 --- a/apps/readest-app/src/services/tts/TTSController.ts +++ b/apps/readest-app/src/services/tts/TTSController.ts @@ -82,8 +82,9 @@ export class TTSController extends EventTarget { if (!ssml) return; ssml = ssml .replace(/[–—]/g, ',') - .replace(/\.{3,}/g, '') - .replace(/·/g, ''); + .replace('', ' ') + .replace(/\.{3,}/g, ' ') + .replace(/·/g, ' '); return ssml; } diff --git a/apps/readest-app/src/services/tts/WebSpeechClient.ts b/apps/readest-app/src/services/tts/WebSpeechClient.ts index 9ee4f070..52faf3f0 100644 --- a/apps/readest-app/src/services/tts/WebSpeechClient.ts +++ b/apps/readest-app/src/services/tts/WebSpeechClient.ts @@ -1,8 +1,6 @@ import { getUserLocale } from '@/utils/misc'; import { TTSClient, TTSMessageEvent, TTSVoice } from './TTSClient'; -import { AsyncQueue } from '@/utils/queue'; -import { findSSMLMark, parseSSMLLang, parseSSMLMarks } from '@/utils/ssml'; -import { isCJKStr } from '@/utils/lang'; +import { parseSSMLLang, parseSSMLMarks } from '@/utils/ssml'; import { TTSGranularity } from '@/types/view'; import { TTSUtils } from './TTSUtils'; @@ -46,105 +44,30 @@ interface TTSBoundaryEvent { error?: string; } -async function* speakWithBoundary( - ssml: string, - getRate: () => number, - getPitch: () => number, - getVoice: () => SpeechSynthesisVoice | null, -) { - const lang = parseSSMLLang(ssml); - const { plainText, marks } = parseSSMLMarks(ssml); - // console.log('ssml', ssml, marks); - // console.log('text', plainText); - - const synth = window.speechSynthesis; - const utterance = new SpeechSynthesisUtterance(plainText); - - utterance.rate = getRate(); - utterance.pitch = getPitch(); - const voice = getVoice(); - if (voice) { - utterance.voice = voice; - } - if (lang) { - utterance.lang = lang; - } - - const queue = new AsyncQueue(); - - utterance.onboundary = (event: SpeechSynthesisEvent) => { - utterance.rate = getRate(); - utterance.pitch = getPitch(); - const voice = getVoice(); - if (voice) { - utterance.voice = voice; - } - const mark = findSSMLMark(event.charIndex, marks); - // console.log('boundary', event.charIndex, mark); - queue.enqueue({ - type: 'boundary', - speaking: true, - name: event.name, - mark: mark?.name ?? '', - charIndex: event.charIndex, - charLength: event.charLength, - }); - }; - - utterance.onend = () => { - queue.enqueue({ type: 'end', speaking: false }); - queue.finish(); - }; - - utterance.onerror = (event) => { - queue.enqueue({ type: 'error', speaking: false, error: event.error }); - queue.finish(); - }; - - synth.speak(utterance); - - while (true) { - const ev = await queue.dequeue(); - if (ev === null) { - break; - } - yield ev; - } -} - async function* speakWithMarks( ssml: string, + defaultLang: string, getRate: () => number, getPitch: () => number, - getVoice: () => SpeechSynthesisVoice | null, + getVoice: (lang: string) => Promise, ) { - const { plainText, marks } = parseSSMLMarks(ssml); - const lang = parseSSMLLang(ssml); - - const isCJK = (lang: string | null) => { - const cjkLangs = ['zh', 'ja', 'kr']; - if (lang && cjkLangs.some((cjk) => lang.startsWith(cjk))) return true; - return isCJKStr(plainText); - }; - - if (!isCJK(lang)) { - yield* speakWithBoundary(ssml, getRate, getPitch, getVoice); - return; - } + const { marks } = parseSSMLMarks(ssml); const synth = window.speechSynthesis; const utterance = new SpeechSynthesisUtterance(); for (const mark of marks) { + const { language } = mark; + const voiceLang = language || defaultLang; utterance.text = mark.text; utterance.rate = getRate(); utterance.pitch = getPitch(); - const voice = getVoice(); + const voice = await getVoice(voiceLang); if (voice) { utterance.voice = voice; } - if (lang) { - utterance.lang = lang; + if (voiceLang) { + utterance.lang = voiceLang; } yield { @@ -178,7 +101,6 @@ export class WebSpeechClient implements TTSClient { #rate = 1.0; #pitch = 1.0; #voice: SpeechSynthesisVoice | null = null; - #currentVoiceLang = ''; #voices: SpeechSynthesisVoice[] = []; #synth = window.speechSynthesis; available = true; @@ -209,6 +131,18 @@ export class WebSpeechClient implements TTSClient { return this.available; } + getVoiceFromLang = async (lang: string) => { + const preferredVoiceId = TTSUtils.getPreferredVoice('web-speech', lang); + const preferredVoice = this.#voices.find((v) => v.voiceURI === preferredVoiceId); + if (preferredVoice) { + this.#voice = preferredVoice; + } else { + const voiceId = (await this.getVoices(lang))[0]?.id ?? ''; + this.#voice = this.#voices.find((v) => v.voiceURI === voiceId) || null; + } + return this.#voice; + }; + async *speak( ssml: string, signal: AbortSignal, @@ -217,24 +151,17 @@ export class WebSpeechClient implements TTSClient { // no need to preload for web speech if (preload) return; - let lang = parseSSMLLang(ssml) || 'en'; - if (lang === 'en' && this.#primaryLang && this.#primaryLang !== 'en') { - lang = this.#primaryLang; - } - if (!this.#voice || this.#currentVoiceLang !== lang) { - const preferredVoiceId = TTSUtils.getPreferredVoice('web-speech', lang); - const preferredVoice = this.#voices.find((v) => v.voiceURI === preferredVoiceId); - const voiceId = (await this.getVoices(lang))[0]?.id ?? ''; - this.#voice = preferredVoice - ? preferredVoice - : this.#voices.find((v) => v.voiceURI === voiceId) || null; - this.#currentVoiceLang = lang; + let defaultLang = parseSSMLLang(ssml) || 'en'; + if (defaultLang === 'en' && this.#primaryLang && this.#primaryLang !== 'en') { + defaultLang = this.#primaryLang; } + for await (const ev of speakWithMarks( ssml, + defaultLang, () => this.#rate, () => this.#pitch, - () => this.#voice, + this.getVoiceFromLang, )) { if (signal.aborted) { console.log('TTS aborted'); @@ -301,9 +228,6 @@ export class WebSpeechClient implements TTSClient { } async getVoices(lang: string) { - if (this.#currentVoiceLang) { - lang = this.#currentVoiceLang; - } const locale = lang === 'en' ? getUserLocale(lang) || lang : lang; const isValidVoice = (id: string) => { return !id.includes('com.apple') || id.includes('com.apple.voice.compact'); diff --git a/apps/readest-app/src/utils/ssml.ts b/apps/readest-app/src/utils/ssml.ts index 0ecb25df..90237132 100644 --- a/apps/readest-app/src/utils/ssml.ts +++ b/apps/readest-app/src/utils/ssml.ts @@ -2,35 +2,65 @@ type TTSMark = { offset: number; name: string; text: string; + language?: string; }; -export const parseSSMLMarks = (ssml: string) => { - ssml = ssml.replace(/]*>/i, ''); - ssml = ssml.replace(/<\/speak>/i, ''); +const cleanTextContent = (text: string) => + text.replace(/\r\n/g, ' ').replace(/\r/g, ' ').replace(/\n/g, ' ').trimStart(); + +export const parseSSMLMarks = (ssml: string) => { + const speakMatch = ssml.match(/]*xml:lang="([^"]+)"[^>]*>/i); + const defaultLang = speakMatch?.[1] ?? 'en'; + ssml = ssml.replace(/]*>/i, '').replace(/<\/speak>/i, ''); - const markRegex = //g; let plainText = ''; const marks: TTSMark[] = []; - let match; - while ((match = markRegex.exec(ssml)) !== null) { - const markTagEndIndex = markRegex.lastIndex; - const nextMarkIndex = ssml.indexOf(']+>/g, '') - .replace(/\r\n/g, ' ') - .replace(/\r/g, ' ') - .replace(/\n/g, ' ') - .trimStart(); - plainText += cleanedChunk; + let activeMark: string | null = null; + let currentLang = defaultLang; + const langStack: string[] = []; - const offset = plainText.length - cleanedChunk.length; - const markName = match[1]!; - marks.push({ offset, name: markName, text: cleanedChunk }); + const tagRegex = /<(\/?)(\w+)([^>]*)>|([^<]+)/g; + + let match: RegExpExecArray | null; + while ((match = tagRegex.exec(ssml)) !== null) { + if (match[4]) { + const rawText = match[4]; + const text = cleanTextContent(rawText); + if (text && activeMark) { + const offset = plainText.length; + plainText += text; + marks.push({ + offset, + name: activeMark, + text, + language: currentLang, + }); + } else { + plainText += cleanTextContent(rawText); + } + } else { + const isEnd = match[1] === '/'; + const tagName = match[2]; + const attr = match[3]; + + if (tagName === 'mark' && !isEnd) { + const nameMatch = attr?.match(/name="([^"]+)"/); + if (nameMatch) { + activeMark = nameMatch[1]!; + } + } else if (tagName === 'lang') { + if (!isEnd) { + langStack.push(currentLang); + const langMatch = attr?.match(/xml:lang="([^"]+)"/); + if (langMatch) { + currentLang = langMatch[1]!; + } + } else { + currentLang = langStack.pop() ?? defaultLang; + } + } + } } return { plainText, marks };