feat: add bilingual TTS support for bilingual books, closes #1226 (#1230)

apps/readest-app/src/libs/edgeTTS.ts

@@ -174,11 +174,12 @@ export class EdgeSpeechTTS {
     });
 
     const genSSML = (lang: string, text: string, voice: string, rate: number) => {
+      const cleanedText = text.replace(/^<break\b[^>]*>/i, '');
       return `
         <speak version="1.0" xml:lang="${lang}">
           <voice name="${voice}">
             <prosody rate="${rate}">
-              ${text}
+              ${cleanedText}
             </prosody>
           </voice>
         </speak>
@@ -255,6 +256,12 @@ export class EdgeSpeechTTS {
         }
       });
 
+      ws.addEventListener('close', () => {
+        if (!audioData.byteLength) {
+          reject(new Error('No audio data received.'));
+        }
+      });
+
       ws.addEventListener('error', () => {
         ws.close();
         reject(new Error('WebSocket error occurred.'));

apps/readest-app/src/services/tts/EdgeTTSClient.ts

@@ -10,7 +10,6 @@ export class EdgeTTSClient implements TTSClient {
   #rate = 1.0;
   #pitch = 1.0;
   #voice: TTSVoice | null = null;
-  #currentVoiceLang = '';
   #voices: TTSVoice[] = [];
   #edgeTTS: EdgeSpeechTTS;
 
@@ -45,25 +44,26 @@ export class EdgeTTSClient implements TTSClient {
     return { lang, text, voice: voiceId, rate: this.#rate, pitch: this.#pitch } as EdgeTTSPayload;
   };
 
+  getVoiceIdFromLang = async (lang: string) => {
+    let voiceId = 'en-US-AriaNeural';
+    const preferredVoiceId = TTSUtils.getPreferredVoice('edge-tts', lang);
+    const preferredVoice = this.#voices.find((v) => v.id === preferredVoiceId);
+    this.#voice = preferredVoice ? preferredVoice : (await this.getVoices(lang))[0] || null;
+    if (this.#voice) {
+      voiceId = this.#voice.id;
+    }
+    return voiceId;
+  };
+
   async *speak(
     ssml: string,
     signal: AbortSignal,
     preload = false,
   ): AsyncGenerator<TTSMessageEvent> {
     const { marks } = parseSSMLMarks(ssml);
-    let lang = parseSSMLLang(ssml) || 'en';
-    if (lang === 'en' && this.#primaryLang && this.#primaryLang !== 'en') {
-      lang = this.#primaryLang;
-    }
-    let voiceId = 'en-US-AriaNeural';
-    if (!this.#voice || this.#currentVoiceLang !== lang) {
-      const preferredVoiceId = TTSUtils.getPreferredVoice('edge-tts', lang);
-      const preferredVoice = this.#voices.find((v) => v.id === preferredVoiceId);
-      this.#voice = preferredVoice ? preferredVoice : (await this.getVoices(lang))[0] || null;
-      this.#currentVoiceLang = lang;
-    }
-    if (this.#voice) {
-      voiceId = this.#voice.id;
+    let defaultLang = parseSSMLLang(ssml) || 'en';
+    if (defaultLang === 'en' && this.#primaryLang && this.#primaryLang !== 'en') {
+      defaultLang = this.#primaryLang;
     }
 
     if (preload) {
@@ -71,16 +71,24 @@ export class EdgeTTSClient implements TTSClient {
       const maxImmediate = 2;
       for (let i = 0; i < Math.min(maxImmediate, marks.length); i++) {
         const mark = marks[i]!;
-        await this.#edgeTTS.createAudio(this.getPayload(lang, mark.text, voiceId)).catch((err) => {
-          console.warn('Error preloading mark', i, err);
-        });
+        const { language } = mark;
+        const voiceLang = language || defaultLang;
+        const voiceId = await this.getVoiceIdFromLang(voiceLang);
+        await this.#edgeTTS
+          .createAudio(this.getPayload(voiceLang, mark.text, voiceId))
+          .catch((err) => {
+            console.warn('Error preloading mark', i, err);
+          });
       }
       if (marks.length > maxImmediate) {
         (async () => {
           for (let i = maxImmediate; i < marks.length; i++) {
             const mark = marks[i]!;
             try {
-              await this.#edgeTTS.createAudio(this.getPayload(lang, mark.text, voiceId));
+              const { language } = mark;
+              const voiceLang = language || defaultLang;
+              const voiceId = await this.getVoiceIdFromLang(voiceLang);
+              await this.#edgeTTS.createAudio(this.getPayload(voiceLang, mark.text, voiceId));
             } catch (err) {
               console.warn('Error preloading mark (bg)', i, err);
             }
@@ -107,7 +115,12 @@ export class EdgeTTSClient implements TTSClient {
         break;
       }
       try {
-        const blob = await this.#edgeTTS.createAudio(this.getPayload(lang, mark.text, voiceId));
+        const { language } = mark;
+        const voiceLang = language || defaultLang;
+        const voiceId = await this.getVoiceIdFromLang(voiceLang);
+        const blob = await this.#edgeTTS.createAudio(
+          this.getPayload(voiceLang, mark.text, voiceId),
+        );
         const url = URL.createObjectURL(blob);
         this.#audioElement = new Audio(url);
         const audio = this.#audioElement;
@@ -240,9 +253,6 @@ export class EdgeTTSClient implements TTSClient {
   }
 
   async getVoices(lang: string): Promise<TTSVoice[]> {
-    if (this.#currentVoiceLang) {
-      lang = this.#currentVoiceLang;
-    }
     const locale = lang === 'en' ? getUserLocale(lang) || lang : lang;
     const voices = await this.getAllVoices();
     return voices

apps/readest-app/src/services/tts/TTSController.ts

@@ -82,8 +82,9 @@ export class TTSController extends EventTarget {
     if (!ssml) return;
     ssml = ssml
       .replace(/[–—]/g, ',')
-      .replace(/\.{3,}/g, '<break time="400ms"/>')
-      .replace(/·/g, '<break time="200ms"/>');
+      .replace('<break/>', ' ')
+      .replace(/\.{3,}/g, '   ')
+      .replace(/·/g, ' ');
 
     return ssml;
   }

apps/readest-app/src/services/tts/WebSpeechClient.ts

@@ -1,8 +1,6 @@
 import { getUserLocale } from '@/utils/misc';
 import { TTSClient, TTSMessageEvent, TTSVoice } from './TTSClient';
-import { AsyncQueue } from '@/utils/queue';
-import { findSSMLMark, parseSSMLLang, parseSSMLMarks } from '@/utils/ssml';
-import { isCJKStr } from '@/utils/lang';
+import { parseSSMLLang, parseSSMLMarks } from '@/utils/ssml';
 import { TTSGranularity } from '@/types/view';
 import { TTSUtils } from './TTSUtils';
 
@@ -46,105 +44,30 @@ interface TTSBoundaryEvent {
   error?: string;
 }
 
-async function* speakWithBoundary(
-  ssml: string,
-  getRate: () => number,
-  getPitch: () => number,
-  getVoice: () => SpeechSynthesisVoice | null,
-) {
-  const lang = parseSSMLLang(ssml);
-  const { plainText, marks } = parseSSMLMarks(ssml);
-  // console.log('ssml', ssml, marks);
-  // console.log('text', plainText);
-
-  const synth = window.speechSynthesis;
-  const utterance = new SpeechSynthesisUtterance(plainText);
-
-  utterance.rate = getRate();
-  utterance.pitch = getPitch();
-  const voice = getVoice();
-  if (voice) {
-    utterance.voice = voice;
-  }
-  if (lang) {
-    utterance.lang = lang;
-  }
-
-  const queue = new AsyncQueue<TTSBoundaryEvent>();
-
-  utterance.onboundary = (event: SpeechSynthesisEvent) => {
-    utterance.rate = getRate();
-    utterance.pitch = getPitch();
-    const voice = getVoice();
-    if (voice) {
-      utterance.voice = voice;
-    }
-    const mark = findSSMLMark(event.charIndex, marks);
-    // console.log('boundary', event.charIndex, mark);
-    queue.enqueue({
-      type: 'boundary',
-      speaking: true,
-      name: event.name,
-      mark: mark?.name ?? '',
-      charIndex: event.charIndex,
-      charLength: event.charLength,
-    });
-  };
-
-  utterance.onend = () => {
-    queue.enqueue({ type: 'end', speaking: false });
-    queue.finish();
-  };
-
-  utterance.onerror = (event) => {
-    queue.enqueue({ type: 'error', speaking: false, error: event.error });
-    queue.finish();
-  };
-
-  synth.speak(utterance);
-
-  while (true) {
-    const ev = await queue.dequeue();
-    if (ev === null) {
-      break;
-    }
-    yield ev;
-  }
-}
-
 async function* speakWithMarks(
   ssml: string,
+  defaultLang: string,
   getRate: () => number,
   getPitch: () => number,
-  getVoice: () => SpeechSynthesisVoice | null,
+  getVoice: (lang: string) => Promise<SpeechSynthesisVoice | null>,
 ) {
-  const { plainText, marks } = parseSSMLMarks(ssml);
-  const lang = parseSSMLLang(ssml);
-
-  const isCJK = (lang: string | null) => {
-    const cjkLangs = ['zh', 'ja', 'kr'];
-    if (lang && cjkLangs.some((cjk) => lang.startsWith(cjk))) return true;
-    return isCJKStr(plainText);
-  };
-
-  if (!isCJK(lang)) {
-    yield* speakWithBoundary(ssml, getRate, getPitch, getVoice);
-    return;
-  }
+  const { marks } = parseSSMLMarks(ssml);
 
   const synth = window.speechSynthesis;
 
   const utterance = new SpeechSynthesisUtterance();
   for (const mark of marks) {
+    const { language } = mark;
+    const voiceLang = language || defaultLang;
     utterance.text = mark.text;
     utterance.rate = getRate();
     utterance.pitch = getPitch();
-    const voice = getVoice();
+    const voice = await getVoice(voiceLang);
     if (voice) {
       utterance.voice = voice;
     }
-    if (lang) {
-      utterance.lang = lang;
+    if (voiceLang) {
+      utterance.lang = voiceLang;
     }
 
     yield {
@@ -178,7 +101,6 @@ export class WebSpeechClient implements TTSClient {
   #rate = 1.0;
   #pitch = 1.0;
   #voice: SpeechSynthesisVoice | null = null;
-  #currentVoiceLang = '';
   #voices: SpeechSynthesisVoice[] = [];
   #synth = window.speechSynthesis;
   available = true;
@@ -209,6 +131,18 @@ export class WebSpeechClient implements TTSClient {
     return this.available;
   }
 
+  getVoiceFromLang = async (lang: string) => {
+    const preferredVoiceId = TTSUtils.getPreferredVoice('web-speech', lang);
+    const preferredVoice = this.#voices.find((v) => v.voiceURI === preferredVoiceId);
+    if (preferredVoice) {
+      this.#voice = preferredVoice;
+    } else {
+      const voiceId = (await this.getVoices(lang))[0]?.id ?? '';
+      this.#voice = this.#voices.find((v) => v.voiceURI === voiceId) || null;
+    }
+    return this.#voice;
+  };
+
   async *speak(
     ssml: string,
     signal: AbortSignal,
@@ -217,24 +151,17 @@ export class WebSpeechClient implements TTSClient {
     // no need to preload for web speech
     if (preload) return;
 
-    let lang = parseSSMLLang(ssml) || 'en';
-    if (lang === 'en' && this.#primaryLang && this.#primaryLang !== 'en') {
-      lang = this.#primaryLang;
-    }
-    if (!this.#voice || this.#currentVoiceLang !== lang) {
-      const preferredVoiceId = TTSUtils.getPreferredVoice('web-speech', lang);
-      const preferredVoice = this.#voices.find((v) => v.voiceURI === preferredVoiceId);
-      const voiceId = (await this.getVoices(lang))[0]?.id ?? '';
-      this.#voice = preferredVoice
-        ? preferredVoice
-        : this.#voices.find((v) => v.voiceURI === voiceId) || null;
-      this.#currentVoiceLang = lang;
+    let defaultLang = parseSSMLLang(ssml) || 'en';
+    if (defaultLang === 'en' && this.#primaryLang && this.#primaryLang !== 'en') {
+      defaultLang = this.#primaryLang;
     }
+
     for await (const ev of speakWithMarks(
       ssml,
+      defaultLang,
       () => this.#rate,
       () => this.#pitch,
-      () => this.#voice,
+      this.getVoiceFromLang,
     )) {
       if (signal.aborted) {
         console.log('TTS aborted');
@@ -301,9 +228,6 @@ export class WebSpeechClient implements TTSClient {
   }
 
   async getVoices(lang: string) {
-    if (this.#currentVoiceLang) {
-      lang = this.#currentVoiceLang;
-    }
     const locale = lang === 'en' ? getUserLocale(lang) || lang : lang;
     const isValidVoice = (id: string) => {
       return !id.includes('com.apple') || id.includes('com.apple.voice.compact');

apps/readest-app/src/utils/ssml.ts

@@ -2,35 +2,65 @@ type TTSMark = {
   offset: number;
   name: string;
   text: string;
+  language?: string;
 };
 
-export const parseSSMLMarks = (ssml: string) => {
-  ssml = ssml.replace(/<speak[^>]*>/i, '');
-  ssml = ssml.replace(/<\/speak>/i, '');
+const cleanTextContent = (text: string) =>
+  text.replace(/\r\n/g, '  ').replace(/\r/g, ' ').replace(/\n/g, ' ').trimStart();
+
+export const parseSSMLMarks = (ssml: string) => {
+  const speakMatch = ssml.match(/<speak[^>]*xml:lang="([^"]+)"[^>]*>/i);
+  const defaultLang = speakMatch?.[1] ?? 'en';
+  ssml = ssml.replace(/<speak[^>]*>/i, '').replace(/<\/speak>/i, '');
 
-  const markRegex = /<mark\s+name="([^"]+)"\s*\/>/g;
   let plainText = '';
   const marks: TTSMark[] = [];
 
-  let match;
-  while ((match = markRegex.exec(ssml)) !== null) {
-    const markTagEndIndex = markRegex.lastIndex;
-    const nextMarkIndex = ssml.indexOf('<mark', markTagEndIndex);
-    const nextChunk = ssml.slice(
-      markTagEndIndex,
-      nextMarkIndex !== -1 ? nextMarkIndex : ssml.length,
-    );
-    const cleanedChunk = nextChunk
-      .replace(/<[^>]+>/g, '')
-      .replace(/\r\n/g, '  ')
-      .replace(/\r/g, ' ')
-      .replace(/\n/g, ' ')
-      .trimStart();
-    plainText += cleanedChunk;
+  let activeMark: string | null = null;
+  let currentLang = defaultLang;
+  const langStack: string[] = [];
 
-    const offset = plainText.length - cleanedChunk.length;
-    const markName = match[1]!;
-    marks.push({ offset, name: markName, text: cleanedChunk });
+  const tagRegex = /<(\/?)(\w+)([^>]*)>|([^<]+)/g;
+
+  let match: RegExpExecArray | null;
+  while ((match = tagRegex.exec(ssml)) !== null) {
+    if (match[4]) {
+      const rawText = match[4];
+      const text = cleanTextContent(rawText);
+      if (text && activeMark) {
+        const offset = plainText.length;
+        plainText += text;
+        marks.push({
+          offset,
+          name: activeMark,
+          text,
+          language: currentLang,
+        });
+      } else {
+        plainText += cleanTextContent(rawText);
+      }
+    } else {
+      const isEnd = match[1] === '/';
+      const tagName = match[2];
+      const attr = match[3];
+
+      if (tagName === 'mark' && !isEnd) {
+        const nameMatch = attr?.match(/name="([^"]+)"/);
+        if (nameMatch) {
+          activeMark = nameMatch[1]!;
+        }
+      } else if (tagName === 'lang') {
+        if (!isEnd) {
+          langStack.push(currentLang);
+          const langMatch = attr?.match(/xml:lang="([^"]+)"/);
+          if (langMatch) {
+            currentLang = langMatch[1]!;
+          }
+        } else {
+          currentLang = langStack.pop() ?? defaultLang;
+        }
+      }
+    }
   }
 
   return { plainText, marks };