tts: select voice in bilingual TTS mode (#1263)

README.md

@@ -48,12 +48,12 @@
 | **Annotations and Highlighting**        | Add highlights, bookmarks, and notes to enhance your reading experience.                       | ✅         |
 | **Excerpt Text for Note-Taking**        | Easily excerpt text from books for detailed notes and analysis.                                | ✅         |
 | **Dictionary/Wikipedia Lookup**         | Instantly look up words and terms when reading.                                                | ✅         |
-| **Translate with DeepL**                | Translate selected text instantly using DeepL for accurate translations.                       | ✅         |
 | **[Parallel Read][link-parallel-read]** | Read two books or documents simultaneously in a split-screen view.                             | ✅         |
 | **Customize Font and Layout**           | Adjust font, layout, theme mode, and theme colors for a personalized experience.               | ✅         |
 | **File Association and Open With**      | Quickly open files in Readest in your file browser with one-click.                             | ✅         |
 | **Sync across Platforms**               | Synchronize book files, reading progress, notes, and bookmarks across all supported platforms. | ✅         |
-| **Text-to-Speech (TTS) Support**        | Enable text-to-speech functionality for a more accessible reading experience.                  | ✅         |
+| **Translate with DeepL**                | From a single sentence to the entire book—translate instantly with DeepL.                      | ✅         |
+| **Text-to-Speech (TTS) Support**        | Enjoy smooth, multilingual narration—even within a single book.                                | ✅         |
 | **Library Management**                  | Organize, sort, and manage your entire ebook library.                                          | ✅         |
 
 ## Planned Features

apps/readest-app/src/app/reader/components/tts/TTSControl.tsx

@@ -140,7 +140,6 @@ const TTSControl = () => {
 
         ttsController.setLang(lang);
         ttsController.setRate(viewSettings.ttsRate);
-        ttsController.setVoice(viewSettings.ttsVoice);
         ttsController.speak(ssml);
         ttsControllerRef.current = ttsController;
       }
@@ -236,15 +235,15 @@ const TTSControl = () => {
 
   // eslint-disable-next-line react-hooks/exhaustive-deps
   const handleSetVoice = useCallback(
-    throttle(async (voice: string) => {
+    throttle(async (voice: string, lang: string) => {
       const ttsController = ttsControllerRef.current;
       if (ttsController) {
         if (ttsController.state === 'playing') {
           await ttsController.stop();
-          await ttsController.setVoice(voice);
+          await ttsController.setVoice(voice, lang);
           await ttsController.start();
         } else {
-          await ttsController.setVoice(voice);
+          await ttsController.setVoice(voice, lang);
         }
       }
     }, 3000),
@@ -311,6 +310,10 @@ const TTSControl = () => {
 
   const togglePopup = () => {
     updatePanelPosition();
+    if (!showPanel && ttsControllerRef.current) {
+      const speakingLang = ttsControllerRef.current.getSpeakingLang() || ttsLang;
+      setTtsLang(speakingLang);
+    }
     setShowPanel((prev) => !prev);
   };
 

apps/readest-app/src/app/reader/components/tts/TTSPanel.tsx

@@ -21,7 +21,7 @@ type TTSPanelProps = {
   onForward: () => void;
   onSetRate: (rate: number) => void;
   onGetVoices: (lang: string) => Promise<TTSVoice[]>;
-  onSetVoice: (voice: string) => void;
+  onSetVoice: (voice: string, lang: string) => void;
   onGetVoiceId: () => string;
   onSelectTimeout: (value: number) => void;
 };
@@ -144,8 +144,8 @@ const TTSPanel = ({
     saveSettings(envConfig, settings);
   };
 
-  const handleSelectVoice = (voice: string) => {
-    onSetVoice(voice);
+  const handleSelectVoice = (voice: string, lang: string) => {
+    onSetVoice(voice, lang);
     setSelectedVoice(voice);
     const viewSettings = getViewSettings(bookKey)!;
     viewSettings.ttsVoice = voice;
@@ -282,7 +282,7 @@ const TTSPanel = ({
             {voices.map((voice, index) => (
               <li
                 key={`${index}-${voice.id}`}
-                onClick={() => !voice.disabled && handleSelectVoice(voice.id)}
+                onClick={() => !voice.disabled && handleSelectVoice(voice.id, voice.lang)}
               >
                 <div className='flex items-center px-2'>
                   <span style={{ minWidth: `${defaultIconSize}px` }}>

apps/readest-app/src/services/tts/EdgeTTSClient.ts

@@ -7,6 +7,7 @@ import { TTSUtils } from './TTSUtils';
 
 export class EdgeTTSClient implements TTSClient {
   #primaryLang = 'en';
+  #speakingLang = '';
   #rate = 1.0;
   #pitch = 1.0;
   #voice: TTSVoice | null = null;
@@ -118,6 +119,7 @@ export class EdgeTTSClient implements TTSClient {
         const { language } = mark;
         const voiceLang = language || defaultLang;
         const voiceId = await this.getVoiceIdFromLang(voiceLang);
+        this.#speakingLang = voiceLang;
         const blob = await this.#edgeTTS.createAudio(
           this.getPayload(voiceLang, mark.text, voiceId),
         );
@@ -286,4 +288,8 @@ export class EdgeTTSClient implements TTSClient {
   getVoiceId(): string {
     return this.#voice?.id || '';
   }
+
+  getSpeakingLang(): string {
+    return this.#speakingLang;
+  }
 }

apps/readest-app/src/services/tts/TTSClient.ts

@@ -29,4 +29,5 @@ export interface TTSClient {
   getVoices(lang: string): Promise<TTSVoice[]>;
   getGranularities(): TTSGranularity[];
   getVoiceId(): string;
+  getSpeakingLang(): string;
 }

apps/readest-app/src/services/tts/TTSController.ts

@@ -247,7 +247,7 @@ export class TTSController extends EventTarget {
     return [...ttsEdgeVoices, ...ttsWebVoices];
   }
 
-  async setVoice(voiceId: string) {
+  async setVoice(voiceId: string, lang: string) {
     this.state = 'setvoice-paused';
     const useEdgeTTS = !!this.ttsEdgeVoices.find(
       (voice) => (voiceId === '' || voice.id === voiceId) && !voice.disabled,
@@ -255,11 +255,11 @@ export class TTSController extends EventTarget {
     if (useEdgeTTS) {
       this.ttsClient = this.ttsEdgeClient;
       await this.ttsClient.setRate(this.ttsRate);
-      TTSUtils.setPreferredVoice('edge-tts', this.ttsLang, voiceId);
+      TTSUtils.setPreferredVoice('edge-tts', lang, voiceId);
     } else {
       this.ttsClient = this.ttsWebClient;
       await this.ttsClient.setRate(this.ttsRate);
-      TTSUtils.setPreferredVoice('web-speech', this.ttsLang, voiceId);
+      TTSUtils.setPreferredVoice('web-speech', lang, voiceId);
     }
     await this.ttsClient.setVoice(voiceId);
   }
@@ -268,6 +268,10 @@ export class TTSController extends EventTarget {
     return this.ttsClient.getVoiceId();
   }
 
+  getSpeakingLang() {
+    return this.ttsClient.getSpeakingLang();
+  }
+
   error(e: unknown) {
     console.error(e);
     this.state = 'stopped';

apps/readest-app/src/services/tts/WebSpeechClient.ts

@@ -50,6 +50,7 @@ async function* speakWithMarks(
   getRate: () => number,
   getPitch: () => number,
   getVoice: (lang: string) => Promise<SpeechSynthesisVoice | null>,
+  setSpeakingLang: (lang: string) => void = () => {},
 ) {
   const { marks } = parseSSMLMarks(ssml);
 
@@ -59,6 +60,7 @@ async function* speakWithMarks(
   for (const mark of marks) {
     const { language } = mark;
     const voiceLang = language || defaultLang;
+    setSpeakingLang(voiceLang);
     utterance.text = mark.text;
     utterance.rate = getRate();
     utterance.pitch = getPitch();
@@ -98,6 +100,7 @@ async function* speakWithMarks(
 
 export class WebSpeechClient implements TTSClient {
   #primaryLang = 'en';
+  #speakingLang = '';
   #rate = 1.0;
   #pitch = 1.0;
   #voice: SpeechSynthesisVoice | null = null;
@@ -162,6 +165,7 @@ export class WebSpeechClient implements TTSClient {
       () => this.#rate,
       () => this.#pitch,
       this.getVoiceFromLang,
+      (lang) => (this.#speakingLang = lang),
     )) {
       if (signal.aborted) {
         console.log('TTS aborted');
@@ -275,4 +279,8 @@ export class WebSpeechClient implements TTSClient {
   getVoiceId(): string {
     return this.#voice?.voiceURI ?? '';
   }
+
+  getSpeakingLang(): string {
+    return this.#speakingLang;
+  }
 }