fix(tts): abortable prefetch of tts data, closes #2037 (#2110)

apps/readest-app/src/app/reader/components/tts/TTSPanel.tsx

@@ -235,7 +235,7 @@ const TTSPanel = ({
       </div>
       <div className='flex items-center justify-between space-x-2'>
         <button
-          onClick={onBackward}
+          onClick={() => onBackward()}
           className='rounded-full p-1 transition-transform duration-200 hover:scale-105'
           title={_('Previous Paragraph')}
           aria-label={_('Previous Paragraph')}
@@ -255,7 +255,7 @@ const TTSPanel = ({
           )}
         </button>
         <button
-          onClick={onForward}
+          onClick={() => onForward()}
           className='rounded-full p-1 transition-transform duration-200 hover:scale-105'
           title={_('Next Paragraph')}
           aria-label={_('Next Paragraph')}

apps/readest-app/src/libs/edgeTTS.ts

@@ -238,7 +238,12 @@ const hashPayload = (payload: EdgeTTSPayload): string => {
 
 export class EdgeSpeechTTS {
   static voices = genVoiceList(EDGE_TTS_VOICES);
-  private static audioCache = new LRUCache<string, ArrayBuffer>(200);
+  private static audioCache = new LRUCache<string, Blob>(200);
+  private static audioUrlCache = new LRUCache<string, string>(200, (_, url) => {
+    if (url.startsWith('blob:')) {
+      URL.revokeObjectURL(url);
+    }
+  });
 
   constructor() {}
 
@@ -361,16 +366,19 @@ export class EdgeSpeechTTS {
     return this.#fetchEdgeSpeechWs(payload);
   }
 
-  async createAudio(payload: EdgeTTSPayload): Promise<Blob> {
+  async createAudioUrl(payload: EdgeTTSPayload): Promise<string> {
     const cacheKey = hashPayload(payload);
-    if (EdgeSpeechTTS.audioCache.has(cacheKey)) {
-      return new Blob([EdgeSpeechTTS.audioCache.get(cacheKey)!], { type: 'audio/mpeg' });
+    if (EdgeSpeechTTS.audioUrlCache.has(cacheKey)) {
+      return EdgeSpeechTTS.audioUrlCache.get(cacheKey)!;
     }
     try {
       const res = await this.create(payload);
       const arrayBuffer = await res.arrayBuffer();
-      EdgeSpeechTTS.audioCache.set(cacheKey, arrayBuffer);
-      return new Blob([arrayBuffer], { type: 'audio/mpeg' });
+      const blob = new Blob([arrayBuffer], { type: 'audio/mpeg' });
+      const objectUrl = URL.createObjectURL(blob);
+      EdgeSpeechTTS.audioCache.set(cacheKey, blob);
+      EdgeSpeechTTS.audioUrlCache.set(cacheKey, objectUrl);
+      return objectUrl;
     } catch (error) {
       throw error;
     }

apps/readest-app/src/services/tts/EdgeTTSClient.ts

@@ -67,12 +67,13 @@ export class EdgeTTSClient implements TTSClient {
       // preload the first 2 marks immediately and the rest in the background
       const maxImmediate = 2;
       for (let i = 0; i < Math.min(maxImmediate, marks.length); i++) {
+        if (signal.aborted) break;
         const mark = marks[i]!;
         const { language: voiceLang } = mark;
         const voiceId = await this.getVoiceIdFromLang(voiceLang);
         this.#currentVoiceId = voiceId;
         await this.#edgeTTS
-          .createAudio(this.getPayload(voiceLang, mark.text, voiceId))
+          .createAudioUrl(this.getPayload(voiceLang, mark.text, voiceId))
           .catch((err) => {
             console.warn('Error preloading mark', i, err);
           });
@@ -82,9 +83,10 @@ export class EdgeTTSClient implements TTSClient {
           for (let i = maxImmediate; i < marks.length; i++) {
             const mark = marks[i]!;
             try {
+              if (signal.aborted) break;
               const { language: voiceLang } = mark;
               const voiceId = await this.getVoiceIdFromLang(voiceLang);
-              await this.#edgeTTS.createAudio(this.getPayload(voiceLang, mark.text, voiceId));
+              await this.#edgeTTS.createAudioUrl(this.getPayload(voiceLang, mark.text, voiceId));
             } catch (err) {
               console.warn('Error preloading mark (bg)', i, err);
             }
@@ -102,23 +104,27 @@ export class EdgeTTSClient implements TTSClient {
 
     await this.stopInternal();
     // Reuse the same Audio element inside the ssml session
+    if (!this.#audioElement) {
       this.#audioElement = new Audio();
+    }
     const audio = this.#audioElement;
     audio.setAttribute('x-webkit-airplay', 'deny');
     audio.preload = 'auto';
 
     for (const mark of marks) {
+      this.controller?.dispatchSpeakMark(mark);
       let abortHandler: null | (() => void) = null;
       try {
         const { language: voiceLang } = mark;
         const voiceId = await this.getVoiceIdFromLang(voiceLang);
         this.#speakingLang = voiceLang;
-        const blob = await this.#edgeTTS.createAudio(
+        const audioUrl = await this.#edgeTTS.createAudioUrl(
           this.getPayload(voiceLang, mark.text, voiceId),
         );
-        audio.src = URL.createObjectURL(blob);
-
-        this.controller?.dispatchSpeakMark(mark);
+        if (signal.aborted) {
+          yield { code: 'error', message: 'Aborted' } as TTSMessageEvent;
+          break;
+        }
 
         yield {
           code: 'boundary',
@@ -130,11 +136,7 @@ export class EdgeTTSClient implements TTSClient {
           const cleanUp = () => {
             audio.onended = null;
             audio.onerror = null;
-            audio.pause();
-            if (audio.src) {
-              URL.revokeObjectURL(audio.src);
-              audio.removeAttribute('src');
-            }
+            audio.src = '';
           };
           abortHandler = () => {
             cleanUp();
@@ -156,6 +158,7 @@ export class EdgeTTSClient implements TTSClient {
             resolve({ code: 'error', message: 'Audio playback error' });
           };
           this.#isPlaying = true;
+          audio.src = audioUrl;
           audio.play().catch((err) => {
             cleanUp();
             console.error('Failed to play audio:', err);
@@ -229,11 +232,7 @@ export class EdgeTTSClient implements TTSClient {
       if (this.#audioElement?.onended) {
         this.#audioElement.onended(new Event('stopped'));
       }
-      if (this.#audioElement.src?.startsWith('blob:')) {
-        URL.revokeObjectURL(this.#audioElement.src);
-      }
       this.#audioElement.src = '';
-      this.#audioElement = null;
     }
   }
 

apps/readest-app/src/services/tts/TTSController.ts

@@ -117,9 +117,9 @@ export class TTSController extends EventTarget {
     );
   }
 
-  async preloadSSML(ssml: string | undefined) {
+  async preloadSSML(ssml: string | undefined, signal: AbortSignal) {
     if (!ssml) return;
-    const iter = await this.ttsClient.speak(ssml, new AbortController().signal, true);
+    const iter = await this.ttsClient.speak(ssml, signal, true);
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     for await (const _ of iter);
   }
@@ -130,7 +130,7 @@ export class TTSController extends EventTarget {
     let preloaded = 0;
     for (let i = 0; i < count; i++) {
       const ssml = this.#preprocessSSML(tts.next());
-      this.preloadSSML(ssml);
+      this.preloadSSML(ssml, new AbortController().signal);
       if (ssml) preloaded++;
     }
     for (let i = 0; i < preloaded; i++) {
@@ -161,8 +161,12 @@ export class TTSController extends EventTarget {
       try {
         console.log('TTS speak');
         this.state = 'playing';
+
+        signal.addEventListener('abort', () => {
+          resolve();
+        });
+
         ssml = this.#preprocessSSML(await ssml);
-        await this.preloadSSML(ssml);
         if (!ssml) {
           this.#nossmlCnt++;
           // FIXME: in case we are at the end of the book, need a better way to handle this
@@ -181,18 +185,17 @@ export class TTSController extends EventTarget {
         if (!plainText || marks.length === 0) {
           resolve();
           return await this.forward();
+        } else {
+          this.dispatchSpeakMark(marks[0]);
         }
+        await this.preloadSSML(ssml, signal);
         const iter = await this.ttsClient.speak(ssml, signal);
         let lastCode;
-        for await (const { code, mark } of iter) {
+        for await (const { code } of iter) {
           if (signal.aborted) {
             resolve();
             return;
           }
-          if (mark && this.state === 'playing') {
-            const range = this.view.tts?.setMark(mark);
-            this.dispatchEvent(new CustomEvent('tts-highlight-mark', { detail: range }));
-          }
           lastCode = code;
         }
 
@@ -208,10 +211,13 @@ export class TTSController extends EventTarget {
           reject(e);
         }
       } finally {
+        if (this.#currentSpeakAbortController) {
+          this.#currentSpeakAbortController.abort();
           this.#currentSpeakAbortController = null;
-        this.#currentSpeakPromise = null;
+        }
       }
     });
+
     await this.#currentSpeakPromise.catch((e) => this.error(e));
   }
 
@@ -366,6 +372,10 @@ export class TTSController extends EventTarget {
 
   dispatchSpeakMark(mark?: TTSMark) {
     this.dispatchEvent(new CustomEvent('tts-speak-mark', { detail: mark || { text: '' } }));
+    if (mark) {
+      const range = this.view.tts?.setMark(mark.name);
+      this.dispatchEvent(new CustomEvent('tts-highlight-mark', { detail: range }));
+    }
   }
 
   error(e: unknown) {

apps/readest-app/src/utils/lru.ts

@@ -1,13 +1,15 @@
 export class LRUCache<K, V> {
   private capacity: number;
   private map: Map<K, V>;
+  private onEvict?: (key: K, value: V) => void;
 
-  constructor(capacity: number) {
+  constructor(capacity: number, onEvict?: (key: K, value: V) => void) {
     if (capacity <= 0) {
       throw new Error('LRUCache capacity must be greater than 0');
     }
     this.capacity = capacity;
     this.map = new Map();
+    this.onEvict = onEvict;
   }
 
   get(key: K): V | undefined {
@@ -22,11 +24,19 @@ export class LRUCache<K, V> {
 
   set(key: K, value: V): void {
     if (this.map.has(key)) {
+      const oldValue = this.map.get(key)!;
       this.map.delete(key);
+      if (this.onEvict) {
+        this.onEvict(key, oldValue);
+      }
     } else if (this.map.size === this.capacity) {
       const oldestKey = this.map.keys().next().value;
       if (oldestKey) {
+        const oldestValue = this.map.get(oldestKey)!;
         this.map.delete(oldestKey);
+        if (this.onEvict) {
+          this.onEvict(oldestKey, oldestValue);
+        }
       }
     }
     this.map.set(key, value);
@@ -37,10 +47,23 @@ export class LRUCache<K, V> {
   }
 
   delete(key: K): boolean {
-    return this.map.delete(key);
+    if (this.map.has(key)) {
+      const value = this.map.get(key)!;
+      const deleted = this.map.delete(key);
+      if (deleted && this.onEvict) {
+        this.onEvict(key, value);
+      }
+      return deleted;
+    }
+    return false;
   }
 
   clear(): void {
+    if (this.onEvict) {
+      for (const [key, value] of this.map) {
+        this.onEvict(key, value);
+      }
+    }
     this.map.clear();
   }