From 87069c7308fa1ae6e6cc53eae2c46d6342bf089f Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Mon, 20 Jan 2025 01:53:12 +0100 Subject: [PATCH] Fix TTS on iOS browsers, closes #172 (#198) --- .../app/reader/components/tts/TTSControl.tsx | 20 ++-- apps/readest-app/src/libs/edgeTTS.ts | 4 +- .../src/services/tts/EdgeTTSClient.ts | 57 ++++++++---- .../src/services/tts/TTSController.ts | 92 ++++++++++++------- 4 files changed, 106 insertions(+), 67 deletions(-) diff --git a/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx b/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx index f179459e..d549ec5b 100644 --- a/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx +++ b/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx @@ -162,13 +162,9 @@ const TTSControl = () => { throttle(async (rate: number) => { const ttsController = ttsControllerRef.current; if (ttsController) { - if (ttsController.state === 'playing') { - await ttsController.stop(); - await ttsController.setRate(rate); - await ttsController.start(); - } else { - await ttsController.setRate(rate); - } + await ttsController.stop(); + await ttsController.setRate(rate); + await ttsController.start(); } }, 3000), [], @@ -179,13 +175,9 @@ const TTSControl = () => { throttle(async (voice: string) => { const ttsController = ttsControllerRef.current; if (ttsController) { - if (ttsController.state === 'playing') { - await ttsController.stop(); - await ttsController.setVoice(voice); - await ttsController.start(); - } else { - await ttsController.setVoice(voice); - } + await ttsController.stop(); + await ttsController.setVoice(voice); + await ttsController.start(); } }, 3000), [], diff --git a/apps/readest-app/src/libs/edgeTTS.ts b/apps/readest-app/src/libs/edgeTTS.ts index 2bbea905..4253d40c 100644 --- a/apps/readest-app/src/libs/edgeTTS.ts +++ b/apps/readest-app/src/libs/edgeTTS.ts @@ -144,6 +144,7 @@ const hashPayload = (payload: EdgeTTSPayload): string => { export class EdgeSpeechTTS { static voices = genVoiceList(EDGE_TTS_VOICES); private static audioCache = new LRUCache(200); + private audioContext = new AudioContext(); constructor() {} @@ -274,8 +275,7 @@ export class EdgeSpeechTTS { try { const res = await this.create(payload); const arrayBuffer = await res.arrayBuffer(); - const audioContext = new AudioContext(); - const audioBuffer = await audioContext.decodeAudioData(arrayBuffer.slice(0)); + const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer.slice(0)); EdgeSpeechTTS.audioCache.set(cacheKey, audioBuffer); return audioBuffer; } catch (error) { diff --git a/apps/readest-app/src/services/tts/EdgeTTSClient.ts b/apps/readest-app/src/services/tts/EdgeTTSClient.ts index 9e47f9aa..ca5ed0fe 100644 --- a/apps/readest-app/src/services/tts/EdgeTTSClient.ts +++ b/apps/readest-app/src/services/tts/EdgeTTSClient.ts @@ -11,7 +11,7 @@ export class EdgeTTSClient implements TTSClient { #voices: TTSVoice[] = []; #edgeTTS: EdgeSpeechTTS; - #audioContext: AudioContext | null = null; + static #audioContext: AudioContext | null; #sourceNode: AudioBufferSourceNode | null = null; #isPlaying = false; #pausedAt = 0; @@ -23,9 +23,19 @@ export class EdgeTTSClient implements TTSClient { this.#edgeTTS = new EdgeSpeechTTS(); } + async initializeAudioContext() { + if (!EdgeTTSClient.#audioContext) { + EdgeTTSClient.#audioContext = new AudioContext(); + } + if (EdgeTTSClient.#audioContext.state === 'suspended') { + await EdgeTTSClient.#audioContext.resume(); + } + } + async init() { this.#voices = EdgeSpeechTTS.voices; try { + await this.initializeAudioContext(); await this.#edgeTTS.create({ lang: 'en', text: 'test', @@ -88,10 +98,12 @@ export class EdgeTTSClient implements TTSClient { this.#audioBuffer = await this.#edgeTTS.createAudio( this.getPayload(lang, mark.text, voiceId), ); - this.#audioContext = new AudioContext(); - this.#sourceNode = this.#audioContext.createBufferSource(); + if (!EdgeTTSClient.#audioContext) { + EdgeTTSClient.#audioContext = new AudioContext(); + } + this.#sourceNode = EdgeTTSClient.#audioContext.createBufferSource(); this.#sourceNode.buffer = this.#audioBuffer; - this.#sourceNode.connect(this.#audioContext.destination); + this.#sourceNode.connect(EdgeTTSClient.#audioContext.destination); yield { code: 'boundary', @@ -100,11 +112,18 @@ export class EdgeTTSClient implements TTSClient { }; const result = await new Promise((resolve) => { - if (this.#audioContext === null || this.#sourceNode === null) { + if (EdgeTTSClient.#audioContext === null || this.#sourceNode === null) { throw new Error('Audio context or source node is null'); } - this.#sourceNode.onended = () => { - this.#isPlaying = false; + this.#sourceNode.onended = (event: Event) => { + // chunk finished speaking or aborted speaking + if (signal.aborted || event.type === 'stopped') { + resolve({ + code: 'error', + message: 'Aborted', + }); + return; + } resolve({ code: 'end', message: `Chunk finished: ${mark.name}`, @@ -117,9 +136,12 @@ export class EdgeTTSClient implements TTSClient { }); return; } + if (EdgeTTSClient.#audioContext.state === 'suspended') { + EdgeTTSClient.#audioContext.resume(); + } this.#sourceNode.start(0); this.#isPlaying = true; - this.#startedAt = this.#audioContext.currentTime; + this.#startedAt = EdgeTTSClient.#audioContext.currentTime; }); yield result; } catch (error) { @@ -144,17 +166,17 @@ export class EdgeTTSClient implements TTSClient { } async pause() { - if (!this.#isPlaying || !this.#audioContext || !this.#sourceNode) return; - this.#pausedAt = this.#audioContext.currentTime - this.#startedAt; - await this.#audioContext.suspend(); + if (!this.#isPlaying || !EdgeTTSClient.#audioContext) return; + this.#pausedAt = EdgeTTSClient.#audioContext.currentTime - this.#startedAt; + await EdgeTTSClient.#audioContext.suspend(); this.#isPlaying = false; } async resume() { - if (this.#isPlaying || !this.#audioContext || !this.#sourceNode) return; - await this.#audioContext.resume(); + if (this.#isPlaying || !EdgeTTSClient.#audioContext) return; + await EdgeTTSClient.#audioContext.resume(); this.#isPlaying = true; - this.#startedAt = this.#audioContext.currentTime - this.#pausedAt; + this.#startedAt = EdgeTTSClient.#audioContext.currentTime - this.#pausedAt; } async stop() { @@ -168,6 +190,9 @@ export class EdgeTTSClient implements TTSClient { if (this.#sourceNode) { try { this.#sourceNode.stop(); + if (this.#sourceNode?.onended) { + this.#sourceNode.onended(new Event('stopped')); + } } catch (err) { if (!(err instanceof Error) || err.name !== 'InvalidStateError') { console.log('Error stopping source node:', err); @@ -176,10 +201,6 @@ export class EdgeTTSClient implements TTSClient { this.#sourceNode.disconnect(); this.#sourceNode = null; } - if (this.#audioContext) { - await this.#audioContext.close(); - this.#audioContext = null; - } this.#audioBuffer = null; } diff --git a/apps/readest-app/src/services/tts/TTSController.ts b/apps/readest-app/src/services/tts/TTSController.ts index b2978c01..dee3c5d2 100644 --- a/apps/readest-app/src/services/tts/TTSController.ts +++ b/apps/readest-app/src/services/tts/TTSController.ts @@ -17,6 +17,7 @@ export class TTSController extends EventTarget { view: FoliateView; #nossmlCnt: number = 0; #currentSpeakAbortController: AbortController | null = null; + #currentSpeakPromise: Promise | null = null; ttsRate: number = 1.0; ttsClient: TTSClient; @@ -61,7 +62,7 @@ export class TTSController extends EventTarget { for await (const _ of iter); } - async preloadNextSSML(count: number) { + async preloadNextSSML(count: number = 2) { const tts = this.view.tts; if (!tts) return; let preloaded = 0; @@ -80,44 +81,60 @@ export class TTSController extends EventTarget { this.#currentSpeakAbortController = new AbortController(); const { signal } = this.#currentSpeakAbortController; - try { - console.log('TTS speak'); - this.state = 'playing'; - ssml = await ssml; - await this.preloadSSML(ssml); - if (!ssml) { - this.#nossmlCnt++; - // FIXME: in case we are at the end of the book, need a better way to handle this - if (this.#nossmlCnt < 10 && this.state === 'playing') { - await this.view.next(1); + this.#currentSpeakPromise = new Promise(async (resolve, reject) => { + try { + console.log('TTS speak'); + this.state = 'playing'; + ssml = await ssml; + await this.preloadSSML(ssml); + if (!ssml) { + this.#nossmlCnt++; + // FIXME: in case we are at the end of the book, need a better way to handle this + if (this.#nossmlCnt < 10 && this.state === 'playing') { + await this.view.next(1); + await this.forward(); + } + return; + } else { + this.#nossmlCnt = 0; + } + + const iter = await this.ttsClient.speak(ssml, signal); + let lastCode: TTSMessageCode = 'boundary'; + for await (const { code, mark } of iter) { + if (signal.aborted) { + resolve(); + return; + } + if (mark && this.state === 'playing') { + this.view.tts?.setMark(mark); + } + lastCode = code; + } + + if (lastCode === 'end' && this.state === 'playing') { + resolve(); await this.forward(); } - return; - } else { - this.#nossmlCnt = 0; - } - - const iter = await this.ttsClient.speak(ssml, signal); - let lastCode: TTSMessageCode = 'boundary'; - for await (const { code, mark } of iter) { - if (mark && this.state === 'playing') { - this.view.tts?.setMark(mark); + resolve(); + } catch (e) { + if (signal.aborted) { + resolve(); + } else { + reject(e); } - lastCode = code; + } finally { + this.#currentSpeakAbortController = null; + this.#currentSpeakPromise = null; } - - if (lastCode === 'end' && this.state === 'playing') { - await this.forward(); - } - } finally { - this.#currentSpeakAbortController = null; - } + }); + await this.#currentSpeakPromise.catch((e) => this.error(e)); } async speak(ssml: string | Promise) { await this.initViewTTS(); this.#speak(ssml).catch((e) => this.error(e)); - this.preloadNextSSML(2); + this.preloadNextSSML(); } play() { @@ -131,8 +148,11 @@ export class TTSController extends EventTarget { async start() { await this.initViewTTS(); const ssml = this.state.includes('paused') ? this.view.tts?.resume() : this.view.tts?.start(); + if (this.state.includes('paused')) { + this.resume(); + } this.#speak(ssml); - this.preloadNextSSML(2); + this.preloadNextSSML(); } async pause() { @@ -146,11 +166,15 @@ export class TTSController extends EventTarget { } async stop() { - this.state = 'stopped'; if (this.#currentSpeakAbortController) { this.#currentSpeakAbortController.abort(); } await this.ttsClient.stop().catch((e) => this.error(e)); + + if (this.#currentSpeakPromise) { + await this.#currentSpeakPromise.catch((e) => this.error(e)); + } + this.state = 'stopped'; } // goto previous sentence @@ -160,6 +184,7 @@ export class TTSController extends EventTarget { await this.stop(); this.#speak(this.view.tts?.prev()); } else { + await this.stop(); this.state = 'backward-paused'; this.view.tts?.prev(true); } @@ -171,8 +196,9 @@ export class TTSController extends EventTarget { if (this.state === 'playing') { await this.stop(); this.#speak(this.view.tts?.next()); - this.preloadNextSSML(2); + this.preloadNextSSML(); } else { + await this.stop(); this.state = 'forward-paused'; this.view.tts?.next(true); }