From 66f369b269b4ea90565c52afbf45a4bf8088a45e Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Mon, 13 Jan 2025 00:19:49 +0100 Subject: [PATCH] Fix multiple voices when setting rate and voices in Firefox (#146) --- .../app/reader/components/tts/TTSControl.tsx | 61 ++++++++++-------- .../src/services/tts/EdgeTTSClient.ts | 13 +++- .../readest-app/src/services/tts/TTSClient.ts | 2 +- .../src/services/tts/TTSController.ts | 64 +++++++++++-------- .../src/services/tts/WebSpeechClient.ts | 11 +++- apps/readest-app/src/utils/ui.ts | 28 ++++++++ 6 files changed, 121 insertions(+), 58 deletions(-) create mode 100644 apps/readest-app/src/utils/ui.ts diff --git a/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx b/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx index facc061c..3f6e2634 100644 --- a/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx +++ b/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx @@ -1,10 +1,11 @@ -import React, { useState, useRef, useEffect } from 'react'; +import React, { useState, useRef, useEffect, useCallback } from 'react'; import { useReaderStore } from '@/store/readerStore'; import { useTranslation } from '@/hooks/useTranslation'; import { TTSController } from '@/services/tts/TTSController'; import { getPopupPosition, Position } from '@/utils/sel'; import { eventDispatcher } from '@/utils/event'; import { parseSSMLLang } from '@/utils/ssml'; +import { throttle } from '@/utils/ui'; import Popup from '@/components/Popup'; import TTSPanel from './TTSPanel'; import TTSIcon from './TTSIcon'; @@ -97,10 +98,12 @@ const TTSControl = () => { if (!ttsController) return; if (isPlaying) { - await ttsController.pause(); setIsPlaying(false); setIsPaused(true); + await ttsController.pause(); } else if (isPaused) { + setIsPlaying(true); + setIsPaused(false); // start for forward/backward/setvoice-paused // set rate don't pause the tts if (ttsController.state === 'paused') { @@ -108,8 +111,6 @@ const TTSControl = () => { } else { await ttsController.start(); } - setIsPlaying(true); - setIsPaused(false); } }; @@ -139,31 +140,39 @@ const TTSControl = () => { }; // rate range: 0.5 - 3, 1.0 is normal speed - const handleSetRate = async (rate: number) => { - const ttsController = ttsControllerRef.current; - if (ttsController) { - if (ttsController.state === 'playing') { - await ttsController.pause(); - await ttsController.setRate(rate); - await ttsController.start(); - } else { - await ttsController.setRate(rate); + // eslint-disable-next-line react-hooks/exhaustive-deps + const handleSetRate = useCallback( + throttle(async (rate: number) => { + const ttsController = ttsControllerRef.current; + if (ttsController) { + if (ttsController.state === 'playing') { + await ttsController.stop(); + await ttsController.setRate(rate); + await ttsController.start(); + } else { + await ttsController.setRate(rate); + } } - } - }; + }, 2000), + [], + ); - const handleSetVoice = async (voice: string) => { - const ttsController = ttsControllerRef.current; - if (ttsController) { - if (ttsController.state === 'playing') { - await ttsController.pause(); - await ttsController.setVoice(voice); - await ttsController.start(); - } else { - await ttsController.setVoice(voice); + // eslint-disable-next-line react-hooks/exhaustive-deps + const handleSetVoice = useCallback( + throttle(async (voice: string) => { + const ttsController = ttsControllerRef.current; + if (ttsController) { + if (ttsController.state === 'playing') { + await ttsController.stop(); + await ttsController.setVoice(voice); + await ttsController.start(); + } else { + await ttsController.setVoice(voice); + } } - } - }; + }, 2000), + [], + ); const handleGetVoices = async (lang: string) => { const ttsController = ttsControllerRef.current; diff --git a/apps/readest-app/src/services/tts/EdgeTTSClient.ts b/apps/readest-app/src/services/tts/EdgeTTSClient.ts index 08243d04..1c015165 100644 --- a/apps/readest-app/src/services/tts/EdgeTTSClient.ts +++ b/apps/readest-app/src/services/tts/EdgeTTSClient.ts @@ -44,7 +44,7 @@ export class EdgeTTSClient implements TTSClient { return { lang, text, voice: voiceId, rate: this.#rate, pitch: this.#pitch } as EdgeTTSPayload; }; - async *speak(ssml: string): AsyncGenerator { + async *speak(ssml: string, signal: AbortSignal): AsyncGenerator { const { marks } = parseSSMLMarks(ssml); const lang = parseSSMLLang(ssml) || 'en'; @@ -69,6 +69,13 @@ export class EdgeTTSClient implements TTSClient { } for (const mark of marks) { + if (signal.aborted) { + yield { + code: 'error', + message: 'Aborted', + }; + break; + } try { this.#audioBuffer = await this.#edgeTTS.createAudio( this.getPayload(lang, mark.text, voiceId), @@ -147,7 +154,9 @@ export class EdgeTTSClient implements TTSClient { try { this.#sourceNode.stop(); } catch (err) { - console.error('Error stopping source node:', err); + if (!(err instanceof Error) || err.name !== 'InvalidStateError') { + console.log('Error stopping source node:', err); + } } this.#sourceNode.disconnect(); this.#sourceNode = null; diff --git a/apps/readest-app/src/services/tts/TTSClient.ts b/apps/readest-app/src/services/tts/TTSClient.ts index d20f4682..3e454355 100644 --- a/apps/readest-app/src/services/tts/TTSClient.ts +++ b/apps/readest-app/src/services/tts/TTSClient.ts @@ -17,7 +17,7 @@ export interface TTSVoice { export interface TTSClient { init(): Promise; - speak(ssml: string): AsyncIterable; + speak(ssml: string, signal: AbortSignal): AsyncIterable; pause(): Promise; resume(): Promise; stop(): Promise; diff --git a/apps/readest-app/src/services/tts/TTSController.ts b/apps/readest-app/src/services/tts/TTSController.ts index eda6d438..9305270e 100644 --- a/apps/readest-app/src/services/tts/TTSController.ts +++ b/apps/readest-app/src/services/tts/TTSController.ts @@ -9,12 +9,14 @@ type TTSState = | 'paused' | 'backward-paused' | 'forward-paused' + | 'setrate-paused' | 'setvoice-paused'; export class TTSController extends EventTarget { state: TTSState = 'stopped'; view: FoliateView; #nossmlCnt: number = 0; + #currentSpeakAbortController: AbortController | null = null; ttsRate: number = 1.0; ttsClient: TTSClient; @@ -49,37 +51,44 @@ export class TTSController extends EventTarget { if (!supportedGranularities.includes(granularity)) { granularity = supportedGranularities[0]!; } - await this.ttsClient.stop(); await this.view.initTTS(granularity); } async #speak(ssml: string | undefined | Promise) { - console.log('TTS speak'); - this.state = 'playing'; - ssml = await ssml; - if (!ssml) { - this.#nossmlCnt++; - // FIXME: in case we are at the end of the book, need a better way to handle this - if (this.#nossmlCnt < 10 && this.state === 'playing') { - await this.view.next(1); + await this.stop(); + this.#currentSpeakAbortController = new AbortController(); + const { signal } = this.#currentSpeakAbortController; + + try { + console.log('TTS speak'); + this.state = 'playing'; + ssml = await ssml; + if (!ssml) { + this.#nossmlCnt++; + // FIXME: in case we are at the end of the book, need a better way to handle this + if (this.#nossmlCnt < 10 && this.state === 'playing') { + await this.view.next(1); + await this.forward(); + } + return; + } else { + this.#nossmlCnt = 0; + } + + const iter = await this.ttsClient.speak(ssml, signal); + let lastCode: TTSMessageCode = 'boundary'; + for await (const { code, mark } of iter) { + if (mark && this.state === 'playing') { + this.view.tts?.setMark(mark); + } + lastCode = code; + } + + if (lastCode === 'end' && this.state === 'playing') { await this.forward(); } - return; - } else { - this.#nossmlCnt = 0; - } - - const iter = await this.ttsClient.speak(ssml); - let lastCode: TTSMessageCode = 'boundary'; - for await (const { code, mark } of iter) { - if (mark && this.state === 'playing') { - this.view.tts?.setMark(mark); - } - lastCode = code; - } - - if (lastCode === 'end' && this.state === 'playing') { - await this.forward(); + } finally { + this.#currentSpeakAbortController = null; } } @@ -116,6 +125,9 @@ export class TTSController extends EventTarget { async stop() { this.state = 'stopped'; + if (this.#currentSpeakAbortController) { + this.#currentSpeakAbortController.abort(); + } await this.ttsClient.stop().catch((e) => this.error(e)); } @@ -144,6 +156,7 @@ export class TTSController extends EventTarget { } async setRate(rate: number) { + this.state = 'setrate-paused'; this.ttsRate = rate; await this.ttsClient.setRate(this.ttsRate); } @@ -156,7 +169,6 @@ export class TTSController extends EventTarget { async setVoice(voiceId: string) { this.state = 'setvoice-paused'; - await this.ttsClient.stop(); const useEdgeTTS = !!this.ttsEdgeVoices.find( (voice) => (voiceId === '' || voice.id === voiceId) && !voice.disabled, ); diff --git a/apps/readest-app/src/services/tts/WebSpeechClient.ts b/apps/readest-app/src/services/tts/WebSpeechClient.ts index 9b5e8a56..fdabb475 100644 --- a/apps/readest-app/src/services/tts/WebSpeechClient.ts +++ b/apps/readest-app/src/services/tts/WebSpeechClient.ts @@ -132,9 +132,9 @@ async function* speakWithMarks( const synth = window.speechSynthesis; + const utterance = new SpeechSynthesisUtterance(); for (const mark of marks) { - const utterance = new SpeechSynthesisUtterance(mark.text); - + utterance.text = mark.text; utterance.rate = getRate(); utterance.pitch = getPitch(); const voice = getVoice(); @@ -205,7 +205,7 @@ export class WebSpeechClient implements TTSClient { return this.available; } - async *speak(ssml: string): AsyncGenerator { + async *speak(ssml: string, signal: AbortSignal): AsyncGenerator { const lang = parseSSMLLang(ssml) || 'en'; if (!this.#voice) { const voices = await this.getVoices(lang); @@ -218,6 +218,11 @@ export class WebSpeechClient implements TTSClient { () => this.#pitch, () => this.#voice, )) { + if (signal.aborted) { + console.log('TTS aborted'); + yield { code: 'error', message: 'Aborted' } as TTSMessageEvent; + return; + } if (ev.type === 'boundary') { yield { code: 'boundary', diff --git a/apps/readest-app/src/utils/ui.ts b/apps/readest-app/src/utils/ui.ts new file mode 100644 index 00000000..42047e06 --- /dev/null +++ b/apps/readest-app/src/utils/ui.ts @@ -0,0 +1,28 @@ +export const throttle = ) => void | Promise>( + func: T, + delay: number, +): ((...args: Parameters) => void) => { + let lastCall = 0; + let timeout: ReturnType | null = null; + + return (...args: Parameters): void => { + const now = Date.now(); + const remaining = delay - (now - lastCall); + + const callFunc = () => { + lastCall = Date.now(); + timeout = null; + func(...args); + }; + + if (remaining <= 0) { + if (timeout) { + clearTimeout(timeout); + timeout = null; + } + callFunc(); + } else if (!timeout) { + timeout = setTimeout(callFunc, remaining); + } + }; +};