akai-tools/readest
2
0
Fork 0
forked from akai/readest
Code Pull Requests Activity

Fix TTS on iOS browsers, closes #172 (#198)

Browse Source
This commit is contained in:
Huang Xin
2025-01-20 01:53:12 +01:00
committed by GitHub
parent ca2a62a921
commit 87069c7308
4 changed files with 106 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/readest-app/src/app/reader/components/tts/TTSControl.tsx
+6 -14
View File
@@ -162,13 +162,9 @@ const TTSControl = () => {
throttle(async (rate: number) => {
const ttsController = ttsControllerRef.current;
if (ttsController) {
if (ttsController.state === 'playing') {
await ttsController.stop();
await ttsController.setRate(rate);
await ttsController.start();
} else {
await ttsController.setRate(rate);
}
await ttsController.stop();
await ttsController.setRate(rate);
await ttsController.start();
}
}, 3000),
[],
@@ -179,13 +175,9 @@ const TTSControl = () => {
throttle(async (voice: string) => {
const ttsController = ttsControllerRef.current;
if (ttsController) {
if (ttsController.state === 'playing') {
await ttsController.stop();
await ttsController.setVoice(voice);
await ttsController.start();
} else {
await ttsController.setVoice(voice);
}
await ttsController.stop();
await ttsController.setVoice(voice);
await ttsController.start();
}
}, 3000),
[],
apps/readest-app/src/libs/edgeTTS.ts
+2 -2
View File
@@ -144,6 +144,7 @@ const hashPayload = (payload: EdgeTTSPayload): string => {
export class EdgeSpeechTTS {
static voices = genVoiceList(EDGE_TTS_VOICES);
private static audioCache = new LRUCache<string, AudioBuffer>(200);
private audioContext = new AudioContext();
constructor() {}
@@ -274,8 +275,7 @@ export class EdgeSpeechTTS {
try {
const res = await this.create(payload);
const arrayBuffer = await res.arrayBuffer();
const audioContext = new AudioContext();
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer.slice(0));
const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer.slice(0));
EdgeSpeechTTS.audioCache.set(cacheKey, audioBuffer);
return audioBuffer;
} catch (error) {
apps/readest-app/src/services/tts/EdgeTTSClient.ts
+39 -18
View File
@@ -11,7 +11,7 @@ export class EdgeTTSClient implements TTSClient {
#voices: TTSVoice[] = [];
#edgeTTS: EdgeSpeechTTS;
#audioContext: AudioContext | null = null;
static #audioContext: AudioContext | null;
#sourceNode: AudioBufferSourceNode | null = null;
#isPlaying = false;
#pausedAt = 0;
@@ -23,9 +23,19 @@ export class EdgeTTSClient implements TTSClient {
this.#edgeTTS = new EdgeSpeechTTS();
}
async initializeAudioContext() {
if (!EdgeTTSClient.#audioContext) {
EdgeTTSClient.#audioContext = new AudioContext();
}
if (EdgeTTSClient.#audioContext.state === 'suspended') {
await EdgeTTSClient.#audioContext.resume();
}
}
async init() {
this.#voices = EdgeSpeechTTS.voices;
try {
await this.initializeAudioContext();
await this.#edgeTTS.create({
lang: 'en',
text: 'test',
@@ -88,10 +98,12 @@ export class EdgeTTSClient implements TTSClient {
this.#audioBuffer = await this.#edgeTTS.createAudio(
this.getPayload(lang, mark.text, voiceId),
);
this.#audioContext = new AudioContext();
this.#sourceNode = this.#audioContext.createBufferSource();
if (!EdgeTTSClient.#audioContext) {
EdgeTTSClient.#audioContext = new AudioContext();
}
this.#sourceNode = EdgeTTSClient.#audioContext.createBufferSource();
this.#sourceNode.buffer = this.#audioBuffer;
this.#sourceNode.connect(this.#audioContext.destination);
this.#sourceNode.connect(EdgeTTSClient.#audioContext.destination);
yield {
code: 'boundary',
@@ -100,11 +112,18 @@ export class EdgeTTSClient implements TTSClient {
};
const result = await new Promise<TTSMessageEvent>((resolve) => {
if (this.#audioContext === null || this.#sourceNode === null) {
if (EdgeTTSClient.#audioContext === null || this.#sourceNode === null) {
throw new Error('Audio context or source node is null');
}
this.#sourceNode.onended = () => {
this.#isPlaying = false;
this.#sourceNode.onended = (event: Event) => {
// chunk finished speaking or aborted speaking
if (signal.aborted || event.type === 'stopped') {
resolve({
code: 'error',
message: 'Aborted',
});
return;
}
resolve({
code: 'end',
message: `Chunk finished: ${mark.name}`,
@@ -117,9 +136,12 @@ export class EdgeTTSClient implements TTSClient {
});
return;
}
if (EdgeTTSClient.#audioContext.state === 'suspended') {
EdgeTTSClient.#audioContext.resume();
}
this.#sourceNode.start(0);
this.#isPlaying = true;
this.#startedAt = this.#audioContext.currentTime;
this.#startedAt = EdgeTTSClient.#audioContext.currentTime;
});
yield result;
} catch (error) {
@@ -144,17 +166,17 @@ export class EdgeTTSClient implements TTSClient {
}
async pause() {
if (!this.#isPlaying || !this.#audioContext || !this.#sourceNode) return;
this.#pausedAt = this.#audioContext.currentTime - this.#startedAt;
await this.#audioContext.suspend();
if (!this.#isPlaying || !EdgeTTSClient.#audioContext) return;
this.#pausedAt = EdgeTTSClient.#audioContext.currentTime - this.#startedAt;
await EdgeTTSClient.#audioContext.suspend();
this.#isPlaying = false;
}
async resume() {
if (this.#isPlaying || !this.#audioContext || !this.#sourceNode) return;
await this.#audioContext.resume();
if (this.#isPlaying || !EdgeTTSClient.#audioContext) return;
await EdgeTTSClient.#audioContext.resume();
this.#isPlaying = true;
this.#startedAt = this.#audioContext.currentTime - this.#pausedAt;
this.#startedAt = EdgeTTSClient.#audioContext.currentTime - this.#pausedAt;
}
async stop() {
@@ -168,6 +190,9 @@ export class EdgeTTSClient implements TTSClient {
if (this.#sourceNode) {
try {
this.#sourceNode.stop();
if (this.#sourceNode?.onended) {
this.#sourceNode.onended(new Event('stopped'));
}
} catch (err) {
if (!(err instanceof Error) || err.name !== 'InvalidStateError') {
console.log('Error stopping source node:', err);
@@ -176,10 +201,6 @@ export class EdgeTTSClient implements TTSClient {
this.#sourceNode.disconnect();
this.#sourceNode = null;
}
if (this.#audioContext) {
await this.#audioContext.close();
this.#audioContext = null;
}
this.#audioBuffer = null;
}
apps/readest-app/src/services/tts/TTSController.ts
+59 -33
View File
@@ -17,6 +17,7 @@ export class TTSController extends EventTarget {
view: FoliateView;
#nossmlCnt: number = 0;
#currentSpeakAbortController: AbortController | null = null;
#currentSpeakPromise: Promise<void> | null = null;
ttsRate: number = 1.0;
ttsClient: TTSClient;
@@ -61,7 +62,7 @@ export class TTSController extends EventTarget {
for await (const _ of iter);
}
async preloadNextSSML(count: number) {
async preloadNextSSML(count: number = 2) {
const tts = this.view.tts;
if (!tts) return;
let preloaded = 0;
@@ -80,44 +81,60 @@ export class TTSController extends EventTarget {
this.#currentSpeakAbortController = new AbortController();
const { signal } = this.#currentSpeakAbortController;
try {
console.log('TTS speak');
this.state = 'playing';
ssml = await ssml;
await this.preloadSSML(ssml);
if (!ssml) {
this.#nossmlCnt++;
// FIXME: in case we are at the end of the book, need a better way to handle this
if (this.#nossmlCnt < 10 && this.state === 'playing') {
await this.view.next(1);
this.#currentSpeakPromise = new Promise(async (resolve, reject) => {
try {
console.log('TTS speak');
this.state = 'playing';
ssml = await ssml;
await this.preloadSSML(ssml);
if (!ssml) {
this.#nossmlCnt++;
// FIXME: in case we are at the end of the book, need a better way to handle this
if (this.#nossmlCnt < 10 && this.state === 'playing') {
await this.view.next(1);
await this.forward();
}
return;
} else {
this.#nossmlCnt = 0;
}
const iter = await this.ttsClient.speak(ssml, signal);
let lastCode: TTSMessageCode = 'boundary';
for await (const { code, mark } of iter) {
if (signal.aborted) {
resolve();
return;
}
if (mark && this.state === 'playing') {
this.view.tts?.setMark(mark);
}
lastCode = code;
}
if (lastCode === 'end' && this.state === 'playing') {
resolve();
await this.forward();
}
return;
} else {
this.#nossmlCnt = 0;
}
const iter = await this.ttsClient.speak(ssml, signal);
let lastCode: TTSMessageCode = 'boundary';
for await (const { code, mark } of iter) {
if (mark && this.state === 'playing') {
this.view.tts?.setMark(mark);
resolve();
} catch (e) {
if (signal.aborted) {
resolve();
} else {
reject(e);
}
lastCode = code;
} finally {
this.#currentSpeakAbortController = null;
this.#currentSpeakPromise = null;
}
if (lastCode === 'end' && this.state === 'playing') {
await this.forward();
}
} finally {
this.#currentSpeakAbortController = null;
}
});
await this.#currentSpeakPromise.catch((e) => this.error(e));
}
async speak(ssml: string | Promise<string>) {
await this.initViewTTS();
this.#speak(ssml).catch((e) => this.error(e));
this.preloadNextSSML(2);
this.preloadNextSSML();
}
play() {
@@ -131,8 +148,11 @@ export class TTSController extends EventTarget {
async start() {
await this.initViewTTS();
const ssml = this.state.includes('paused') ? this.view.tts?.resume() : this.view.tts?.start();
if (this.state.includes('paused')) {
this.resume();
}
this.#speak(ssml);
this.preloadNextSSML(2);
this.preloadNextSSML();
}
async pause() {
@@ -146,11 +166,15 @@ export class TTSController extends EventTarget {
}
async stop() {
this.state = 'stopped';
if (this.#currentSpeakAbortController) {
this.#currentSpeakAbortController.abort();
}
await this.ttsClient.stop().catch((e) => this.error(e));
if (this.#currentSpeakPromise) {
await this.#currentSpeakPromise.catch((e) => this.error(e));
}
this.state = 'stopped';
}
// goto previous sentence
@@ -160,6 +184,7 @@ export class TTSController extends EventTarget {
await this.stop();
this.#speak(this.view.tts?.prev());
} else {
await this.stop();
this.state = 'backward-paused';
this.view.tts?.prev(true);
}
@@ -171,8 +196,9 @@ export class TTSController extends EventTarget {
if (this.state === 'playing') {
await this.stop();
this.#speak(this.view.tts?.next());
this.preloadNextSSML(2);
this.preloadNextSSML();
} else {
await this.stop();
this.state = 'forward-paused';
this.view.tts?.next(true);
}