Huang Xin
545 lines
16 KiB
TypeScript
545 lines
16 KiB
TypeScript
import { FoliateView } from '@/types/view';
|
|
import { AppService } from '@/types/system';
|
|
import { filterSSMLWithLang, parseSSMLMarks } from '@/utils/ssml';
|
|
import { Overlayer } from 'foliate-js/overlayer.js';
|
|
import { TTSGranularity, TTSHighlightOptions, TTSMark, TTSVoice } from './types';
|
|
import { createRejectFilter } from '@/utils/node';
|
|
import { WebSpeechClient } from './WebSpeechClient';
|
|
import { NativeTTSClient } from './NativeTTSClient';
|
|
import { EdgeTTSClient } from './EdgeTTSClient';
|
|
import { TTSUtils } from './TTSUtils';
|
|
import { TTSClient } from './TTSClient';
|
|
import { isValidLang } from '@/utils/lang';
|
|
|
|
type TTSState =
|
|
| 'stopped'
|
|
| 'playing'
|
|
| 'paused'
|
|
| 'stop-paused'
|
|
| 'backward-paused'
|
|
| 'forward-paused'
|
|
| 'setrate-paused'
|
|
| 'setvoice-paused';
|
|
|
|
const HIGHLIGHT_KEY = 'tts-highlight';
|
|
|
|
export class TTSController extends EventTarget {
|
|
appService: AppService | null = null;
|
|
view: FoliateView;
|
|
isAuthenticated: boolean = false;
|
|
preprocessCallback?: (ssml: string) => Promise<string>;
|
|
onSectionChange?: (sectionIndex: number) => Promise<void>;
|
|
#nossmlCnt: number = 0;
|
|
#currentSpeakAbortController: AbortController | null = null;
|
|
#currentSpeakPromise: Promise<void> | null = null;
|
|
#isPreloading: boolean = false;
|
|
#ttsSectionIndex: number = -1;
|
|
|
|
state: TTSState = 'stopped';
|
|
ttsLang: string = '';
|
|
ttsRate: number = 1.0;
|
|
ttsClient: TTSClient;
|
|
ttsWebClient: TTSClient;
|
|
ttsEdgeClient: TTSClient;
|
|
ttsNativeClient: TTSClient | null = null;
|
|
ttsWebVoices: TTSVoice[] = [];
|
|
ttsEdgeVoices: TTSVoice[] = [];
|
|
ttsNativeVoices: TTSVoice[] = [];
|
|
ttsTargetLang: string = '';
|
|
|
|
options: TTSHighlightOptions = { style: 'highlight', color: 'gray' };
|
|
|
|
constructor(
|
|
appService: AppService | null,
|
|
view: FoliateView,
|
|
isAuthenticated: boolean = false,
|
|
preprocessCallback?: (ssml: string) => Promise<string>,
|
|
onSectionChange?: (sectionIndex: number) => Promise<void>,
|
|
) {
|
|
super();
|
|
this.ttsWebClient = new WebSpeechClient(this);
|
|
this.ttsEdgeClient = new EdgeTTSClient(this, appService);
|
|
// TODO: implement native TTS client for iOS and PC
|
|
if (appService?.isAndroidApp) {
|
|
this.ttsNativeClient = new NativeTTSClient(this);
|
|
}
|
|
this.ttsClient = this.ttsWebClient;
|
|
this.appService = appService;
|
|
this.view = view;
|
|
this.isAuthenticated = isAuthenticated;
|
|
this.preprocessCallback = preprocessCallback;
|
|
this.onSectionChange = onSectionChange;
|
|
}
|
|
|
|
async init() {
|
|
const availableClients = [];
|
|
if (await this.ttsEdgeClient.init()) {
|
|
availableClients.push(this.ttsEdgeClient);
|
|
}
|
|
if (this.ttsNativeClient && (await this.ttsNativeClient.init())) {
|
|
availableClients.push(this.ttsNativeClient);
|
|
this.ttsNativeVoices = await this.ttsNativeClient.getAllVoices();
|
|
}
|
|
if (await this.ttsWebClient.init()) {
|
|
availableClients.push(this.ttsWebClient);
|
|
}
|
|
this.ttsClient = availableClients[0] || this.ttsWebClient;
|
|
const preferredClientName = TTSUtils.getPreferredClient();
|
|
if (preferredClientName) {
|
|
const preferredClient = availableClients.find(
|
|
(client) => client.name === preferredClientName,
|
|
);
|
|
if (preferredClient) {
|
|
this.ttsClient = preferredClient;
|
|
}
|
|
}
|
|
this.ttsWebVoices = await this.ttsWebClient.getAllVoices();
|
|
this.ttsEdgeVoices = await this.ttsEdgeClient.getAllVoices();
|
|
}
|
|
|
|
#getHighlighter() {
|
|
return (range: Range) => {
|
|
const { doc, index, overlayer } = this.view.renderer.getContents()[0] as {
|
|
doc: Document;
|
|
index?: number;
|
|
overlayer?: Overlayer;
|
|
};
|
|
if (!doc || index === undefined || index !== this.#ttsSectionIndex) {
|
|
return;
|
|
}
|
|
try {
|
|
const cfi = this.view.getCFI(index, range);
|
|
const visibleRange = this.view.resolveCFI(cfi).anchor(doc);
|
|
const { style, color } = this.options;
|
|
overlayer?.remove(HIGHLIGHT_KEY);
|
|
overlayer?.add(HIGHLIGHT_KEY, visibleRange, Overlayer[style], { color });
|
|
} catch {}
|
|
};
|
|
}
|
|
|
|
#clearHighlighter() {
|
|
const { overlayer } = (this.view.renderer.getContents()?.[0] || {}) as { overlayer: Overlayer };
|
|
overlayer?.remove(HIGHLIGHT_KEY);
|
|
}
|
|
|
|
async initViewTTS(options?: TTSHighlightOptions) {
|
|
if (options) {
|
|
this.options.style = options.style;
|
|
this.options.color = options.color;
|
|
}
|
|
const currentSectionIndex = this.view.renderer.getContents()[0]?.index ?? 0;
|
|
if (this.#ttsSectionIndex === -1) {
|
|
await this.#initTTSForSection(currentSectionIndex);
|
|
}
|
|
}
|
|
|
|
async #initTTSForSection(sectionIndex: number): Promise<boolean> {
|
|
const sections = this.view.book.sections;
|
|
if (!sections || sectionIndex < 0 || sectionIndex >= sections.length) {
|
|
return false;
|
|
}
|
|
|
|
const section = sections[sectionIndex];
|
|
if (!section?.createDocument) {
|
|
return false;
|
|
}
|
|
|
|
this.#ttsSectionIndex = sectionIndex;
|
|
|
|
const currentSection = this.view.renderer.getContents()[0];
|
|
if (currentSection?.index !== sectionIndex) {
|
|
await this.onSectionChange?.(sectionIndex);
|
|
}
|
|
|
|
let doc: Document;
|
|
if (currentSection?.index === sectionIndex && currentSection?.doc) {
|
|
doc = currentSection.doc;
|
|
} else {
|
|
doc = await section.createDocument();
|
|
const html = doc.querySelector('html');
|
|
const lang = html?.getAttribute('lang') || html?.getAttribute('xml:lang') || '';
|
|
if (html && !isValidLang(lang) && this.ttsLang) {
|
|
html.setAttribute('lang', this.ttsLang);
|
|
html.setAttribute('xml:lang', this.ttsLang);
|
|
}
|
|
}
|
|
|
|
if (this.view.tts && this.view.tts.doc === doc) {
|
|
return true;
|
|
}
|
|
|
|
const { TTS } = await import('foliate-js/tts.js');
|
|
const { textWalker } = await import('foliate-js/text-walker.js');
|
|
let granularity: TTSGranularity = this.view.language.isCJK ? 'sentence' : 'word';
|
|
const supportedGranularities = this.ttsClient.getGranularities();
|
|
if (!supportedGranularities.includes(granularity)) {
|
|
granularity = supportedGranularities[0]!;
|
|
}
|
|
|
|
this.view.tts = new TTS(
|
|
doc,
|
|
textWalker,
|
|
createRejectFilter({
|
|
tags: ['rt'],
|
|
contents: [{ tag: 'a', content: /^[\[\(]?[\*\d]+[\)\]]?$/ }],
|
|
}),
|
|
this.#getHighlighter(),
|
|
granularity,
|
|
);
|
|
console.log(`Initialized TTS for section ${sectionIndex}`);
|
|
|
|
return true;
|
|
}
|
|
|
|
async #initTTSForNextSection(): Promise<boolean> {
|
|
const nextIndex = this.#ttsSectionIndex + 1;
|
|
const sections = this.view.book.sections;
|
|
|
|
if (!sections || nextIndex >= sections.length) {
|
|
return false;
|
|
}
|
|
|
|
return await this.#initTTSForSection(nextIndex);
|
|
}
|
|
|
|
async #initTTSForPrevSection(): Promise<boolean> {
|
|
const prevIndex = this.#ttsSectionIndex - 1;
|
|
|
|
if (prevIndex < 0) {
|
|
return false;
|
|
}
|
|
|
|
return await this.#initTTSForSection(prevIndex);
|
|
}
|
|
|
|
async #handleNavigationWithSSML(ssml: string | undefined, isPlaying: boolean) {
|
|
if (isPlaying) this.#speak(ssml);
|
|
}
|
|
|
|
async #handleNavigationWithoutSSML(initSection: () => Promise<boolean>, isPlaying: boolean) {
|
|
if (await initSection()) {
|
|
if (isPlaying) {
|
|
this.#speak(this.view.tts?.start());
|
|
} else {
|
|
this.view.tts?.start();
|
|
}
|
|
} else {
|
|
await this.stop();
|
|
}
|
|
}
|
|
|
|
async preloadSSML(ssml: string | undefined, signal: AbortSignal) {
|
|
if (!ssml) return;
|
|
const iter = await this.ttsClient.speak(ssml, signal, true);
|
|
for await (const _ of iter);
|
|
}
|
|
|
|
async preloadNextSSML(count: number = 4) {
|
|
const tts = this.view.tts;
|
|
if (!tts) return;
|
|
|
|
this.#isPreloading = true;
|
|
const ssmls: string[] = [];
|
|
for (let i = 0; i < count; i++) {
|
|
const ssml = await this.#preprocessSSML(tts.next());
|
|
if (!ssml) break;
|
|
ssmls.push(ssml);
|
|
}
|
|
for (let i = 0; i < ssmls.length; i++) {
|
|
tts.prev();
|
|
}
|
|
this.#isPreloading = false;
|
|
await Promise.all(ssmls.map((ssml) => this.preloadSSML(ssml, new AbortController().signal)));
|
|
}
|
|
|
|
async #preprocessSSML(ssml?: string) {
|
|
if (!ssml) return;
|
|
ssml = ssml
|
|
.replace(/<emphasis[^>]*>([^<]+)<\/emphasis>/g, '$1')
|
|
.replace(/[–—]/g, ',')
|
|
.replace('<break/>', ' ')
|
|
.replace(/\.{3,}/g, ' ')
|
|
.replace(/……/g, ' ')
|
|
.replace(/\*/g, ' ')
|
|
.replace(/·/g, ' ');
|
|
|
|
if (this.ttsTargetLang) {
|
|
ssml = filterSSMLWithLang(ssml, this.ttsTargetLang);
|
|
}
|
|
|
|
if (this.preprocessCallback) {
|
|
ssml = await this.preprocessCallback(ssml);
|
|
}
|
|
|
|
return ssml;
|
|
}
|
|
|
|
async #speak(ssml: string | undefined | Promise<string>, oneTime = false) {
|
|
await this.stop();
|
|
this.#currentSpeakAbortController = new AbortController();
|
|
const { signal } = this.#currentSpeakAbortController;
|
|
|
|
this.#currentSpeakPromise = new Promise(async (resolve, reject) => {
|
|
try {
|
|
console.log('TTS speak');
|
|
this.state = 'playing';
|
|
|
|
signal.addEventListener('abort', () => {
|
|
resolve();
|
|
});
|
|
|
|
ssml = await this.#preprocessSSML(await ssml);
|
|
if (!ssml) {
|
|
this.#nossmlCnt++;
|
|
// FIXME: in case we are at the end of the book, need a better way to handle this
|
|
if (this.#nossmlCnt < 10 && this.state === 'playing' && !oneTime) {
|
|
resolve();
|
|
if (await this.#initTTSForNextSection()) {
|
|
await this.forward();
|
|
} else {
|
|
await this.stop();
|
|
}
|
|
}
|
|
console.log('no SSML, skipping for', this.#nossmlCnt);
|
|
return;
|
|
} else {
|
|
this.#nossmlCnt = 0;
|
|
}
|
|
|
|
const { plainText, marks } = parseSSMLMarks(ssml);
|
|
if (!oneTime) {
|
|
if (!plainText || marks.length === 0) {
|
|
resolve();
|
|
return await this.forward();
|
|
} else {
|
|
this.dispatchSpeakMark(marks[0]);
|
|
}
|
|
await this.preloadSSML(ssml, signal);
|
|
}
|
|
const iter = await this.ttsClient.speak(ssml, signal);
|
|
let lastCode;
|
|
for await (const { code } of iter) {
|
|
if (signal.aborted) {
|
|
resolve();
|
|
return;
|
|
}
|
|
lastCode = code;
|
|
}
|
|
|
|
if (lastCode === 'end' && this.state === 'playing' && !oneTime) {
|
|
resolve();
|
|
await this.forward();
|
|
}
|
|
resolve();
|
|
} catch (e) {
|
|
if (signal.aborted) {
|
|
resolve();
|
|
} else {
|
|
reject(e);
|
|
}
|
|
} finally {
|
|
if (this.#currentSpeakAbortController) {
|
|
this.#currentSpeakAbortController.abort();
|
|
this.#currentSpeakAbortController = null;
|
|
}
|
|
}
|
|
});
|
|
|
|
await this.#currentSpeakPromise.catch((e) => this.error(e));
|
|
}
|
|
|
|
async speak(ssml: string | Promise<string>, oneTime = false, oneTimeCallback?: () => void) {
|
|
await this.initViewTTS();
|
|
this.#speak(ssml, oneTime)
|
|
.then(() => {
|
|
if (oneTime && oneTimeCallback) {
|
|
oneTimeCallback();
|
|
}
|
|
})
|
|
.catch((e) => this.error(e));
|
|
if (!oneTime) {
|
|
this.preloadNextSSML();
|
|
this.dispatchSpeakMark();
|
|
}
|
|
}
|
|
|
|
play() {
|
|
if (this.state !== 'playing') {
|
|
this.start();
|
|
} else {
|
|
this.pause();
|
|
}
|
|
}
|
|
|
|
async start() {
|
|
await this.initViewTTS();
|
|
const ssml = this.state.includes('paused') ? this.view.tts?.resume() : this.view.tts?.start();
|
|
if (this.state.includes('paused')) {
|
|
this.resume();
|
|
}
|
|
this.#speak(ssml);
|
|
this.preloadNextSSML();
|
|
}
|
|
|
|
async pause() {
|
|
this.state = 'paused';
|
|
if (!(await this.ttsClient.pause().catch((e) => this.error(e)))) {
|
|
await this.stop();
|
|
this.state = 'stop-paused';
|
|
}
|
|
}
|
|
|
|
async resume() {
|
|
this.state = 'playing';
|
|
await this.ttsClient.resume().catch((e) => this.error(e));
|
|
}
|
|
|
|
async stop() {
|
|
if (this.#currentSpeakAbortController) {
|
|
this.#currentSpeakAbortController.abort();
|
|
}
|
|
await this.ttsClient.stop().catch((e) => this.error(e));
|
|
|
|
if (this.#currentSpeakPromise) {
|
|
const timeout = new Promise((_, reject) =>
|
|
setTimeout(() => reject(new Error('Stop operation timed out')), 3000),
|
|
);
|
|
await Promise.race([this.#currentSpeakPromise.catch((e) => this.error(e)), timeout]).catch(
|
|
(e) => this.error(e),
|
|
);
|
|
this.#currentSpeakPromise = null;
|
|
}
|
|
this.state = 'stopped';
|
|
}
|
|
|
|
// goto previous mark/paragraph
|
|
async backward(byMark = false) {
|
|
await this.initViewTTS();
|
|
const isPlaying = this.state === 'playing';
|
|
await this.stop();
|
|
if (!isPlaying) this.state = 'backward-paused';
|
|
|
|
const ssml = byMark ? this.view.tts?.prevMark(!isPlaying) : this.view.tts?.prev(!isPlaying);
|
|
if (!ssml) {
|
|
await this.#handleNavigationWithoutSSML(() => this.#initTTSForPrevSection(), isPlaying);
|
|
} else {
|
|
await this.#handleNavigationWithSSML(ssml, isPlaying);
|
|
}
|
|
}
|
|
|
|
// goto next mark/paragraph
|
|
async forward(byMark = false) {
|
|
await this.initViewTTS();
|
|
const isPlaying = this.state === 'playing';
|
|
await this.stop();
|
|
if (!isPlaying) this.state = 'forward-paused';
|
|
|
|
const ssml = byMark ? this.view.tts?.nextMark(!isPlaying) : this.view.tts?.next(!isPlaying);
|
|
if (!ssml) {
|
|
await this.#handleNavigationWithoutSSML(() => this.#initTTSForNextSection(), isPlaying);
|
|
} else {
|
|
await this.#handleNavigationWithSSML(ssml, isPlaying);
|
|
}
|
|
if (isPlaying && !byMark) this.preloadNextSSML();
|
|
}
|
|
|
|
async setLang(lang: string) {
|
|
this.ttsLang = lang;
|
|
this.setPrimaryLang(lang);
|
|
}
|
|
|
|
async setPrimaryLang(lang: string) {
|
|
if (this.ttsEdgeClient.initialized) this.ttsEdgeClient.setPrimaryLang(lang);
|
|
if (this.ttsWebClient.initialized) this.ttsWebClient.setPrimaryLang(lang);
|
|
if (this.ttsNativeClient?.initialized) this.ttsNativeClient?.setPrimaryLang(lang);
|
|
}
|
|
|
|
async setRate(rate: number) {
|
|
this.state = 'setrate-paused';
|
|
this.ttsRate = rate;
|
|
await this.ttsClient.setRate(this.ttsRate);
|
|
}
|
|
|
|
async getVoices(lang: string) {
|
|
const ttsWebVoices = await this.ttsWebClient.getVoices(lang);
|
|
const ttsEdgeVoices = await this.ttsEdgeClient.getVoices(lang);
|
|
const ttsNativeVoices = (await this.ttsNativeClient?.getVoices(lang)) ?? [];
|
|
|
|
const voicesGroups = [...ttsNativeVoices, ...ttsEdgeVoices, ...ttsWebVoices];
|
|
return voicesGroups;
|
|
}
|
|
|
|
async setVoice(voiceId: string, lang: string) {
|
|
this.state = 'setvoice-paused';
|
|
const useEdgeTTS = !!this.ttsEdgeVoices.find(
|
|
(voice) => (voiceId === '' || voice.id === voiceId) && !voice.disabled,
|
|
);
|
|
const useNativeTTS = !!this.ttsNativeVoices.find(
|
|
(voice) => (voiceId === '' || voice.id === voiceId) && !voice.disabled,
|
|
);
|
|
if (useEdgeTTS) {
|
|
this.ttsClient = this.ttsEdgeClient;
|
|
await this.ttsClient.setRate(this.ttsRate);
|
|
} else if (useNativeTTS) {
|
|
if (!this.ttsNativeClient) {
|
|
throw new Error('Native TTS client is not available');
|
|
}
|
|
this.ttsClient = this.ttsNativeClient;
|
|
await this.ttsClient.setRate(this.ttsRate);
|
|
} else {
|
|
this.ttsClient = this.ttsWebClient;
|
|
await this.ttsClient.setRate(this.ttsRate);
|
|
}
|
|
TTSUtils.setPreferredClient(this.ttsClient.name);
|
|
TTSUtils.setPreferredVoice(this.ttsClient.name, lang, voiceId);
|
|
await this.ttsClient.setVoice(voiceId);
|
|
}
|
|
|
|
getVoiceId() {
|
|
return this.ttsClient.getVoiceId();
|
|
}
|
|
|
|
getSpeakingLang() {
|
|
return this.ttsClient.getSpeakingLang();
|
|
}
|
|
|
|
setTargetLang(lang: string) {
|
|
this.ttsTargetLang = lang;
|
|
}
|
|
|
|
dispatchSpeakMark(mark?: TTSMark) {
|
|
this.dispatchEvent(new CustomEvent('tts-speak-mark', { detail: mark || { text: '' } }));
|
|
if (mark && mark.name !== '-1') {
|
|
if (this.#isPreloading) {
|
|
setTimeout(() => this.dispatchSpeakMark(mark), 500);
|
|
} else {
|
|
const range = this.view.tts?.setMark(mark.name);
|
|
try {
|
|
const cfi = this.view.getCFI(this.#ttsSectionIndex, range);
|
|
this.dispatchEvent(new CustomEvent('tts-highlight-mark', { detail: { cfi } }));
|
|
} catch {}
|
|
}
|
|
}
|
|
}
|
|
|
|
error(e: unknown) {
|
|
console.error(e);
|
|
this.state = 'stopped';
|
|
}
|
|
|
|
async shutdown() {
|
|
await this.stop();
|
|
this.#clearHighlighter();
|
|
this.#ttsSectionIndex = -1;
|
|
if (this.ttsWebClient.initialized) {
|
|
await this.ttsWebClient.shutdown();
|
|
}
|
|
if (this.ttsEdgeClient.initialized) {
|
|
await this.ttsEdgeClient.shutdown();
|
|
}
|
|
if (this.ttsNativeClient?.initialized) {
|
|
await this.ttsNativeClient.shutdown();
|
|
}
|
|
}
|
|
}
|