diff --git a/apps/readest-app/src/__tests__/libs/edgeTTS-word-boundaries.test.ts b/apps/readest-app/src/__tests__/libs/edgeTTS-word-boundaries.test.ts new file mode 100644 index 00000000..779bc949 --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/edgeTTS-word-boundaries.test.ts @@ -0,0 +1,216 @@ +import { describe, test, expect, vi, beforeEach } from 'vitest'; + +// Controllable WebSocket fake for the browser (isomorphic-ws) transport. +const wsState = vi.hoisted(() => ({ + instances: [] as Array<{ + url: string; + binaryType: string; + listeners: Record void>>; + sent: unknown[]; + emit: (type: string, event?: unknown) => void; + }>, +})); + +vi.mock('isomorphic-ws', () => ({ + default: class MockWebSocket { + url: string; + opts?: unknown; + binaryType = ''; + listeners: Record void>> = {}; + sent: unknown[] = []; + constructor(url: string, opts?: unknown) { + this.url = url; + this.opts = opts; + wsState.instances.push(this); + } + addEventListener(type: string, cb: (event: unknown) => void) { + (this.listeners[type] ??= []).push(cb); + } + send(data: unknown) { + this.sent.push(data); + } + close() {} + emit(type: string, event?: unknown) { + for (const cb of this.listeners[type] ?? []) cb(event); + } + }, +})); + +vi.mock('@/services/environment', () => ({ + getAPIBaseUrl: () => 'http://localhost/api', + isTauriAppPlatform: () => false, +})); + +vi.mock('@/utils/supabase', () => ({ + supabase: { auth: { getSession: async () => ({ data: { session: null } }) } }, + createSupabaseClient: () => ({}), + createSupabaseAdminClient: () => ({}), +})); + +// Controllable stub for the authenticated HTTPS proxy fetch. +const httpState = vi.hoisted(() => ({ + headers: {} as Record, + body: new Uint8Array([1, 2, 3]), +})); +vi.mock('@/utils/fetch', () => ({ + fetchWithAuth: vi.fn( + async () => new Response(httpState.body, { status: 200, headers: httpState.headers }), + ), +})); + +const makeBinaryAudioFrame = (audio: Uint8Array) => { + const header = new TextEncoder().encode('Path:audio\r\n'); + const buf = new ArrayBuffer(2 + header.length + audio.length); + new DataView(buf).setInt16(0, header.length); + new Uint8Array(buf).set(header, 2); + new Uint8Array(buf).set(audio, 2 + header.length); + return buf; +}; + +const makeMetadataFrame = (text: string, offset: number, duration: number) => + 'X-RequestId:abc\r\nContent-Type:application/json; charset=utf-8\r\nPath:audio.metadata\r\n\r\n' + + JSON.stringify({ + Metadata: [ + { + Type: 'WordBoundary', + Data: { Offset: offset, Duration: duration, text: { Text: text, Length: text.length } }, + }, + ], + }); + +describe('EdgeSpeechTTS.createAudio word boundaries (browser WebSocket path)', () => { + beforeEach(() => { + wsState.instances.length = 0; + (URL as unknown as { createObjectURL?: (blob: Blob) => string }).createObjectURL = vi.fn( + () => 'blob:mock-object-url', + ); + }); + + test('captures word boundaries from audio.metadata frames and caches them', async () => { + const { EdgeSpeechTTS } = await import('@/libs/edgeTTS'); + const tts = new EdgeSpeechTTS('wss'); + const payload = { + lang: 'en', + text: 'Hello brave world', + voice: 'en-US-AriaNeural', + rate: 1.0, + pitch: 1.0, + }; + + const promise = tts.createAudio(payload); + await vi.waitFor(() => expect(wsState.instances.length).toBe(1)); + const ws = wsState.instances[0]!; + // In a browser (jsdom has `window`), the WebSocket constructor must be + // called WITHOUT an options argument: native WebSocket treats a second + // argument as subprotocols and throws SyntaxError on an options object. + expect((ws as unknown as { opts?: unknown }).opts).toBeUndefined(); + ws.emit('open'); + ws.emit('message', { data: makeMetadataFrame('Hello', 1000000, 4000000) }); + ws.emit('message', { data: makeBinaryAudioFrame(new Uint8Array([1, 2, 3, 4])) }); + ws.emit('message', { data: makeMetadataFrame('brave', 6000000, 4000000) }); + ws.emit('message', { data: 'Path:turn.end\r\n\r\n' }); + + const { url, boundaries } = await promise; + expect(url).toBe('blob:mock-object-url'); + expect(boundaries).toEqual([ + { offset: 1000000, duration: 4000000, text: 'Hello' }, + { offset: 6000000, duration: 4000000, text: 'brave' }, + ]); + + // A second call for the same payload is served from the cache: no new + // WebSocket connection, same boundaries. + const cached = await tts.createAudio(payload); + expect(wsState.instances.length).toBe(1); + expect(cached.url).toBe('blob:mock-object-url'); + expect(cached.boundaries).toEqual(boundaries); + }); + + test('resolves with empty boundaries when no metadata frames arrive', async () => { + const { EdgeSpeechTTS } = await import('@/libs/edgeTTS'); + const tts = new EdgeSpeechTTS('wss'); + + const promise = tts.createAudio({ + lang: 'en', + text: 'No metadata here', + voice: 'en-US-AriaNeural', + rate: 1.0, + pitch: 1.0, + }); + await vi.waitFor(() => expect(wsState.instances.length).toBe(1)); + const ws = wsState.instances[0]!; + ws.emit('open'); + ws.emit('message', { data: makeBinaryAudioFrame(new Uint8Array([9, 9])) }); + ws.emit('message', { data: 'Path:turn.end\r\n\r\n' }); + + const { boundaries } = await promise; + expect(boundaries).toEqual([]); + }); +}); + +describe('word-boundary header (de)serialization', () => { + test('round-trips boundaries, ASCII-safe for HTTP headers, incl. non-ASCII text', async () => { + const { serializeWordBoundaries, parseWordBoundariesHeader } = await import('@/libs/edgeTTS'); + const boundaries = [ + { offset: 1000000, duration: 4000000, text: 'Hello' }, + { offset: 6000000, duration: 4000000, text: 'café—世界' }, + ]; + const header = serializeWordBoundaries(boundaries); + // HTTP header values must be ASCII; non-ASCII text would corrupt the header. + expect([...header].every((c) => c.charCodeAt(0) < 128)).toBe(true); + expect(parseWordBoundariesHeader(header)).toEqual(boundaries); + }); + + test('parse returns [] for null, malformed, or non-boundary payloads', async () => { + const { parseWordBoundariesHeader } = await import('@/libs/edgeTTS'); + expect(parseWordBoundariesHeader(null)).toEqual([]); + expect(parseWordBoundariesHeader('not-json')).toEqual([]); + expect(parseWordBoundariesHeader(encodeURIComponent(JSON.stringify({ x: 1 })))).toEqual([]); + expect(parseWordBoundariesHeader(encodeURIComponent(JSON.stringify([{ text: 'x' }])))).toEqual( + [], + ); + }); +}); + +describe('EdgeSpeechTTS.createAudio over the HTTPS proxy (word boundaries via header)', () => { + beforeEach(() => { + httpState.headers = {}; + httpState.body = new Uint8Array([1, 2, 3]); + (URL as unknown as { createObjectURL?: (blob: Blob) => string }).createObjectURL = vi.fn( + () => 'blob:mock-object-url', + ); + }); + + test('parses word boundaries from the X-TTS-Word-Boundaries response header', async () => { + const { EdgeSpeechTTS, serializeWordBoundaries, WORD_BOUNDARIES_HEADER } = await import( + '@/libs/edgeTTS' + ); + const boundaries = [ + { offset: 1000000, duration: 4000000, text: 'Hello' }, + { offset: 6000000, duration: 4000000, text: 'world' }, + ]; + httpState.headers = { [WORD_BOUNDARIES_HEADER]: serializeWordBoundaries(boundaries) }; + + const tts = new EdgeSpeechTTS('https'); + const result = await tts.createAudio({ + lang: 'en', + text: 'Hello world https', + voice: 'en-US-AriaNeural', + rate: 1.0, + pitch: 1.0, + }); + expect(result.boundaries).toEqual(boundaries); + }); + + test('returns empty boundaries when the proxy omits the header', async () => { + const { EdgeSpeechTTS } = await import('@/libs/edgeTTS'); + const tts = new EdgeSpeechTTS('https'); + const result = await tts.createAudio({ + lang: 'en', + text: 'No header https', + voice: 'en-US-AriaNeural', + rate: 1.0, + pitch: 1.0, + }); + expect(result.boundaries).toEqual([]); + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/edgeTTS.test.ts b/apps/readest-app/src/__tests__/libs/edgeTTS.test.ts index c9cd1e1f..a9ca60da 100644 --- a/apps/readest-app/src/__tests__/libs/edgeTTS.test.ts +++ b/apps/readest-app/src/__tests__/libs/edgeTTS.test.ts @@ -213,4 +213,115 @@ describe('EdgeSpeechTTS on Cloudflare Workers', () => { }), ).rejects.toThrow(); }); + + test('captures word boundaries from audio.metadata frames', async () => { + const listeners: Record void>> = {}; + const mockSocket = { + accept: vi.fn(), + send: vi.fn(), + close: vi.fn(), + addEventListener: vi.fn((type: string, cb: (event: unknown) => void) => { + (listeners[type] ??= []).push(cb); + }), + removeEventListener: vi.fn(), + }; + + const audioFrame = () => { + const headerText = 'X-RequestId:1\r\nContent-Type:audio/mpeg\r\nPath:audio\r\n'; + const headerBytes = new TextEncoder().encode(headerText); + const audioBody = new Uint8Array([0xaa, 0xbb]); + const frame = new Uint8Array(2 + headerBytes.byteLength + audioBody.byteLength); + new DataView(frame.buffer).setInt16(0, headerBytes.byteLength); + frame.set(headerBytes, 2); + frame.set(audioBody, 2 + headerBytes.byteLength); + return frame.buffer; + }; + const metadataFrame = (text: string, offset: number) => + 'X-RequestId:1\r\nContent-Type:application/json; charset=utf-8\r\nPath:audio.metadata\r\n\r\n' + + JSON.stringify({ + Metadata: [ + { Type: 'WordBoundary', Data: { Offset: offset, Duration: 1000, text: { Text: text } } }, + ], + }); + + let sendCount = 0; + mockSocket.send.mockImplementation(() => { + sendCount++; + if (sendCount === 2) { + queueMicrotask(() => { + for (const cb of listeners['message'] ?? []) + cb({ data: metadataFrame('Hello', 1000000) }); + for (const cb of listeners['message'] ?? []) cb({ data: audioFrame() }); + for (const cb of listeners['message'] ?? []) + cb({ data: metadataFrame('brave', 6000000) }); + for (const cb of listeners['message'] ?? []) + cb({ data: 'X-RequestId:1\r\nPath:turn.end\r\n\r\n' }); + }); + } + }); + + globalThis.fetch = vi + .fn() + .mockResolvedValue({ status: 101, webSocket: mockSocket }) as unknown as typeof fetch; + + const { EdgeSpeechTTS } = await import('@/libs/edgeTTS'); + const tts = new EdgeSpeechTTS('wss'); + const { boundaries } = await tts.createWithBoundaries({ + lang: 'en-US', + text: 'Hello brave', + voice: 'en-US-AriaNeural', + rate: 1.0, + pitch: 1.0, + }); + expect(boundaries).toEqual([ + { offset: 1000000, duration: 1000, text: 'Hello' }, + { offset: 6000000, duration: 1000, text: 'brave' }, + ]); + }); +}); + +describe('parseAudioMetadataBody', () => { + test('extracts WordBoundary entries from an audio.metadata body', async () => { + const { parseAudioMetadataBody } = await import('@/libs/edgeTTS'); + const body = JSON.stringify({ + Metadata: [ + { + Type: 'WordBoundary', + Data: { + Offset: 1000000, + Duration: 4250000, + text: { Text: 'Dr.', Length: 3, BoundaryType: 'WordBoundary' }, + }, + }, + ], + }); + expect(parseAudioMetadataBody(body)).toEqual([ + { offset: 1000000, duration: 4250000, text: 'Dr.' }, + ]); + }); + + test('ignores non-WordBoundary metadata entries', async () => { + const { parseAudioMetadataBody } = await import('@/libs/edgeTTS'); + const body = JSON.stringify({ + Metadata: [ + { Type: 'SessionEnd', Data: { Offset: 99 } }, + { + Type: 'WordBoundary', + Data: { Offset: 5000000, Duration: 1000000, text: { Text: 'hello' } }, + }, + ], + }); + expect(parseAudioMetadataBody(body)).toEqual([ + { offset: 5000000, duration: 1000000, text: 'hello' }, + ]); + }); + + test('returns an empty list for malformed or empty bodies', async () => { + const { parseAudioMetadataBody } = await import('@/libs/edgeTTS'); + expect(parseAudioMetadataBody('not json')).toEqual([]); + expect(parseAudioMetadataBody('{}')).toEqual([]); + expect( + parseAudioMetadataBody(JSON.stringify({ Metadata: [{ Type: 'WordBoundary' }] })), + ).toEqual([]); + }); }); diff --git a/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts b/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts index 7fc5c10f..6a86f559 100644 --- a/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts +++ b/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts @@ -5,6 +5,13 @@ let createBehavior: () => Promise = () => Promise.resolve(undefined); // Shared mock control for createAudioUrl() and parsed SSML marks let createAudioUrlBehavior = vi.fn<() => Promise>(() => Promise.resolve('blob:mock-url')); +type MockAudioResult = { + url: string; + boundaries: Array<{ offset: number; duration: number; text: string }>; +}; +let createAudioBehavior = vi.fn<() => Promise>(() => + Promise.resolve({ url: 'blob:mock-url', boundaries: [] }), +); let parsedMarks: Array<{ name: string; text: string; language: string }> = []; // --- Mocks --- @@ -21,6 +28,7 @@ vi.mock('@/libs/edgeTTS', () => { static voices = voices; create = vi.fn().mockImplementation(() => createBehavior()); createAudioUrl = vi.fn().mockImplementation(() => createAudioUrlBehavior()); + createAudio = vi.fn().mockImplementation(() => createAudioBehavior()); }, EDGE_TTS_PROTOCOL: 'wss', }; @@ -63,6 +71,9 @@ describe('EdgeTTSClient', () => { beforeEach(() => { createBehavior = () => Promise.resolve(undefined); createAudioUrlBehavior = vi.fn<() => Promise>(() => Promise.resolve('blob:mock-url')); + createAudioBehavior = vi.fn<() => Promise>(() => + Promise.resolve({ url: 'blob:mock-url', boundaries: [] }), + ); parsedMarks = []; client = new EdgeTTSClient(); }); @@ -221,6 +232,12 @@ describe('EdgeTTSClient', () => { }); }); + describe('supportsWordBoundaries', () => { + test('returns true (Edge reports word-boundary timings)', () => { + expect(client.supportsWordBoundaries()).toBe(true); + }); + }); + describe('getGranularities', () => { test('returns array with sentence granularity only', () => { const granularities = client.getGranularities(); @@ -479,4 +496,137 @@ describe('EdgeTTSClient', () => { await expect(client.stop()).resolves.toBeUndefined(); }); }); + + describe('word boundary tracking during playback', () => { + class MockAudio { + static instances: MockAudio[] = []; + src = ''; + currentTime = 0; + preload = ''; + playbackRate = 1; + onended: ((e?: Event) => void) | null = null; + onerror: ((e?: unknown) => void) | null = null; + constructor() { + MockAudio.instances.push(this); + } + setAttribute() {} + play() { + return Promise.resolve(); + } + pause() {} + } + + let rafCallbacks: Map; + let rafId = 0; + const runRaf = () => { + const cbs = [...rafCallbacks.values()]; + rafCallbacks.clear(); + for (const cb of cbs) cb(0); + }; + const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + + let mockController: { + dispatchSpeakMark: ReturnType; + prepareSpeakWords: ReturnType; + dispatchSpeakWord: ReturnType; + }; + + beforeEach(() => { + MockAudio.instances = []; + rafCallbacks = new Map(); + vi.stubGlobal('Audio', MockAudio); + vi.stubGlobal('requestAnimationFrame', (cb: FrameRequestCallback) => { + rafCallbacks.set(++rafId, cb); + return rafId; + }); + vi.stubGlobal('cancelAnimationFrame', (id: number) => { + rafCallbacks.delete(id); + }); + mockController = { + dispatchSpeakMark: vi.fn(), + prepareSpeakWords: vi.fn(), + dispatchSpeakWord: vi.fn(), + }; + client = new EdgeTTSClient(mockController as unknown as TTSController); + parsedMarks = [{ name: '0', text: 'Hello brave world', language: 'en' }]; + createAudioBehavior = vi.fn(() => + Promise.resolve({ + url: 'blob:mock-url', + boundaries: [ + { offset: 1_000_000, duration: 4_000_000, text: 'Hello' }, + { offset: 6_000_000, duration: 4_000_000, text: 'brave' }, + { offset: 11_000_000, duration: 4_000_000, text: 'world' }, + ], + }), + ); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + const startSpeak = async () => { + await client.init(); + const it = client.speak('', new AbortController().signal); + const first = await it.next(); + expect((first.value as { code: string }).code).toBe('boundary'); + const resultPromise = it.next(); + await flush(); + const audio = MockAudio.instances.at(-1)!; + return { it, resultPromise, audio }; + }; + + test('prepares speak words and dispatches word indexes as playback advances', async () => { + const { resultPromise, audio } = await startSpeak(); + + expect(mockController.prepareSpeakWords).toHaveBeenCalledWith(['Hello', 'brave', 'world']); + + audio.currentTime = 0.11; + runRaf(); + expect(mockController.dispatchSpeakWord).toHaveBeenCalledWith(0); + + audio.currentTime = 0.65; + runRaf(); + expect(mockController.dispatchSpeakWord).toHaveBeenLastCalledWith(1); + + // Same word index is not re-dispatched on subsequent frames. + const callCount = mockController.dispatchSpeakWord.mock.calls.length; + runRaf(); + expect(mockController.dispatchSpeakWord.mock.calls.length).toBe(callCount); + + audio.onended?.(); + const result = await resultPromise; + expect((result.value as { code: string }).code).toBe('end'); + }); + + test('stops dispatching after the chunk ends', async () => { + const { resultPromise, audio } = await startSpeak(); + + audio.currentTime = 0.11; + runRaf(); + const callCount = mockController.dispatchSpeakWord.mock.calls.length; + + audio.onended?.(); + await resultPromise; + + audio.currentTime = 1.2; + runRaf(); + expect(mockController.dispatchSpeakWord.mock.calls.length).toBe(callCount); + }); + + test('hands empty words to the controller and does not track when no boundaries', async () => { + createAudioBehavior = vi.fn(() => Promise.resolve({ url: 'blob:mock-url', boundaries: [] })); + const { resultPromise, audio } = await startSpeak(); + + // Empty words are still forwarded so the controller can draw the + // sentence-highlight fallback; no per-word tracking is started. + expect(mockController.prepareSpeakWords).toHaveBeenCalledWith([]); + audio.currentTime = 0.5; + runRaf(); + expect(mockController.dispatchSpeakWord).not.toHaveBeenCalled(); + + audio.onended?.(); + await resultPromise; + }); + }); }); diff --git a/apps/readest-app/src/__tests__/services/tts-controller.test.ts b/apps/readest-app/src/__tests__/services/tts-controller.test.ts index 0d8dc06a..54d896b8 100644 --- a/apps/readest-app/src/__tests__/services/tts-controller.test.ts +++ b/apps/readest-app/src/__tests__/services/tts-controller.test.ts @@ -97,6 +97,7 @@ function createMockTTSClient(name: string): TTSClient { getAllVoices: vi.fn().mockResolvedValue([]), getVoices: vi.fn().mockResolvedValue([]), getGranularities: vi.fn().mockReturnValue(['word', 'sentence'] as TTSGranularity[]), + supportsWordBoundaries: vi.fn().mockReturnValue(name === 'edge'), getVoiceId: vi.fn().mockReturnValue('voice-1'), getSpeakingLang: vi.fn().mockReturnValue('en'), }; @@ -553,6 +554,191 @@ describe('TTSController', () => { }); }); + describe('word highlighting (prepareSpeakWords / dispatchSpeakWord)', () => { + const getOverlayer = () => + ( + mockView.renderer.getContents() as unknown as Array<{ + overlayer: { add: ReturnType; remove: ReturnType }; + }> + )[0]!.overlayer; + + const makeSentenceRange = () => { + document.body.innerHTML = '

Hello brave world

'; + const textNode = document.body.firstElementChild!.firstChild as Text; + const range = document.createRange(); + range.setStart(textNode, 0); + range.setEnd(textNode, textNode.length); + return range; + }; + + const armWithSentence = async (range: Range, markName = '0') => { + await controller.initViewTTS(0); + mockView.tts = { + setMark: vi.fn().mockReturnValue(range), + getLastRange: vi.fn().mockImplementation(() => range.cloneRange()), + } as unknown as FoliateView['tts']; + controller.dispatchSpeakMark({ + offset: 0, + name: markName, + text: 'Hello brave world', + language: 'en', + }); + }; + + test('prepareSpeakWords immediately highlights the first word (no sentence flash)', async () => { + await armWithSentence(makeSentenceRange()); + vi.mocked(mockView.getCFI).mockClear(); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('Hello'); + expect(getOverlayer().add).toHaveBeenCalledTimes(1); + }); + + test('prepareSpeakWords with no words falls back to the full-sentence highlight', async () => { + await armWithSentence(makeSentenceRange()); + vi.mocked(mockView.getCFI).mockClear(); + controller.prepareSpeakWords([]); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('Hello brave world'); + expect(getOverlayer().add).toHaveBeenCalledTimes(1); + }); + + test('dispatchSpeakWord highlights the word sub-range of the current sentence', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + + vi.mocked(mockView.getCFI).mockClear(); + getOverlayer().add.mockClear(); + controller.dispatchSpeakWord(1); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('brave'); + expect(getOverlayer().add).toHaveBeenCalledTimes(1); + expect(getOverlayer().add.mock.calls[0]![0]).toBe('tts-highlight'); + }); + + test('word indexes can be dispatched out of order after a seek back', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + + controller.dispatchSpeakWord(2); + vi.mocked(mockView.getCFI).mockClear(); + controller.dispatchSpeakWord(0); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('Hello'); + }); + + test('does not highlight words for one-time marks (name -1)', async () => { + await armWithSentence(makeSentenceRange(), '-1'); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + controller.dispatchSpeakWord(0); + + expect(getOverlayer().add).not.toHaveBeenCalled(); + }); + + test('a new speak mark clears previously prepared words', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + expect(getOverlayer().add).toHaveBeenCalledTimes(1); + + controller.dispatchSpeakMark({ + offset: 0, + name: '1', + text: 'Hello brave world', + language: 'en', + }); + getOverlayer().add.mockClear(); + controller.dispatchSpeakWord(0); + expect(getOverlayer().add).not.toHaveBeenCalled(); + }); + + test('unmatched first word does not highlight but later words still align', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['BOGUS', 'brave']); + + // First word unmatched → no eager highlight. + expect(getOverlayer().add).not.toHaveBeenCalled(); + + vi.mocked(mockView.getCFI).mockClear(); + controller.dispatchSpeakWord(1); + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('brave'); + }); + + test('reapplyCurrentHighlight re-draws the current word during word mode', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + controller.dispatchSpeakWord(1); + + vi.mocked(mockView.getCFI).mockClear(); + controller.reapplyCurrentHighlight(); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('brave'); + }); + + test('reapplyCurrentHighlight re-draws the whole sentence when not in word mode', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords([]); + + vi.mocked(mockView.getCFI).mockClear(); + controller.reapplyCurrentHighlight(); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('Hello brave world'); + }); + + test('dispatchSpeakWord emits tts-highlight-word for word-level page following', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + + const listener = vi.fn(); + controller.addEventListener('tts-highlight-word', listener); + vi.mocked(mockView.getCFI).mockReturnValue('cfi-word'); + controller.dispatchSpeakWord(1); + + expect(listener).toHaveBeenCalledTimes(1); + const ev = listener.mock.calls[0]![0] as CustomEvent; + expect(ev.detail).toEqual({ cfi: 'cfi-word' }); + }); + + test('dispatchSpeakWord does not emit a word event when the word is unmatched', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['BOGUS', 'brave']); + + const listener = vi.fn(); + controller.addEventListener('tts-highlight-word', listener); + controller.dispatchSpeakWord(0); // unmatched → no range, no event + expect(listener).not.toHaveBeenCalled(); + }); + + test('getCurrentHighlightCfi returns the word cfi in word mode, null otherwise', async () => { + await armWithSentence(makeSentenceRange()); + // Not in word mode until prepareSpeakWords with words. + expect(controller.getCurrentHighlightCfi()).toBeNull(); + + vi.mocked(mockView.getCFI).mockReturnValue('cfi-word'); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + controller.dispatchSpeakWord(1); + expect(controller.getCurrentHighlightCfi()).toBe('cfi-word'); + + // Empty (sentence fallback) leaves word mode → null so the caller uses + // the sentence-level ttsLocation. + controller.prepareSpeakWords([]); + expect(controller.getCurrentHighlightCfi()).toBeNull(); + }); + }); + describe('getSpokenSentence', () => { test('returns the trimmed text and cfi of the current sentence', async () => { await controller.initViewTTS(0); diff --git a/apps/readest-app/src/__tests__/services/tts-word-highlight.test.ts b/apps/readest-app/src/__tests__/services/tts-word-highlight.test.ts new file mode 100644 index 00000000..c6879188 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/tts-word-highlight.test.ts @@ -0,0 +1,151 @@ +import { describe, test, expect } from 'vitest'; +import { + computeWordOffsets, + findBoundaryIndexAtTime, + getTextSubRange, +} from '@/services/tts/wordHighlight'; + +describe('computeWordOffsets', () => { + test('maps words to offsets in order', () => { + const text = 'Dr. Smith bought 23 apples'; + const offsets = computeWordOffsets(text, ['Dr.', 'Smith', 'bought', '23', 'apples']); + expect(offsets).toEqual([ + { start: 0, end: 3 }, + { start: 4, end: 9 }, + { start: 10, end: 16 }, + { start: 17, end: 19 }, + { start: 20, end: 26 }, + ]); + }); + + test('repeated words advance past earlier occurrences', () => { + const text = 'the cat and the hat'; + const offsets = computeWordOffsets(text, ['the', 'cat', 'and', 'the', 'hat']); + expect(offsets![3]).toEqual({ start: 12, end: 15 }); + }); + + test('short word is not matched inside an earlier longer word', () => { + const text = 'and a cat'; + const offsets = computeWordOffsets(text, ['and', 'a', 'cat']); + expect(offsets).toEqual([ + { start: 0, end: 3 }, + { start: 4, end: 5 }, + { start: 6, end: 9 }, + ]); + }); + + test('missing word yields null without advancing the cursor', () => { + const text = 'hello world'; + const offsets = computeWordOffsets(text, ['hello', 'BOGUS', 'world']); + expect(offsets).toEqual([{ start: 0, end: 5 }, null, { start: 6, end: 11 }]); + }); + + test('empty or whitespace-only word yields null', () => { + const offsets = computeWordOffsets('hello world', ['hello', ' ', 'world']); + expect(offsets).toEqual([{ start: 0, end: 5 }, null, { start: 6, end: 11 }]); + }); + + test('words with attached punctuation match document text verbatim', () => { + const text = 'paid $5.50 for them, you know.'; + const offsets = computeWordOffsets(text, ['paid', '$5.50', 'for', 'them', 'you', 'know']); + expect(offsets).toEqual([ + { start: 0, end: 4 }, + { start: 5, end: 10 }, + { start: 11, end: 14 }, + { start: 15, end: 19 }, + { start: 21, end: 24 }, + { start: 25, end: 29 }, + ]); + }); +}); + +describe('findBoundaryIndexAtTime', () => { + // offsets are in 100-nanosecond ticks: 10_000_000 ticks = 1 second + const boundaries = [{ offset: 1_000_000 }, { offset: 5_000_000 }, { offset: 11_000_000 }]; + + test('returns -1 for an empty list', () => { + expect(findBoundaryIndexAtTime([], 1)).toBe(-1); + }); + + test('returns -1 before the first boundary', () => { + expect(findBoundaryIndexAtTime(boundaries, 0.05)).toBe(-1); + }); + + test('returns the boundary exactly at its start time', () => { + expect(findBoundaryIndexAtTime(boundaries, 0.1)).toBe(0); + }); + + test('returns the latest boundary at or before the given time', () => { + expect(findBoundaryIndexAtTime(boundaries, 0.7)).toBe(1); + }); + + test('returns the last boundary after the stream end', () => { + expect(findBoundaryIndexAtTime(boundaries, 10)).toBe(2); + }); +}); + +describe('getTextSubRange', () => { + const makeBaseRange = (html: string) => { + document.body.innerHTML = html; + const root = document.body.firstElementChild!; + const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT); + const textNodes: Text[] = []; + for (let n = walker.nextNode(); n; n = walker.nextNode()) textNodes.push(n as Text); + const range = document.createRange(); + range.setStart(textNodes[0]!, 0); + range.setEnd(textNodes.at(-1)!, textNodes.at(-1)!.length); + return range; + }; + + test('extracts a word inside a single text node', () => { + const base = makeBaseRange('

Hello brave world

'); + const sub = getTextSubRange(base, 6, 11); + expect(sub?.toString()).toBe('brave'); + }); + + test('extracts a word inside a nested element', () => { + const base = makeBaseRange('

Hello brave world

'); + expect(base.toString()).toBe('Hello brave world'); + const sub = getTextSubRange(base, 6, 11); + expect(sub?.toString()).toBe('brave'); + }); + + test('extracts a span crossing element boundaries', () => { + const base = makeBaseRange('

Hello brave world

'); + const sub = getTextSubRange(base, 4, 14); + expect(sub?.toString()).toBe('o brave wo'); + }); + + test('offsets are relative to a base range starting mid-node', () => { + document.body.innerHTML = '

Hello brave world

'; + const textNode = document.body.firstElementChild!.firstChild as Text; + const base = document.createRange(); + base.setStart(textNode, 3); + base.setEnd(textNode, textNode.length); + expect(base.toString()).toBe('lo brave world'); + const sub = getTextSubRange(base, 3, 8); + expect(sub?.toString()).toBe('brave'); + }); + + test('ignores text nodes outside the base range', () => { + document.body.innerHTML = '

before Hello world after

'; + const textNode = document.getElementById('target')!.firstChild as Text; + const base = document.createRange(); + base.setStart(textNode, 0); + base.setEnd(textNode, textNode.length); + const sub = getTextSubRange(base, 6, 11); + expect(sub?.toString()).toBe('world'); + }); + + test('returns null when offsets exceed the base range text', () => { + const base = makeBaseRange('

Hello

'); + expect(getTextSubRange(base, 0, 100)).toBeNull(); + }); + + test('returns null for an empty or inverted span', () => { + const base = makeBaseRange('

Hello

'); + expect(getTextSubRange(base, 2, 2)).toBeNull(); + expect(getTextSubRange(base, 3, 2)).toBeNull(); + expect(getTextSubRange(base, -1, 2)).toBeNull(); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/web-speech-client.test.ts b/apps/readest-app/src/__tests__/services/web-speech-client.test.ts index b1586bcf..d6f80b35 100644 --- a/apps/readest-app/src/__tests__/services/web-speech-client.test.ts +++ b/apps/readest-app/src/__tests__/services/web-speech-client.test.ts @@ -64,4 +64,8 @@ describe('WebSpeechClient getVoices', () => { const en = await client.getVoices('en-US'); expect(en[0]!.voices.map((v) => v.name)).not.toContain('Denise'); }); + + test('does not support word boundaries (sentence highlight only)', () => { + expect(client.supportsWordBoundaries()).toBe(false); + }); }); diff --git a/apps/readest-app/src/app/api/tts/edge/route.ts b/apps/readest-app/src/app/api/tts/edge/route.ts index 9fd19be0..ac676fe5 100644 --- a/apps/readest-app/src/app/api/tts/edge/route.ts +++ b/apps/readest-app/src/app/api/tts/edge/route.ts @@ -1,5 +1,10 @@ import { NextRequest, NextResponse } from 'next/server'; -import { EdgeSpeechTTS, EdgeTTSPayload } from '@/libs/edgeTTS'; +import { + EdgeSpeechTTS, + EdgeTTSPayload, + serializeWordBoundaries, + WORD_BOUNDARIES_HEADER, +} from '@/libs/edgeTTS'; import { validateUserAndToken } from '@/utils/access'; const getLangFromVoice = (voiceId: string): string => { @@ -69,7 +74,7 @@ export async function POST(request: NextRequest) { }; const tts = new EdgeSpeechTTS(); - const response = await tts.create(payload); + const { response, boundaries } = await tts.createWithBoundaries(payload); const arrayBuffer = await response.arrayBuffer(); return new NextResponse(arrayBuffer, { @@ -77,6 +82,7 @@ export async function POST(request: NextRequest) { headers: { 'Content-Type': 'audio/mpeg', 'Content-Length': arrayBuffer.byteLength.toString(), + [WORD_BOUNDARIES_HEADER]: serializeWordBoundaries(boundaries), }, }); } catch (error) { diff --git a/apps/readest-app/src/app/reader/hooks/useTTSControl.ts b/apps/readest-app/src/app/reader/hooks/useTTSControl.ts index 9e540769..1c405ee6 100644 --- a/apps/readest-app/src/app/reader/hooks/useTTSControl.ts +++ b/apps/readest-app/src/app/reader/hooks/useTTSControl.ts @@ -258,13 +258,50 @@ export const useTTSControl = ({ bookKey, onRequestHidePanel }: UseTTSControlProp } }; + // Word-level page following: turn the page as soon as the spoken word + // moves off the visible page, instead of waiting for the next sentence's + // mark. Only navigates when the word is outside the visible range, so + // on-page words don't trigger relocations. + const handleHighlightWord = (e: Event) => { + const { cfi } = (e as CustomEvent<{ cfi: string }>).detail; + const view = getView(bookKey); + if (!cfi || !view || !followingTTSLocationRef.current) return; + + const hlContents = view.renderer.getContents(); + const hlPrimaryIdx = view.renderer.primaryIndex; + const { doc, index: viewSectionIndex } = (hlContents.find((x) => x.index === hlPrimaryIdx) ?? + hlContents[0]) as { doc: Document; index?: number }; + + const { anchor, index: ttsSectionIndex } = view.resolveCFI(cfi); + // Cross-section navigation is driven by the sentence-level mark handler. + if (viewSectionIndex !== ttsSectionIndex) return; + if (hlContents.some(({ doc }) => (doc.getSelection()?.toString().length ?? 0) > 0)) return; + + const wordRange = anchor(doc); + const visibleRange = getProgress(bookKey)?.range as Range | undefined; + if (!wordRange || !visibleRange) return; + + try { + const ahead = wordRange.compareBoundaryPoints(Range.END_TO_START, visibleRange) > 0; + const behind = wordRange.compareBoundaryPoints(Range.START_TO_END, visibleRange) < 0; + if (ahead || behind) { + view.renderer.scrollToAnchor?.(wordRange); + } + } catch { + // Ranges may briefly belong to different documents during a section + // change; the mark handler takes over in that case. + } + }; + ttsController.addEventListener('tts-need-auth', handleNeedAuth); ttsController.addEventListener('tts-speak-mark', handleSpeakMark); ttsController.addEventListener('tts-highlight-mark', handleHighlightMark); + ttsController.addEventListener('tts-highlight-word', handleHighlightWord); return () => { ttsController.removeEventListener('tts-need-auth', handleNeedAuth); ttsController.removeEventListener('tts-speak-mark', handleSpeakMark); ttsController.removeEventListener('tts-highlight-mark', handleHighlightMark); + ttsController.removeEventListener('tts-highlight-word', handleHighlightWord); }; // eslint-disable-next-line react-hooks/exhaustive-deps }, [ttsController, bookKey]); @@ -277,18 +314,22 @@ export const useTTSControl = ({ bookKey, onRequestHidePanel }: UseTTSControlProp const ttsController = ttsControllerRef.current; if (!ttsController) return; - const view = getView(bookKey); const viewSettings = getViewSettings(bookKey); const ttsLocation = viewSettings?.ttsLocation; const { location } = progress || {}; if (!location || !ttsLocation) return; - if (isCfiInLocation(ttsLocation, location)) { + // Check the actual highlighted position against the view. During + // word-by-word playback the word can sit on a different page than the + // sentence's ttsLocation (a sentence spanning a page break), so the word + // position is the correct reference — otherwise the back-to-TTS button + // wrongly appears after the view follows the word onto the next page. + const highlightCfi = ttsController.getCurrentHighlightCfi() ?? ttsLocation; + if (isCfiInLocation(highlightCfi, location)) { setShowBackToCurrentTTSLocation(false); - const range = view?.tts?.getLastRange() as Range | null; - if (range) { - view?.tts?.highlight(range); - } + // Word-aware re-apply: re-draws the current word during word-by-word + // playback instead of redrawing the whole sentence over it. + ttsController.reapplyCurrentHighlight(); } else { const msSinceSectionChange = Date.now() - sectionChangingTimestampRef.current; if (msSinceSectionChange < 2000) return; diff --git a/apps/readest-app/src/libs/edgeTTS.ts b/apps/readest-app/src/libs/edgeTTS.ts index 872c768d..bd9480e1 100644 --- a/apps/readest-app/src/libs/edgeTTS.ts +++ b/apps/readest-app/src/libs/edgeTTS.ts @@ -276,6 +276,73 @@ export interface EdgeTTSPayload { pitch: number; } +// A word boundary reported by the Edge TTS service via `audio.metadata` +// frames. Offsets and durations are in 100-nanosecond ticks relative to the +// start of the audio stream; `text` is the verbatim span of the input text. +export interface TTSWordBoundary { + offset: number; + duration: number; + text: string; +} + +interface AudioMetadataEntry { + Type?: string; + Data?: { + Offset?: number; + Duration?: number; + text?: { Text?: string }; + }; +} + +export const parseAudioMetadataBody = (body: string): TTSWordBoundary[] => { + try { + const parsed = JSON.parse(body) as { Metadata?: AudioMetadataEntry[] }; + const boundaries: TTSWordBoundary[] = []; + for (const entry of parsed.Metadata ?? []) { + if (entry.Type !== 'WordBoundary') continue; + const offset = entry.Data?.Offset; + const text = entry.Data?.text?.Text; + if (typeof offset !== 'number' || typeof text !== 'string' || !text) continue; + boundaries.push({ offset, duration: entry.Data?.Duration ?? 0, text }); + } + return boundaries; + } catch { + return []; + } +}; + +export interface EdgeSpeechAudio { + response: Response; + boundaries: TTSWordBoundary[]; +} + +// Response header used to carry word boundaries through the authenticated +// HTTPS proxy route (`/api/tts/edge`), which streams only the audio body. +export const WORD_BOUNDARIES_HEADER = 'X-TTS-Word-Boundaries'; + +// HTTP header values must be ASCII, but boundary `text` can be any script +// (em-dashes, CJK, accents). Percent-encode the JSON so the header stays +// ASCII-safe across Node, browsers, and Cloudflare Workers. +export const serializeWordBoundaries = (boundaries: TTSWordBoundary[]): string => + encodeURIComponent(JSON.stringify(boundaries)); + +export const parseWordBoundariesHeader = (value: string | null): TTSWordBoundary[] => { + if (!value) return []; + try { + const parsed = JSON.parse(decodeURIComponent(value)); + if (!Array.isArray(parsed)) return []; + return parsed.filter( + (b: unknown): b is TTSWordBoundary => + !!b && + typeof (b as TTSWordBoundary).offset === 'number' && + typeof (b as TTSWordBoundary).duration === 'number' && + typeof (b as TTSWordBoundary).text === 'string', + ); + } catch { + return []; + } +}; + const hashPayload = (payload: EdgeTTSPayload): string => { const base = JSON.stringify(payload); return md5(base); @@ -291,6 +358,7 @@ export class EdgeSpeechTTS { URL.revokeObjectURL(url); } }); + private static boundariesCache = new LRUCache(200); private protocol: EDGE_TTS_PROTOCOL = 'wss'; constructor(protocol?: EDGE_TTS_PROTOCOL) { @@ -322,7 +390,7 @@ export class EdgeSpeechTTS { return response; } - async #fetchEdgeSpeechWs({ lang, text, voice, rate }: EdgeTTSPayload): Promise { + async #fetchEdgeSpeechWs({ lang, text, voice, rate }: EdgeTTSPayload): Promise { const connectId = randomMd5(); const params = new URLSearchParams({ ConnectionId: connectId, @@ -408,17 +476,19 @@ export class EdgeSpeechTTS { const TauriWebSocket = (await import('@tauri-apps/plugin-websocket')).default; const ws = await TauriWebSocket.connect(url, { headers: baseHeaders }); let audioData = new ArrayBuffer(0); + const boundaries: TTSWordBoundary[] = []; const messageUnlisten = await ws.addListener((msg) => { if (msg.type === 'Text') { - const { headers } = getHeadersAndData(msg.data as string); - if (headers['Path'] === 'turn.end') { + const { headers, body } = getHeadersAndData(msg.data as string); + if (headers['Path'] === 'audio.metadata') { + boundaries.push(...parseAudioMetadataBody(body.trim())); + } else if (headers['Path'] === 'turn.end') { ws.disconnect(); messageUnlisten(); if (!audioData.byteLength) { return reject(new Error('No audio data received.')); } - const res = new Response(audioData); - resolve(res); + resolve({ response: new Response(audioData), boundaries }); } } else if (msg.type === 'Binary') { let buffer: ArrayBufferLike; @@ -445,7 +515,10 @@ export class EdgeSpeechTTS { } }); } else if (isCloudflareWorkers()) { - return new Promise((resolve, reject) => { + // The Workers path backs the HTTPS proxy route. It captures both the + // audio body and the word boundaries (audio.metadata frames) so the + // route can forward boundaries via the WORD_BOUNDARIES_HEADER. + return new Promise((resolve, reject) => { (async () => { try { // Cloudflare Workers cannot use the `ws` npm package because it @@ -469,6 +542,7 @@ export class EdgeSpeechTTS { const ws = upgradeResponse.webSocket; let audioData = new ArrayBuffer(0); + const boundaries: TTSWordBoundary[] = []; let settled = false; // Cloudflare Workers deliver binary WebSocket frames as `Blob`, // whose conversion to bytes (`blob.arrayBuffer()`) is async. @@ -504,7 +578,7 @@ export class EdgeSpeechTTS { if (!audioData.byteLength) { reject(new Error('No audio data received.')); } else { - resolve(new Response(audioData)); + resolve({ response: new Response(audioData), boundaries }); } }; @@ -512,7 +586,11 @@ export class EdgeSpeechTTS { if (settled) return; const data = event.data; if (typeof data === 'string') { - const { headers } = getHeadersAndData(data); + const { headers, body } = getHeadersAndData(data); + if (headers['Path'] === 'audio.metadata') { + boundaries.push(...parseAudioMetadataBody(body.trim())); + return; + } if (headers['Path'] === 'turn.end') { // Wait for any in-flight Blob decodes to complete before // deciding whether audio was received. @@ -582,12 +660,18 @@ export class EdgeSpeechTTS { }); } else { return new Promise((resolve, reject) => { - const ws = new WebSocket(url, { - headers: baseHeaders, - }); + // In browsers isomorphic-ws is the native WebSocket, whose second + // argument is a subprotocol list — passing an options object throws + // SyntaxError. Custom headers are only supported (and only needed) + // in Node, where `ws` accepts (url, options). + const ws = + typeof window === 'undefined' + ? new WebSocket(url, { headers: baseHeaders }) + : new WebSocket(url); ws.binaryType = 'arraybuffer'; let audioData = new ArrayBuffer(0); + const boundaries: TTSWordBoundary[] = []; ws.addEventListener('open', () => { ws.send(config); @@ -596,14 +680,15 @@ export class EdgeSpeechTTS { ws.addEventListener('message', (event: WebSocket.MessageEvent) => { if (typeof event.data === 'string') { - const { headers } = getHeadersAndData(event.data); - if (headers['Path'] === 'turn.end') { + const { headers, body } = getHeadersAndData(event.data); + if (headers['Path'] === 'audio.metadata') { + boundaries.push(...parseAudioMetadataBody(body.trim())); + } else if (headers['Path'] === 'turn.end') { ws.close(); if (!audioData.byteLength) { return reject(new Error('No audio data received.')); } - const res = new Response(audioData); - resolve(res); + resolve({ response: new Response(audioData), boundaries }); } } else if (event.data instanceof ArrayBuffer) { const dataView = new DataView(event.data); @@ -631,29 +716,49 @@ export class EdgeSpeechTTS { } } - async create(payload: EdgeTTSPayload): Promise { + async #fetchEdgeSpeech(payload: EdgeTTSPayload): Promise { if (this.protocol === 'https') { - return this.#fetchEdgeSpeechHttp(payload); + // The HTTPS proxy streams the audio body and carries word boundaries in + // the WORD_BOUNDARIES_HEADER response header (see /api/tts/edge route). + const response = await this.#fetchEdgeSpeechHttp(payload); + return { + response, + boundaries: parseWordBoundariesHeader(response.headers.get(WORD_BOUNDARIES_HEADER)), + }; } else { return this.#fetchEdgeSpeechWs(payload); } } - async createAudioUrl(payload: EdgeTTSPayload): Promise { + async create(payload: EdgeTTSPayload): Promise { + return (await this.#fetchEdgeSpeech(payload)).response; + } + + // Server-side helper for the /api/tts/edge route: returns the audio Response + // together with the captured word boundaries so the route can forward them. + async createWithBoundaries(payload: EdgeTTSPayload): Promise { + return this.#fetchEdgeSpeech(payload); + } + + async createAudio( + payload: EdgeTTSPayload, + ): Promise<{ url: string; boundaries: TTSWordBoundary[] }> { const cacheKey = hashPayload(payload); - if (EdgeSpeechTTS.audioUrlCache.has(cacheKey)) { - return EdgeSpeechTTS.audioUrlCache.get(cacheKey)!; - } - try { - const res = await this.create(payload); - const arrayBuffer = await res.arrayBuffer(); - const blob = new Blob([arrayBuffer], { type: 'audio/mpeg' }); - const objectUrl = URL.createObjectURL(blob); - EdgeSpeechTTS.audioCache.set(cacheKey, blob); - EdgeSpeechTTS.audioUrlCache.set(cacheKey, objectUrl); - return objectUrl; - } catch (error) { - throw error; + const cachedUrl = EdgeSpeechTTS.audioUrlCache.get(cacheKey); + if (cachedUrl) { + return { url: cachedUrl, boundaries: EdgeSpeechTTS.boundariesCache.get(cacheKey) ?? [] }; } + const { response, boundaries } = await this.#fetchEdgeSpeech(payload); + const arrayBuffer = await response.arrayBuffer(); + const blob = new Blob([arrayBuffer], { type: 'audio/mpeg' }); + const objectUrl = URL.createObjectURL(blob); + EdgeSpeechTTS.audioCache.set(cacheKey, blob); + EdgeSpeechTTS.audioUrlCache.set(cacheKey, objectUrl); + EdgeSpeechTTS.boundariesCache.set(cacheKey, boundaries); + return { url: objectUrl, boundaries }; + } + + async createAudioUrl(payload: EdgeTTSPayload): Promise { + return (await this.createAudio(payload)).url; } } diff --git a/apps/readest-app/src/services/tts/EdgeTTSClient.ts b/apps/readest-app/src/services/tts/EdgeTTSClient.ts index 219b9ae9..268fa9eb 100644 --- a/apps/readest-app/src/services/tts/EdgeTTSClient.ts +++ b/apps/readest-app/src/services/tts/EdgeTTSClient.ts @@ -1,12 +1,13 @@ import { getUserLocale } from '@/utils/misc'; import { isSameLang } from '@/utils/lang'; import { TTSClient, TTSMessageEvent } from './TTSClient'; -import { EdgeSpeechTTS, EdgeTTSPayload, EDGE_TTS_PROTOCOL } from '@/libs/edgeTTS'; +import { EdgeSpeechTTS, EdgeTTSPayload, EDGE_TTS_PROTOCOL, TTSWordBoundary } from '@/libs/edgeTTS'; import { TTSGranularity, TTSVoice, TTSVoicesGroup } from './types'; import { AppService } from '@/types/system'; import { parseSSMLMarks } from '@/utils/ssml'; import { TTSController } from './TTSController'; import { TTSUtils } from './TTSUtils'; +import { findBoundaryIndexAtTime } from './wordHighlight'; export class EdgeTTSClient implements TTSClient { name = 'edge-tts'; @@ -27,6 +28,7 @@ export class EdgeTTSClient implements TTSClient { #pausedAt = 0; #startedAt = 0; #fadeCompensation: number | null = null; + #wordTrackingRafId: number | null = null; constructor(controller?: TTSController, appService?: AppService | null) { this.controller = controller; @@ -161,9 +163,11 @@ export class EdgeTTSClient implements TTSClient { const { language: voiceLang } = mark; const voiceId = await this.getVoiceIdFromLang(voiceLang); this.#speakingLang = voiceLang; - const audioUrl = await this.#edgeTTS?.createAudioUrl( + const audioResult = await this.#edgeTTS?.createAudio( this.getPayload(voiceLang, mark.text, voiceId), ); + const audioUrl = audioResult?.url; + const boundaries = audioResult?.boundaries ?? []; if (signal.aborted) { yield { code: 'error', message: 'Aborted' } as TTSMessageEvent; break; @@ -177,6 +181,7 @@ export class EdgeTTSClient implements TTSClient { const result = await new Promise((resolve) => { const cleanUp = () => { + this.#stopWordTracking(); audio.onended = null; audio.onerror = null; audio.src = ''; @@ -207,6 +212,7 @@ export class EdgeTTSClient implements TTSClient { }; this.#isPlaying = true; audio.src = audioUrl || ''; + this.#startWordTracking(audio, boundaries); if (!this.appService?.isLinuxApp) { audio.playbackRate = this.#rate; } @@ -243,6 +249,39 @@ export class EdgeTTSClient implements TTSClient { await this.stopInternal(); } + // Follow playback time against the word-boundary metadata of the current + // chunk and tell the controller which word is being spoken so it can + // highlight it. Word offsets are in media time, so audio.playbackRate and + // pause/resume (including the resume rewind on iOS) are handled naturally + // by polling audio.currentTime. + #startWordTracking(audio: HTMLAudioElement, boundaries: TTSWordBoundary[]) { + this.#stopWordTracking(); + const controller = this.controller; + if (!controller) return; + // Always hand the words to the controller — with boundaries it highlights + // word-by-word; with none it draws the sentence highlight that was + // suppressed at mark dispatch (see TTSController.prepareSpeakWords). + controller.prepareSpeakWords(boundaries.map((boundary) => boundary.text)); + if (!boundaries.length) return; + let lastIndex = -1; + const tick = () => { + const index = findBoundaryIndexAtTime(boundaries, audio.currentTime); + if (index !== lastIndex && index >= 0) { + lastIndex = index; + controller.dispatchSpeakWord(index); + } + this.#wordTrackingRafId = requestAnimationFrame(tick); + }; + this.#wordTrackingRafId = requestAnimationFrame(tick); + } + + #stopWordTracking() { + if (this.#wordTrackingRafId !== null) { + cancelAnimationFrame(this.#wordTrackingRafId); + this.#wordTrackingRafId = null; + } + } + async pause() { if (!this.#isPlaying || !this.#audioElement) return true; this.#pausedAt = this.#audioElement.currentTime - this.#startedAt; @@ -281,6 +320,7 @@ export class EdgeTTSClient implements TTSClient { } private async stopInternal() { + this.#stopWordTracking(); this.#isPlaying = false; this.#pausedAt = 0; this.#startedAt = 0; @@ -340,6 +380,10 @@ export class EdgeTTSClient implements TTSClient { this.#primaryLang = lang; } + supportsWordBoundaries(): boolean { + return true; + } + getGranularities(): TTSGranularity[] { return ['sentence']; } diff --git a/apps/readest-app/src/services/tts/NativeTTSClient.ts b/apps/readest-app/src/services/tts/NativeTTSClient.ts index 30974d55..d67de246 100644 --- a/apps/readest-app/src/services/tts/NativeTTSClient.ts +++ b/apps/readest-app/src/services/tts/NativeTTSClient.ts @@ -296,6 +296,10 @@ export class NativeTTSClient implements TTSClient { this.#primaryLang = lang; } + supportsWordBoundaries(): boolean { + return false; + } + getGranularities(): TTSGranularity[] { return ['sentence']; } diff --git a/apps/readest-app/src/services/tts/TTSClient.ts b/apps/readest-app/src/services/tts/TTSClient.ts index 330f91a2..f4630678 100644 --- a/apps/readest-app/src/services/tts/TTSClient.ts +++ b/apps/readest-app/src/services/tts/TTSClient.ts @@ -24,6 +24,9 @@ export interface TTSClient { getAllVoices(): Promise; getVoices(lang: string): Promise; getGranularities(): TTSGranularity[]; + // Whether this client reports word-boundary timings during playback so the + // controller can highlight word-by-word (and suppress the sentence highlight). + supportsWordBoundaries(): boolean; getVoiceId(): string; getSpeakingLang(): string; } diff --git a/apps/readest-app/src/services/tts/TTSController.ts b/apps/readest-app/src/services/tts/TTSController.ts index 031a3f57..917b46d1 100644 --- a/apps/readest-app/src/services/tts/TTSController.ts +++ b/apps/readest-app/src/services/tts/TTSController.ts @@ -10,6 +10,7 @@ import { EdgeTTSClient } from './EdgeTTSClient'; import { TTSUtils } from './TTSUtils'; import { TTSClient } from './TTSClient'; import { isValidLang } from '@/utils/lang'; +import { computeWordOffsets, getTextSubRange, TTSWordOffset } from './wordHighlight'; type TTSState = | 'stopped' @@ -35,6 +36,20 @@ export class TTSController extends EventTarget { #ttsSectionIndex: number = -1; + // Word-level highlight state for the currently spoken chunk. Armed by a + // successful dispatchSpeakMark, populated by prepareSpeakWords when a TTS + // client has word-boundary metadata for the chunk. + #speakWordsArmed = false; + #speakWordBaseRange: Range | null = null; + #speakWordOffsets: (TTSWordOffset | null)[] = []; + #speakWordRanges: (Range | null | undefined)[] = []; + #suppressMarkHighlight = false; + // True while the current chunk is highlighted word-by-word, with the most + // recently highlighted word range. Lets re-highlights (e.g. on page relocate) + // re-apply the word instead of redrawing the whole sentence over it. + #wordHighlightActive = false; + #lastSpeakWordRange: Range | null = null; + state: TTSState = 'stopped'; ttsLang: string = ''; ttsRate: number = 1.0; @@ -111,6 +126,11 @@ export class TTSController extends EventTarget { #getHighlighter() { return (range: Range) => { + // Suppress the sentence highlight that foliate's setMark draws when the + // active client highlights word-by-word. The flag is only set around the + // synchronous setMark call, so word draws (dispatchSpeakWord) and paused + // navigation still highlight normally. + if (this.#suppressMarkHighlight) return; const content = this.#getPrimaryContent(); if (!content) return; const { doc, index, overlayer } = content; @@ -576,12 +596,108 @@ export class TTSController extends EventTarget { } dispatchSpeakMark(mark?: TTSMark) { + this.#resetSpeakWords(); this.dispatchEvent(new CustomEvent('tts-speak-mark', { detail: mark || { text: '' } })); if (mark && mark.name !== '-1') { try { + // When the active client highlights word-by-word, suppress the + // sentence highlight that setMark would otherwise draw, so the page + // doesn't flash the whole sentence before the first word. The fallback + // (no boundaries) is drawn later in prepareSpeakWords. + this.#suppressMarkHighlight = this.ttsClient.supportsWordBoundaries(); const range = this.view.tts?.setMark(mark.name); + this.#suppressMarkHighlight = false; + this.#speakWordsArmed = !!range; const cfi = this.view.getCFI(this.#ttsSectionIndex, range); this.dispatchEvent(new CustomEvent('tts-highlight-mark', { detail: { cfi } })); + } catch { + this.#suppressMarkHighlight = false; + } + } + } + + #resetSpeakWords() { + this.#speakWordsArmed = false; + this.#speakWordBaseRange = null; + this.#speakWordOffsets = []; + this.#speakWordRanges = []; + this.#wordHighlightActive = false; + this.#lastSpeakWordRange = null; + } + + // Re-apply the active highlight after the view relocates (page turn, + // re-render). In word mode this re-draws the current word so the sentence + // never reappears over it; otherwise it re-draws the sentence. + reapplyCurrentHighlight() { + if (this.#wordHighlightActive && this.#lastSpeakWordRange) { + this.#getHighlighter()(this.#lastSpeakWordRange.cloneRange()); + return; + } + const range = this.view.tts?.getLastRange(); + if (range) this.#getHighlighter()(range.cloneRange()); + } + + // CFI of the currently highlighted word during word-by-word playback. Used + // for the "in view" check that drives the back-to-TTS button: when a sentence + // spans a page break, the word can be on a different page than the sentence's + // ttsLocation, so the word position is the accurate reference. Returns null + // outside word mode, where the sentence-level ttsLocation is correct. + getCurrentHighlightCfi(): string | null { + if (!this.#wordHighlightActive || !this.#lastSpeakWordRange || this.#ttsSectionIndex < 0) { + return null; + } + try { + return this.view.getCFI(this.#ttsSectionIndex, this.#lastSpeakWordRange) || null; + } catch { + return null; + } + } + + // Word-level highlighting within the chunk of the last dispatched mark, + // driven by TTS clients that report word boundaries (Edge TTS). It only + // swaps the visual highlight from the sentence to the spoken word — + // ttsLocation, media-session metadata and mark navigation keep their + // sentence-level semantics. + prepareSpeakWords(words: string[]) { + if (!this.#speakWordsArmed) return; + const range = this.view.tts?.getLastRange(); + if (!range) return; + this.#speakWordBaseRange = range; + this.#speakWordOffsets = computeWordOffsets(range.toString(), words); + this.#speakWordRanges = []; + if (words.length === 0) { + // No word boundaries for this chunk: the sentence highlight was + // suppressed at mark dispatch, so draw it now as the fallback. + this.#wordHighlightActive = false; + this.#getHighlighter()(range.cloneRange()); + } else { + // Highlight the first word immediately so the suppressed sentence + // highlight never appears before playback reaches the first boundary. + this.#wordHighlightActive = true; + this.dispatchSpeakWord(0); + } + } + + dispatchSpeakWord(index: number) { + const base = this.#speakWordBaseRange; + if (!base) return; + let range = this.#speakWordRanges[index]; + if (range === undefined) { + const offset = this.#speakWordOffsets[index]; + range = offset ? getTextSubRange(base, offset.start, offset.end) : null; + this.#speakWordRanges[index] = range; + } + if (range) { + this.#lastSpeakWordRange = range; + this.#getHighlighter()(range.cloneRange()); + // Let the view follow the spoken word so it turns the page mid-sentence + // when the word crosses a page boundary, instead of waiting for the next + // sentence's mark. + try { + const cfi = this.view.getCFI(this.#ttsSectionIndex, range); + if (cfi) { + this.dispatchEvent(new CustomEvent('tts-highlight-word', { detail: { cfi } })); + } } catch {} } } diff --git a/apps/readest-app/src/services/tts/WebSpeechClient.ts b/apps/readest-app/src/services/tts/WebSpeechClient.ts index 55e1278a..3a575906 100644 --- a/apps/readest-app/src/services/tts/WebSpeechClient.ts +++ b/apps/readest-app/src/services/tts/WebSpeechClient.ts @@ -266,6 +266,10 @@ export class WebSpeechClient implements TTSClient { return [voicesGroup]; } + supportsWordBoundaries(): boolean { + return false; + } + getGranularities(): TTSGranularity[] { // currently only support sentence boundary and disable word boundary as changing voice // in the middle of speech is not possible for different granularities diff --git a/apps/readest-app/src/services/tts/wordHighlight.ts b/apps/readest-app/src/services/tts/wordHighlight.ts new file mode 100644 index 00000000..744395d8 --- /dev/null +++ b/apps/readest-app/src/services/tts/wordHighlight.ts @@ -0,0 +1,95 @@ +// Helpers for word-level TTS highlighting driven by Edge TTS word-boundary +// metadata. Boundary words are matched sequentially against the text of the +// currently spoken sentence range; an unmatched word (e.g. rewritten by a +// TTS-only proofread rule) is skipped without advancing the search cursor so +// later words still align. + +export interface TTSWordOffset { + start: number; + end: number; +} + +// Edge TTS word-boundary offsets are in 100-nanosecond ticks. +const TICKS_PER_SECOND = 10_000_000; + +export const computeWordOffsets = (text: string, words: string[]): (TTSWordOffset | null)[] => { + const offsets: (TTSWordOffset | null)[] = []; + let cursor = 0; + for (const word of words) { + const trimmed = word.trim(); + if (!trimmed) { + offsets.push(null); + continue; + } + const index = text.indexOf(trimmed, cursor); + if (index === -1) { + offsets.push(null); + continue; + } + offsets.push({ start: index, end: index + trimmed.length }); + cursor = index + trimmed.length; + } + return offsets; +}; + +export const findBoundaryIndexAtTime = ( + boundaries: { offset: number }[], + seconds: number, +): number => { + const ticks = seconds * TICKS_PER_SECOND; + let low = 0; + let high = boundaries.length - 1; + let result = -1; + while (low <= high) { + const mid = (low + high) >> 1; + if (boundaries[mid]!.offset <= ticks) { + result = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + return result; +}; + +// Extract the sub-range covering [start, end) of base.toString(). Walks the +// text nodes intersecting the base range, slicing partially-contained +// boundary nodes the same way Range stringification does. +export const getTextSubRange = (base: Range, start: number, end: number): Range | null => { + if (start < 0 || end <= start) return null; + const root = base.commonAncestorContainer; + const doc = root.ownerDocument ?? (root as Document); + // A TreeWalker never yields its root, which is the node itself when the + // base range lies within a single text node. + const textNodes = function* () { + if (root.nodeType === Node.TEXT_NODE) { + yield root; + return; + } + const walker = doc.createTreeWalker(root, NodeFilter.SHOW_TEXT); + for (let node = walker.nextNode(); node; node = walker.nextNode()) yield node; + }; + let pos = 0; + let startNode: Text | null = null; + let startOffset = 0; + for (const node of textNodes()) { + if (!base.intersectsNode(node)) continue; + const text = node as Text; + const from = node === base.startContainer ? base.startOffset : 0; + const to = node === base.endContainer ? base.endOffset : text.data.length; + if (to <= from) continue; + const len = to - from; + if (!startNode && pos + len > start) { + startNode = text; + startOffset = from + (start - pos); + } + if (startNode && pos + len >= end) { + const range = doc.createRange(); + range.setStart(startNode, startOffset); + range.setEnd(text, from + (end - pos)); + return range; + } + pos += len; + } + return null; +};