From a56cc6c61a60d1de64d26a128701726d2f30098f Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Sat, 13 Jun 2026 14:29:34 +0800 Subject: [PATCH] feat(tts): word-by-word highlighting for Edge TTS, closes #4017 (#4566) Highlight each word as it is spoken (Edge TTS only) instead of keeping the whole sentence highlighted, and keep the view tracking the spoken word across page boundaries. Word boundaries - Capture Edge's audio.metadata WordBoundary frames (offset/duration in 100ns ticks plus the verbatim input-text span) in the Tauri, browser, and Cloudflare-Workers WebSocket transports. - Carry boundaries through the authenticated HTTPS proxy route via an X-TTS-Word-Boundaries response header (percent-encoded JSON, ASCII-safe), so word highlighting works on the web where the browser cannot open the wss connection directly. Cache them alongside the audio blob URL. Highlighting - Sync a requestAnimationFrame loop to audio.currentTime against the boundary table and highlight the word sub-range within the spoken sentence. Synthesis stays sentence-level (natural prosody); only the visual highlight is word-level. - Suppress the sentence highlight when the active client reports word boundaries and draw the first word immediately, so the whole sentence never flashes before the first word. Fall back to the sentence highlight when a chunk has no boundaries (other engines, empty metadata). - Re-apply the current word (not the sentence) when the view relocates. Page following - Turn the page as soon as the spoken word crosses a page boundary (a tts-highlight-word event scrolls only when the word is outside the visible range), instead of waiting for the next sentence. - Check the word's position for the "back to TTS location" badge so it no longer appears while the view follows the word onto the next page. Also fixes a pre-existing bug where the browser WebSocket was constructed with an options object (valid only for the Node ws package), which threw in browsers and made the wss path unusable on the web. Co-authored-by: Claude Fable 5 --- .../libs/edgeTTS-word-boundaries.test.ts | 216 ++++++++++++++++++ .../src/__tests__/libs/edgeTTS.test.ts | 111 +++++++++ .../services/edge-tts-client.test.ts | 150 ++++++++++++ .../__tests__/services/tts-controller.test.ts | 186 +++++++++++++++ .../services/tts-word-highlight.test.ts | 151 ++++++++++++ .../services/web-speech-client.test.ts | 4 + .../readest-app/src/app/api/tts/edge/route.ts | 10 +- .../src/app/reader/hooks/useTTSControl.ts | 53 ++++- apps/readest-app/src/libs/edgeTTS.ts | 167 +++++++++++--- .../src/services/tts/EdgeTTSClient.ts | 48 +++- .../src/services/tts/NativeTTSClient.ts | 4 + .../readest-app/src/services/tts/TTSClient.ts | 3 + .../src/services/tts/TTSController.ts | 116 ++++++++++ .../src/services/tts/WebSpeechClient.ts | 4 + .../src/services/tts/wordHighlight.ts | 95 ++++++++ 15 files changed, 1277 insertions(+), 41 deletions(-) create mode 100644 apps/readest-app/src/__tests__/libs/edgeTTS-word-boundaries.test.ts create mode 100644 apps/readest-app/src/__tests__/services/tts-word-highlight.test.ts create mode 100644 apps/readest-app/src/services/tts/wordHighlight.ts diff --git a/apps/readest-app/src/__tests__/libs/edgeTTS-word-boundaries.test.ts b/apps/readest-app/src/__tests__/libs/edgeTTS-word-boundaries.test.ts new file mode 100644 index 00000000..779bc949 --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/edgeTTS-word-boundaries.test.ts @@ -0,0 +1,216 @@ +import { describe, test, expect, vi, beforeEach } from 'vitest'; + +// Controllable WebSocket fake for the browser (isomorphic-ws) transport. +const wsState = vi.hoisted(() => ({ + instances: [] as Array<{ + url: string; + binaryType: string; + listeners: Record void>>; + sent: unknown[]; + emit: (type: string, event?: unknown) => void; + }>, +})); + +vi.mock('isomorphic-ws', () => ({ + default: class MockWebSocket { + url: string; + opts?: unknown; + binaryType = ''; + listeners: Record void>> = {}; + sent: unknown[] = []; + constructor(url: string, opts?: unknown) { + this.url = url; + this.opts = opts; + wsState.instances.push(this); + } + addEventListener(type: string, cb: (event: unknown) => void) { + (this.listeners[type] ??= []).push(cb); + } + send(data: unknown) { + this.sent.push(data); + } + close() {} + emit(type: string, event?: unknown) { + for (const cb of this.listeners[type] ?? []) cb(event); + } + }, +})); + +vi.mock('@/services/environment', () => ({ + getAPIBaseUrl: () => 'http://localhost/api', + isTauriAppPlatform: () => false, +})); + +vi.mock('@/utils/supabase', () => ({ + supabase: { auth: { getSession: async () => ({ data: { session: null } }) } }, + createSupabaseClient: () => ({}), + createSupabaseAdminClient: () => ({}), +})); + +// Controllable stub for the authenticated HTTPS proxy fetch. +const httpState = vi.hoisted(() => ({ + headers: {} as Record, + body: new Uint8Array([1, 2, 3]), +})); +vi.mock('@/utils/fetch', () => ({ + fetchWithAuth: vi.fn( + async () => new Response(httpState.body, { status: 200, headers: httpState.headers }), + ), +})); + +const makeBinaryAudioFrame = (audio: Uint8Array) => { + const header = new TextEncoder().encode('Path:audio\r\n'); + const buf = new ArrayBuffer(2 + header.length + audio.length); + new DataView(buf).setInt16(0, header.length); + new Uint8Array(buf).set(header, 2); + new Uint8Array(buf).set(audio, 2 + header.length); + return buf; +}; + +const makeMetadataFrame = (text: string, offset: number, duration: number) => + 'X-RequestId:abc\r\nContent-Type:application/json; charset=utf-8\r\nPath:audio.metadata\r\n\r\n' + + JSON.stringify({ + Metadata: [ + { + Type: 'WordBoundary', + Data: { Offset: offset, Duration: duration, text: { Text: text, Length: text.length } }, + }, + ], + }); + +describe('EdgeSpeechTTS.createAudio word boundaries (browser WebSocket path)', () => { + beforeEach(() => { + wsState.instances.length = 0; + (URL as unknown as { createObjectURL?: (blob: Blob) => string }).createObjectURL = vi.fn( + () => 'blob:mock-object-url', + ); + }); + + test('captures word boundaries from audio.metadata frames and caches them', async () => { + const { EdgeSpeechTTS } = await import('@/libs/edgeTTS'); + const tts = new EdgeSpeechTTS('wss'); + const payload = { + lang: 'en', + text: 'Hello brave world', + voice: 'en-US-AriaNeural', + rate: 1.0, + pitch: 1.0, + }; + + const promise = tts.createAudio(payload); + await vi.waitFor(() => expect(wsState.instances.length).toBe(1)); + const ws = wsState.instances[0]!; + // In a browser (jsdom has `window`), the WebSocket constructor must be + // called WITHOUT an options argument: native WebSocket treats a second + // argument as subprotocols and throws SyntaxError on an options object. + expect((ws as unknown as { opts?: unknown }).opts).toBeUndefined(); + ws.emit('open'); + ws.emit('message', { data: makeMetadataFrame('Hello', 1000000, 4000000) }); + ws.emit('message', { data: makeBinaryAudioFrame(new Uint8Array([1, 2, 3, 4])) }); + ws.emit('message', { data: makeMetadataFrame('brave', 6000000, 4000000) }); + ws.emit('message', { data: 'Path:turn.end\r\n\r\n' }); + + const { url, boundaries } = await promise; + expect(url).toBe('blob:mock-object-url'); + expect(boundaries).toEqual([ + { offset: 1000000, duration: 4000000, text: 'Hello' }, + { offset: 6000000, duration: 4000000, text: 'brave' }, + ]); + + // A second call for the same payload is served from the cache: no new + // WebSocket connection, same boundaries. + const cached = await tts.createAudio(payload); + expect(wsState.instances.length).toBe(1); + expect(cached.url).toBe('blob:mock-object-url'); + expect(cached.boundaries).toEqual(boundaries); + }); + + test('resolves with empty boundaries when no metadata frames arrive', async () => { + const { EdgeSpeechTTS } = await import('@/libs/edgeTTS'); + const tts = new EdgeSpeechTTS('wss'); + + const promise = tts.createAudio({ + lang: 'en', + text: 'No metadata here', + voice: 'en-US-AriaNeural', + rate: 1.0, + pitch: 1.0, + }); + await vi.waitFor(() => expect(wsState.instances.length).toBe(1)); + const ws = wsState.instances[0]!; + ws.emit('open'); + ws.emit('message', { data: makeBinaryAudioFrame(new Uint8Array([9, 9])) }); + ws.emit('message', { data: 'Path:turn.end\r\n\r\n' }); + + const { boundaries } = await promise; + expect(boundaries).toEqual([]); + }); +}); + +describe('word-boundary header (de)serialization', () => { + test('round-trips boundaries, ASCII-safe for HTTP headers, incl. non-ASCII text', async () => { + const { serializeWordBoundaries, parseWordBoundariesHeader } = await import('@/libs/edgeTTS'); + const boundaries = [ + { offset: 1000000, duration: 4000000, text: 'Hello' }, + { offset: 6000000, duration: 4000000, text: 'café—世界' }, + ]; + const header = serializeWordBoundaries(boundaries); + // HTTP header values must be ASCII; non-ASCII text would corrupt the header. + expect([...header].every((c) => c.charCodeAt(0) < 128)).toBe(true); + expect(parseWordBoundariesHeader(header)).toEqual(boundaries); + }); + + test('parse returns [] for null, malformed, or non-boundary payloads', async () => { + const { parseWordBoundariesHeader } = await import('@/libs/edgeTTS'); + expect(parseWordBoundariesHeader(null)).toEqual([]); + expect(parseWordBoundariesHeader('not-json')).toEqual([]); + expect(parseWordBoundariesHeader(encodeURIComponent(JSON.stringify({ x: 1 })))).toEqual([]); + expect(parseWordBoundariesHeader(encodeURIComponent(JSON.stringify([{ text: 'x' }])))).toEqual( + [], + ); + }); +}); + +describe('EdgeSpeechTTS.createAudio over the HTTPS proxy (word boundaries via header)', () => { + beforeEach(() => { + httpState.headers = {}; + httpState.body = new Uint8Array([1, 2, 3]); + (URL as unknown as { createObjectURL?: (blob: Blob) => string }).createObjectURL = vi.fn( + () => 'blob:mock-object-url', + ); + }); + + test('parses word boundaries from the X-TTS-Word-Boundaries response header', async () => { + const { EdgeSpeechTTS, serializeWordBoundaries, WORD_BOUNDARIES_HEADER } = await import( + '@/libs/edgeTTS' + ); + const boundaries = [ + { offset: 1000000, duration: 4000000, text: 'Hello' }, + { offset: 6000000, duration: 4000000, text: 'world' }, + ]; + httpState.headers = { [WORD_BOUNDARIES_HEADER]: serializeWordBoundaries(boundaries) }; + + const tts = new EdgeSpeechTTS('https'); + const result = await tts.createAudio({ + lang: 'en', + text: 'Hello world https', + voice: 'en-US-AriaNeural', + rate: 1.0, + pitch: 1.0, + }); + expect(result.boundaries).toEqual(boundaries); + }); + + test('returns empty boundaries when the proxy omits the header', async () => { + const { EdgeSpeechTTS } = await import('@/libs/edgeTTS'); + const tts = new EdgeSpeechTTS('https'); + const result = await tts.createAudio({ + lang: 'en', + text: 'No header https', + voice: 'en-US-AriaNeural', + rate: 1.0, + pitch: 1.0, + }); + expect(result.boundaries).toEqual([]); + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/edgeTTS.test.ts b/apps/readest-app/src/__tests__/libs/edgeTTS.test.ts index c9cd1e1f..a9ca60da 100644 --- a/apps/readest-app/src/__tests__/libs/edgeTTS.test.ts +++ b/apps/readest-app/src/__tests__/libs/edgeTTS.test.ts @@ -213,4 +213,115 @@ describe('EdgeSpeechTTS on Cloudflare Workers', () => { }), ).rejects.toThrow(); }); + + test('captures word boundaries from audio.metadata frames', async () => { + const listeners: Record void>> = {}; + const mockSocket = { + accept: vi.fn(), + send: vi.fn(), + close: vi.fn(), + addEventListener: vi.fn((type: string, cb: (event: unknown) => void) => { + (listeners[type] ??= []).push(cb); + }), + removeEventListener: vi.fn(), + }; + + const audioFrame = () => { + const headerText = 'X-RequestId:1\r\nContent-Type:audio/mpeg\r\nPath:audio\r\n'; + const headerBytes = new TextEncoder().encode(headerText); + const audioBody = new Uint8Array([0xaa, 0xbb]); + const frame = new Uint8Array(2 + headerBytes.byteLength + audioBody.byteLength); + new DataView(frame.buffer).setInt16(0, headerBytes.byteLength); + frame.set(headerBytes, 2); + frame.set(audioBody, 2 + headerBytes.byteLength); + return frame.buffer; + }; + const metadataFrame = (text: string, offset: number) => + 'X-RequestId:1\r\nContent-Type:application/json; charset=utf-8\r\nPath:audio.metadata\r\n\r\n' + + JSON.stringify({ + Metadata: [ + { Type: 'WordBoundary', Data: { Offset: offset, Duration: 1000, text: { Text: text } } }, + ], + }); + + let sendCount = 0; + mockSocket.send.mockImplementation(() => { + sendCount++; + if (sendCount === 2) { + queueMicrotask(() => { + for (const cb of listeners['message'] ?? []) + cb({ data: metadataFrame('Hello', 1000000) }); + for (const cb of listeners['message'] ?? []) cb({ data: audioFrame() }); + for (const cb of listeners['message'] ?? []) + cb({ data: metadataFrame('brave', 6000000) }); + for (const cb of listeners['message'] ?? []) + cb({ data: 'X-RequestId:1\r\nPath:turn.end\r\n\r\n' }); + }); + } + }); + + globalThis.fetch = vi + .fn() + .mockResolvedValue({ status: 101, webSocket: mockSocket }) as unknown as typeof fetch; + + const { EdgeSpeechTTS } = await import('@/libs/edgeTTS'); + const tts = new EdgeSpeechTTS('wss'); + const { boundaries } = await tts.createWithBoundaries({ + lang: 'en-US', + text: 'Hello brave', + voice: 'en-US-AriaNeural', + rate: 1.0, + pitch: 1.0, + }); + expect(boundaries).toEqual([ + { offset: 1000000, duration: 1000, text: 'Hello' }, + { offset: 6000000, duration: 1000, text: 'brave' }, + ]); + }); +}); + +describe('parseAudioMetadataBody', () => { + test('extracts WordBoundary entries from an audio.metadata body', async () => { + const { parseAudioMetadataBody } = await import('@/libs/edgeTTS'); + const body = JSON.stringify({ + Metadata: [ + { + Type: 'WordBoundary', + Data: { + Offset: 1000000, + Duration: 4250000, + text: { Text: 'Dr.', Length: 3, BoundaryType: 'WordBoundary' }, + }, + }, + ], + }); + expect(parseAudioMetadataBody(body)).toEqual([ + { offset: 1000000, duration: 4250000, text: 'Dr.' }, + ]); + }); + + test('ignores non-WordBoundary metadata entries', async () => { + const { parseAudioMetadataBody } = await import('@/libs/edgeTTS'); + const body = JSON.stringify({ + Metadata: [ + { Type: 'SessionEnd', Data: { Offset: 99 } }, + { + Type: 'WordBoundary', + Data: { Offset: 5000000, Duration: 1000000, text: { Text: 'hello' } }, + }, + ], + }); + expect(parseAudioMetadataBody(body)).toEqual([ + { offset: 5000000, duration: 1000000, text: 'hello' }, + ]); + }); + + test('returns an empty list for malformed or empty bodies', async () => { + const { parseAudioMetadataBody } = await import('@/libs/edgeTTS'); + expect(parseAudioMetadataBody('not json')).toEqual([]); + expect(parseAudioMetadataBody('{}')).toEqual([]); + expect( + parseAudioMetadataBody(JSON.stringify({ Metadata: [{ Type: 'WordBoundary' }] })), + ).toEqual([]); + }); }); diff --git a/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts b/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts index 7fc5c10f..6a86f559 100644 --- a/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts +++ b/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts @@ -5,6 +5,13 @@ let createBehavior: () => Promise = () => Promise.resolve(undefined); // Shared mock control for createAudioUrl() and parsed SSML marks let createAudioUrlBehavior = vi.fn<() => Promise>(() => Promise.resolve('blob:mock-url')); +type MockAudioResult = { + url: string; + boundaries: Array<{ offset: number; duration: number; text: string }>; +}; +let createAudioBehavior = vi.fn<() => Promise>(() => + Promise.resolve({ url: 'blob:mock-url', boundaries: [] }), +); let parsedMarks: Array<{ name: string; text: string; language: string }> = []; // --- Mocks --- @@ -21,6 +28,7 @@ vi.mock('@/libs/edgeTTS', () => { static voices = voices; create = vi.fn().mockImplementation(() => createBehavior()); createAudioUrl = vi.fn().mockImplementation(() => createAudioUrlBehavior()); + createAudio = vi.fn().mockImplementation(() => createAudioBehavior()); }, EDGE_TTS_PROTOCOL: 'wss', }; @@ -63,6 +71,9 @@ describe('EdgeTTSClient', () => { beforeEach(() => { createBehavior = () => Promise.resolve(undefined); createAudioUrlBehavior = vi.fn<() => Promise>(() => Promise.resolve('blob:mock-url')); + createAudioBehavior = vi.fn<() => Promise>(() => + Promise.resolve({ url: 'blob:mock-url', boundaries: [] }), + ); parsedMarks = []; client = new EdgeTTSClient(); }); @@ -221,6 +232,12 @@ describe('EdgeTTSClient', () => { }); }); + describe('supportsWordBoundaries', () => { + test('returns true (Edge reports word-boundary timings)', () => { + expect(client.supportsWordBoundaries()).toBe(true); + }); + }); + describe('getGranularities', () => { test('returns array with sentence granularity only', () => { const granularities = client.getGranularities(); @@ -479,4 +496,137 @@ describe('EdgeTTSClient', () => { await expect(client.stop()).resolves.toBeUndefined(); }); }); + + describe('word boundary tracking during playback', () => { + class MockAudio { + static instances: MockAudio[] = []; + src = ''; + currentTime = 0; + preload = ''; + playbackRate = 1; + onended: ((e?: Event) => void) | null = null; + onerror: ((e?: unknown) => void) | null = null; + constructor() { + MockAudio.instances.push(this); + } + setAttribute() {} + play() { + return Promise.resolve(); + } + pause() {} + } + + let rafCallbacks: Map; + let rafId = 0; + const runRaf = () => { + const cbs = [...rafCallbacks.values()]; + rafCallbacks.clear(); + for (const cb of cbs) cb(0); + }; + const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + + let mockController: { + dispatchSpeakMark: ReturnType; + prepareSpeakWords: ReturnType; + dispatchSpeakWord: ReturnType; + }; + + beforeEach(() => { + MockAudio.instances = []; + rafCallbacks = new Map(); + vi.stubGlobal('Audio', MockAudio); + vi.stubGlobal('requestAnimationFrame', (cb: FrameRequestCallback) => { + rafCallbacks.set(++rafId, cb); + return rafId; + }); + vi.stubGlobal('cancelAnimationFrame', (id: number) => { + rafCallbacks.delete(id); + }); + mockController = { + dispatchSpeakMark: vi.fn(), + prepareSpeakWords: vi.fn(), + dispatchSpeakWord: vi.fn(), + }; + client = new EdgeTTSClient(mockController as unknown as TTSController); + parsedMarks = [{ name: '0', text: 'Hello brave world', language: 'en' }]; + createAudioBehavior = vi.fn(() => + Promise.resolve({ + url: 'blob:mock-url', + boundaries: [ + { offset: 1_000_000, duration: 4_000_000, text: 'Hello' }, + { offset: 6_000_000, duration: 4_000_000, text: 'brave' }, + { offset: 11_000_000, duration: 4_000_000, text: 'world' }, + ], + }), + ); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + const startSpeak = async () => { + await client.init(); + const it = client.speak('', new AbortController().signal); + const first = await it.next(); + expect((first.value as { code: string }).code).toBe('boundary'); + const resultPromise = it.next(); + await flush(); + const audio = MockAudio.instances.at(-1)!; + return { it, resultPromise, audio }; + }; + + test('prepares speak words and dispatches word indexes as playback advances', async () => { + const { resultPromise, audio } = await startSpeak(); + + expect(mockController.prepareSpeakWords).toHaveBeenCalledWith(['Hello', 'brave', 'world']); + + audio.currentTime = 0.11; + runRaf(); + expect(mockController.dispatchSpeakWord).toHaveBeenCalledWith(0); + + audio.currentTime = 0.65; + runRaf(); + expect(mockController.dispatchSpeakWord).toHaveBeenLastCalledWith(1); + + // Same word index is not re-dispatched on subsequent frames. + const callCount = mockController.dispatchSpeakWord.mock.calls.length; + runRaf(); + expect(mockController.dispatchSpeakWord.mock.calls.length).toBe(callCount); + + audio.onended?.(); + const result = await resultPromise; + expect((result.value as { code: string }).code).toBe('end'); + }); + + test('stops dispatching after the chunk ends', async () => { + const { resultPromise, audio } = await startSpeak(); + + audio.currentTime = 0.11; + runRaf(); + const callCount = mockController.dispatchSpeakWord.mock.calls.length; + + audio.onended?.(); + await resultPromise; + + audio.currentTime = 1.2; + runRaf(); + expect(mockController.dispatchSpeakWord.mock.calls.length).toBe(callCount); + }); + + test('hands empty words to the controller and does not track when no boundaries', async () => { + createAudioBehavior = vi.fn(() => Promise.resolve({ url: 'blob:mock-url', boundaries: [] })); + const { resultPromise, audio } = await startSpeak(); + + // Empty words are still forwarded so the controller can draw the + // sentence-highlight fallback; no per-word tracking is started. + expect(mockController.prepareSpeakWords).toHaveBeenCalledWith([]); + audio.currentTime = 0.5; + runRaf(); + expect(mockController.dispatchSpeakWord).not.toHaveBeenCalled(); + + audio.onended?.(); + await resultPromise; + }); + }); }); diff --git a/apps/readest-app/src/__tests__/services/tts-controller.test.ts b/apps/readest-app/src/__tests__/services/tts-controller.test.ts index 0d8dc06a..54d896b8 100644 --- a/apps/readest-app/src/__tests__/services/tts-controller.test.ts +++ b/apps/readest-app/src/__tests__/services/tts-controller.test.ts @@ -97,6 +97,7 @@ function createMockTTSClient(name: string): TTSClient { getAllVoices: vi.fn().mockResolvedValue([]), getVoices: vi.fn().mockResolvedValue([]), getGranularities: vi.fn().mockReturnValue(['word', 'sentence'] as TTSGranularity[]), + supportsWordBoundaries: vi.fn().mockReturnValue(name === 'edge'), getVoiceId: vi.fn().mockReturnValue('voice-1'), getSpeakingLang: vi.fn().mockReturnValue('en'), }; @@ -553,6 +554,191 @@ describe('TTSController', () => { }); }); + describe('word highlighting (prepareSpeakWords / dispatchSpeakWord)', () => { + const getOverlayer = () => + ( + mockView.renderer.getContents() as unknown as Array<{ + overlayer: { add: ReturnType; remove: ReturnType }; + }> + )[0]!.overlayer; + + const makeSentenceRange = () => { + document.body.innerHTML = '

Hello brave world

'; + const textNode = document.body.firstElementChild!.firstChild as Text; + const range = document.createRange(); + range.setStart(textNode, 0); + range.setEnd(textNode, textNode.length); + return range; + }; + + const armWithSentence = async (range: Range, markName = '0') => { + await controller.initViewTTS(0); + mockView.tts = { + setMark: vi.fn().mockReturnValue(range), + getLastRange: vi.fn().mockImplementation(() => range.cloneRange()), + } as unknown as FoliateView['tts']; + controller.dispatchSpeakMark({ + offset: 0, + name: markName, + text: 'Hello brave world', + language: 'en', + }); + }; + + test('prepareSpeakWords immediately highlights the first word (no sentence flash)', async () => { + await armWithSentence(makeSentenceRange()); + vi.mocked(mockView.getCFI).mockClear(); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('Hello'); + expect(getOverlayer().add).toHaveBeenCalledTimes(1); + }); + + test('prepareSpeakWords with no words falls back to the full-sentence highlight', async () => { + await armWithSentence(makeSentenceRange()); + vi.mocked(mockView.getCFI).mockClear(); + controller.prepareSpeakWords([]); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('Hello brave world'); + expect(getOverlayer().add).toHaveBeenCalledTimes(1); + }); + + test('dispatchSpeakWord highlights the word sub-range of the current sentence', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + + vi.mocked(mockView.getCFI).mockClear(); + getOverlayer().add.mockClear(); + controller.dispatchSpeakWord(1); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('brave'); + expect(getOverlayer().add).toHaveBeenCalledTimes(1); + expect(getOverlayer().add.mock.calls[0]![0]).toBe('tts-highlight'); + }); + + test('word indexes can be dispatched out of order after a seek back', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + + controller.dispatchSpeakWord(2); + vi.mocked(mockView.getCFI).mockClear(); + controller.dispatchSpeakWord(0); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('Hello'); + }); + + test('does not highlight words for one-time marks (name -1)', async () => { + await armWithSentence(makeSentenceRange(), '-1'); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + controller.dispatchSpeakWord(0); + + expect(getOverlayer().add).not.toHaveBeenCalled(); + }); + + test('a new speak mark clears previously prepared words', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + expect(getOverlayer().add).toHaveBeenCalledTimes(1); + + controller.dispatchSpeakMark({ + offset: 0, + name: '1', + text: 'Hello brave world', + language: 'en', + }); + getOverlayer().add.mockClear(); + controller.dispatchSpeakWord(0); + expect(getOverlayer().add).not.toHaveBeenCalled(); + }); + + test('unmatched first word does not highlight but later words still align', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['BOGUS', 'brave']); + + // First word unmatched → no eager highlight. + expect(getOverlayer().add).not.toHaveBeenCalled(); + + vi.mocked(mockView.getCFI).mockClear(); + controller.dispatchSpeakWord(1); + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('brave'); + }); + + test('reapplyCurrentHighlight re-draws the current word during word mode', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + controller.dispatchSpeakWord(1); + + vi.mocked(mockView.getCFI).mockClear(); + controller.reapplyCurrentHighlight(); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('brave'); + }); + + test('reapplyCurrentHighlight re-draws the whole sentence when not in word mode', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords([]); + + vi.mocked(mockView.getCFI).mockClear(); + controller.reapplyCurrentHighlight(); + + const getCFICalls = vi.mocked(mockView.getCFI).mock.calls; + expect(getCFICalls.length).toBeGreaterThanOrEqual(1); + expect(String(getCFICalls[0]![1])).toBe('Hello brave world'); + }); + + test('dispatchSpeakWord emits tts-highlight-word for word-level page following', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + + const listener = vi.fn(); + controller.addEventListener('tts-highlight-word', listener); + vi.mocked(mockView.getCFI).mockReturnValue('cfi-word'); + controller.dispatchSpeakWord(1); + + expect(listener).toHaveBeenCalledTimes(1); + const ev = listener.mock.calls[0]![0] as CustomEvent; + expect(ev.detail).toEqual({ cfi: 'cfi-word' }); + }); + + test('dispatchSpeakWord does not emit a word event when the word is unmatched', async () => { + await armWithSentence(makeSentenceRange()); + controller.prepareSpeakWords(['BOGUS', 'brave']); + + const listener = vi.fn(); + controller.addEventListener('tts-highlight-word', listener); + controller.dispatchSpeakWord(0); // unmatched → no range, no event + expect(listener).not.toHaveBeenCalled(); + }); + + test('getCurrentHighlightCfi returns the word cfi in word mode, null otherwise', async () => { + await armWithSentence(makeSentenceRange()); + // Not in word mode until prepareSpeakWords with words. + expect(controller.getCurrentHighlightCfi()).toBeNull(); + + vi.mocked(mockView.getCFI).mockReturnValue('cfi-word'); + controller.prepareSpeakWords(['Hello', 'brave', 'world']); + controller.dispatchSpeakWord(1); + expect(controller.getCurrentHighlightCfi()).toBe('cfi-word'); + + // Empty (sentence fallback) leaves word mode → null so the caller uses + // the sentence-level ttsLocation. + controller.prepareSpeakWords([]); + expect(controller.getCurrentHighlightCfi()).toBeNull(); + }); + }); + describe('getSpokenSentence', () => { test('returns the trimmed text and cfi of the current sentence', async () => { await controller.initViewTTS(0); diff --git a/apps/readest-app/src/__tests__/services/tts-word-highlight.test.ts b/apps/readest-app/src/__tests__/services/tts-word-highlight.test.ts new file mode 100644 index 00000000..c6879188 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/tts-word-highlight.test.ts @@ -0,0 +1,151 @@ +import { describe, test, expect } from 'vitest'; +import { + computeWordOffsets, + findBoundaryIndexAtTime, + getTextSubRange, +} from '@/services/tts/wordHighlight'; + +describe('computeWordOffsets', () => { + test('maps words to offsets in order', () => { + const text = 'Dr. Smith bought 23 apples'; + const offsets = computeWordOffsets(text, ['Dr.', 'Smith', 'bought', '23', 'apples']); + expect(offsets).toEqual([ + { start: 0, end: 3 }, + { start: 4, end: 9 }, + { start: 10, end: 16 }, + { start: 17, end: 19 }, + { start: 20, end: 26 }, + ]); + }); + + test('repeated words advance past earlier occurrences', () => { + const text = 'the cat and the hat'; + const offsets = computeWordOffsets(text, ['the', 'cat', 'and', 'the', 'hat']); + expect(offsets![3]).toEqual({ start: 12, end: 15 }); + }); + + test('short word is not matched inside an earlier longer word', () => { + const text = 'and a cat'; + const offsets = computeWordOffsets(text, ['and', 'a', 'cat']); + expect(offsets).toEqual([ + { start: 0, end: 3 }, + { start: 4, end: 5 }, + { start: 6, end: 9 }, + ]); + }); + + test('missing word yields null without advancing the cursor', () => { + const text = 'hello world'; + const offsets = computeWordOffsets(text, ['hello', 'BOGUS', 'world']); + expect(offsets).toEqual([{ start: 0, end: 5 }, null, { start: 6, end: 11 }]); + }); + + test('empty or whitespace-only word yields null', () => { + const offsets = computeWordOffsets('hello world', ['hello', ' ', 'world']); + expect(offsets).toEqual([{ start: 0, end: 5 }, null, { start: 6, end: 11 }]); + }); + + test('words with attached punctuation match document text verbatim', () => { + const text = 'paid $5.50 for them, you know.'; + const offsets = computeWordOffsets(text, ['paid', '$5.50', 'for', 'them', 'you', 'know']); + expect(offsets).toEqual([ + { start: 0, end: 4 }, + { start: 5, end: 10 }, + { start: 11, end: 14 }, + { start: 15, end: 19 }, + { start: 21, end: 24 }, + { start: 25, end: 29 }, + ]); + }); +}); + +describe('findBoundaryIndexAtTime', () => { + // offsets are in 100-nanosecond ticks: 10_000_000 ticks = 1 second + const boundaries = [{ offset: 1_000_000 }, { offset: 5_000_000 }, { offset: 11_000_000 }]; + + test('returns -1 for an empty list', () => { + expect(findBoundaryIndexAtTime([], 1)).toBe(-1); + }); + + test('returns -1 before the first boundary', () => { + expect(findBoundaryIndexAtTime(boundaries, 0.05)).toBe(-1); + }); + + test('returns the boundary exactly at its start time', () => { + expect(findBoundaryIndexAtTime(boundaries, 0.1)).toBe(0); + }); + + test('returns the latest boundary at or before the given time', () => { + expect(findBoundaryIndexAtTime(boundaries, 0.7)).toBe(1); + }); + + test('returns the last boundary after the stream end', () => { + expect(findBoundaryIndexAtTime(boundaries, 10)).toBe(2); + }); +}); + +describe('getTextSubRange', () => { + const makeBaseRange = (html: string) => { + document.body.innerHTML = html; + const root = document.body.firstElementChild!; + const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT); + const textNodes: Text[] = []; + for (let n = walker.nextNode(); n; n = walker.nextNode()) textNodes.push(n as Text); + const range = document.createRange(); + range.setStart(textNodes[0]!, 0); + range.setEnd(textNodes.at(-1)!, textNodes.at(-1)!.length); + return range; + }; + + test('extracts a word inside a single text node', () => { + const base = makeBaseRange('

Hello brave world

'); + const sub = getTextSubRange(base, 6, 11); + expect(sub?.toString()).toBe('brave'); + }); + + test('extracts a word inside a nested element', () => { + const base = makeBaseRange('

Hello brave world

'); + expect(base.toString()).toBe('Hello brave world'); + const sub = getTextSubRange(base, 6, 11); + expect(sub?.toString()).toBe('brave'); + }); + + test('extracts a span crossing element boundaries', () => { + const base = makeBaseRange('

Hello brave world

'); + const sub = getTextSubRange(base, 4, 14); + expect(sub?.toString()).toBe('o brave wo'); + }); + + test('offsets are relative to a base range starting mid-node', () => { + document.body.innerHTML = '

Hello brave world

'; + const textNode = document.body.firstElementChild!.firstChild as Text; + const base = document.createRange(); + base.setStart(textNode, 3); + base.setEnd(textNode, textNode.length); + expect(base.toString()).toBe('lo brave world'); + const sub = getTextSubRange(base, 3, 8); + expect(sub?.toString()).toBe('brave'); + }); + + test('ignores text nodes outside the base range', () => { + document.body.innerHTML = '

before Hello world after

'; + const textNode = document.getElementById('target')!.firstChild as Text; + const base = document.createRange(); + base.setStart(textNode, 0); + base.setEnd(textNode, textNode.length); + const sub = getTextSubRange(base, 6, 11); + expect(sub?.toString()).toBe('world'); + }); + + test('returns null when offsets exceed the base range text', () => { + const base = makeBaseRange('

Hello

'); + expect(getTextSubRange(base, 0, 100)).toBeNull(); + }); + + test('returns null for an empty or inverted span', () => { + const base = makeBaseRange('

Hello

'); + expect(getTextSubRange(base, 2, 2)).toBeNull(); + expect(getTextSubRange(base, 3, 2)).toBeNull(); + expect(getTextSubRange(base, -1, 2)).toBeNull(); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/web-speech-client.test.ts b/apps/readest-app/src/__tests__/services/web-speech-client.test.ts index b1586bcf..d6f80b35 100644 --- a/apps/readest-app/src/__tests__/services/web-speech-client.test.ts +++ b/apps/readest-app/src/__tests__/services/web-speech-client.test.ts @@ -64,4 +64,8 @@ describe('WebSpeechClient getVoices', () => { const en = await client.getVoices('en-US'); expect(en[0]!.voices.map((v) => v.name)).not.toContain('Denise'); }); + + test('does not support word boundaries (sentence highlight only)', () => { + expect(client.supportsWordBoundaries()).toBe(false); + }); }); diff --git a/apps/readest-app/src/app/api/tts/edge/route.ts b/apps/readest-app/src/app/api/tts/edge/route.ts index 9fd19be0..ac676fe5 100644 --- a/apps/readest-app/src/app/api/tts/edge/route.ts +++ b/apps/readest-app/src/app/api/tts/edge/route.ts @@ -1,5 +1,10 @@ import { NextRequest, NextResponse } from 'next/server'; -import { EdgeSpeechTTS, EdgeTTSPayload } from '@/libs/edgeTTS'; +import { + EdgeSpeechTTS, + EdgeTTSPayload, + serializeWordBoundaries, + WORD_BOUNDARIES_HEADER, +} from '@/libs/edgeTTS'; import { validateUserAndToken } from '@/utils/access'; const getLangFromVoice = (voiceId: string): string => { @@ -69,7 +74,7 @@ export async function POST(request: NextRequest) { }; const tts = new EdgeSpeechTTS(); - const response = await tts.create(payload); + const { response, boundaries } = await tts.createWithBoundaries(payload); const arrayBuffer = await response.arrayBuffer(); return new NextResponse(arrayBuffer, { @@ -77,6 +82,7 @@ export async function POST(request: NextRequest) { headers: { 'Content-Type': 'audio/mpeg', 'Content-Length': arrayBuffer.byteLength.toString(), + [WORD_BOUNDARIES_HEADER]: serializeWordBoundaries(boundaries), }, }); } catch (error) { diff --git a/apps/readest-app/src/app/reader/hooks/useTTSControl.ts b/apps/readest-app/src/app/reader/hooks/useTTSControl.ts index 9e540769..1c405ee6 100644 --- a/apps/readest-app/src/app/reader/hooks/useTTSControl.ts +++ b/apps/readest-app/src/app/reader/hooks/useTTSControl.ts @@ -258,13 +258,50 @@ export const useTTSControl = ({ bookKey, onRequestHidePanel }: UseTTSControlProp } }; + // Word-level page following: turn the page as soon as the spoken word + // moves off the visible page, instead of waiting for the next sentence's + // mark. Only navigates when the word is outside the visible range, so + // on-page words don't trigger relocations. + const handleHighlightWord = (e: Event) => { + const { cfi } = (e as CustomEvent<{ cfi: string }>).detail; + const view = getView(bookKey); + if (!cfi || !view || !followingTTSLocationRef.current) return; + + const hlContents = view.renderer.getContents(); + const hlPrimaryIdx = view.renderer.primaryIndex; + const { doc, index: viewSectionIndex } = (hlContents.find((x) => x.index === hlPrimaryIdx) ?? + hlContents[0]) as { doc: Document; index?: number }; + + const { anchor, index: ttsSectionIndex } = view.resolveCFI(cfi); + // Cross-section navigation is driven by the sentence-level mark handler. + if (viewSectionIndex !== ttsSectionIndex) return; + if (hlContents.some(({ doc }) => (doc.getSelection()?.toString().length ?? 0) > 0)) return; + + const wordRange = anchor(doc); + const visibleRange = getProgress(bookKey)?.range as Range | undefined; + if (!wordRange || !visibleRange) return; + + try { + const ahead = wordRange.compareBoundaryPoints(Range.END_TO_START, visibleRange) > 0; + const behind = wordRange.compareBoundaryPoints(Range.START_TO_END, visibleRange) < 0; + if (ahead || behind) { + view.renderer.scrollToAnchor?.(wordRange); + } + } catch { + // Ranges may briefly belong to different documents during a section + // change; the mark handler takes over in that case. + } + }; + ttsController.addEventListener('tts-need-auth', handleNeedAuth); ttsController.addEventListener('tts-speak-mark', handleSpeakMark); ttsController.addEventListener('tts-highlight-mark', handleHighlightMark); + ttsController.addEventListener('tts-highlight-word', handleHighlightWord); return () => { ttsController.removeEventListener('tts-need-auth', handleNeedAuth); ttsController.removeEventListener('tts-speak-mark', handleSpeakMark); ttsController.removeEventListener('tts-highlight-mark', handleHighlightMark); + ttsController.removeEventListener('tts-highlight-word', handleHighlightWord); }; // eslint-disable-next-line react-hooks/exhaustive-deps }, [ttsController, bookKey]); @@ -277,18 +314,22 @@ export const useTTSControl = ({ bookKey, onRequestHidePanel }: UseTTSControlProp const ttsController = ttsControllerRef.current; if (!ttsController) return; - const view = getView(bookKey); const viewSettings = getViewSettings(bookKey); const ttsLocation = viewSettings?.ttsLocation; const { location } = progress || {}; if (!location || !ttsLocation) return; - if (isCfiInLocation(ttsLocation, location)) { + // Check the actual highlighted position against the view. During + // word-by-word playback the word can sit on a different page than the + // sentence's ttsLocation (a sentence spanning a page break), so the word + // position is the correct reference — otherwise the back-to-TTS button + // wrongly appears after the view follows the word onto the next page. + const highlightCfi = ttsController.getCurrentHighlightCfi() ?? ttsLocation; + if (isCfiInLocation(highlightCfi, location)) { setShowBackToCurrentTTSLocation(false); - const range = view?.tts?.getLastRange() as Range | null; - if (range) { - view?.tts?.highlight(range); - } + // Word-aware re-apply: re-draws the current word during word-by-word + // playback instead of redrawing the whole sentence over it. + ttsController.reapplyCurrentHighlight(); } else { const msSinceSectionChange = Date.now() - sectionChangingTimestampRef.current; if (msSinceSectionChange < 2000) return; diff --git a/apps/readest-app/src/libs/edgeTTS.ts b/apps/readest-app/src/libs/edgeTTS.ts index 872c768d..bd9480e1 100644 --- a/apps/readest-app/src/libs/edgeTTS.ts +++ b/apps/readest-app/src/libs/edgeTTS.ts @@ -276,6 +276,73 @@ export interface EdgeTTSPayload { pitch: number; } +// A word boundary reported by the Edge TTS service via `audio.metadata` +// frames. Offsets and durations are in 100-nanosecond ticks relative to the +// start of the audio stream; `text` is the verbatim span of the input text. +export interface TTSWordBoundary { + offset: number; + duration: number; + text: string; +} + +interface AudioMetadataEntry { + Type?: string; + Data?: { + Offset?: number; + Duration?: number; + text?: { Text?: string }; + }; +} + +export const parseAudioMetadataBody = (body: string): TTSWordBoundary[] => { + try { + const parsed = JSON.parse(body) as { Metadata?: AudioMetadataEntry[] }; + const boundaries: TTSWordBoundary[] = []; + for (const entry of parsed.Metadata ?? []) { + if (entry.Type !== 'WordBoundary') continue; + const offset = entry.Data?.Offset; + const text = entry.Data?.text?.Text; + if (typeof offset !== 'number' || typeof text !== 'string' || !text) continue; + boundaries.push({ offset, duration: entry.Data?.Duration ?? 0, text }); + } + return boundaries; + } catch { + return []; + } +}; + +export interface EdgeSpeechAudio { + response: Response; + boundaries: TTSWordBoundary[]; +} + +// Response header used to carry word boundaries through the authenticated +// HTTPS proxy route (`/api/tts/edge`), which streams only the audio body. +export const WORD_BOUNDARIES_HEADER = 'X-TTS-Word-Boundaries'; + +// HTTP header values must be ASCII, but boundary `text` can be any script +// (em-dashes, CJK, accents). Percent-encode the JSON so the header stays +// ASCII-safe across Node, browsers, and Cloudflare Workers. +export const serializeWordBoundaries = (boundaries: TTSWordBoundary[]): string => + encodeURIComponent(JSON.stringify(boundaries)); + +export const parseWordBoundariesHeader = (value: string | null): TTSWordBoundary[] => { + if (!value) return []; + try { + const parsed = JSON.parse(decodeURIComponent(value)); + if (!Array.isArray(parsed)) return []; + return parsed.filter( + (b: unknown): b is TTSWordBoundary => + !!b && + typeof (b as TTSWordBoundary).offset === 'number' && + typeof (b as TTSWordBoundary).duration === 'number' && + typeof (b as TTSWordBoundary).text === 'string', + ); + } catch { + return []; + } +}; + const hashPayload = (payload: EdgeTTSPayload): string => { const base = JSON.stringify(payload); return md5(base); @@ -291,6 +358,7 @@ export class EdgeSpeechTTS { URL.revokeObjectURL(url); } }); + private static boundariesCache = new LRUCache(200); private protocol: EDGE_TTS_PROTOCOL = 'wss'; constructor(protocol?: EDGE_TTS_PROTOCOL) { @@ -322,7 +390,7 @@ export class EdgeSpeechTTS { return response; } - async #fetchEdgeSpeechWs({ lang, text, voice, rate }: EdgeTTSPayload): Promise { + async #fetchEdgeSpeechWs({ lang, text, voice, rate }: EdgeTTSPayload): Promise { const connectId = randomMd5(); const params = new URLSearchParams({ ConnectionId: connectId, @@ -408,17 +476,19 @@ export class EdgeSpeechTTS { const TauriWebSocket = (await import('@tauri-apps/plugin-websocket')).default; const ws = await TauriWebSocket.connect(url, { headers: baseHeaders }); let audioData = new ArrayBuffer(0); + const boundaries: TTSWordBoundary[] = []; const messageUnlisten = await ws.addListener((msg) => { if (msg.type === 'Text') { - const { headers } = getHeadersAndData(msg.data as string); - if (headers['Path'] === 'turn.end') { + const { headers, body } = getHeadersAndData(msg.data as string); + if (headers['Path'] === 'audio.metadata') { + boundaries.push(...parseAudioMetadataBody(body.trim())); + } else if (headers['Path'] === 'turn.end') { ws.disconnect(); messageUnlisten(); if (!audioData.byteLength) { return reject(new Error('No audio data received.')); } - const res = new Response(audioData); - resolve(res); + resolve({ response: new Response(audioData), boundaries }); } } else if (msg.type === 'Binary') { let buffer: ArrayBufferLike; @@ -445,7 +515,10 @@ export class EdgeSpeechTTS { } }); } else if (isCloudflareWorkers()) { - return new Promise((resolve, reject) => { + // The Workers path backs the HTTPS proxy route. It captures both the + // audio body and the word boundaries (audio.metadata frames) so the + // route can forward boundaries via the WORD_BOUNDARIES_HEADER. + return new Promise((resolve, reject) => { (async () => { try { // Cloudflare Workers cannot use the `ws` npm package because it @@ -469,6 +542,7 @@ export class EdgeSpeechTTS { const ws = upgradeResponse.webSocket; let audioData = new ArrayBuffer(0); + const boundaries: TTSWordBoundary[] = []; let settled = false; // Cloudflare Workers deliver binary WebSocket frames as `Blob`, // whose conversion to bytes (`blob.arrayBuffer()`) is async. @@ -504,7 +578,7 @@ export class EdgeSpeechTTS { if (!audioData.byteLength) { reject(new Error('No audio data received.')); } else { - resolve(new Response(audioData)); + resolve({ response: new Response(audioData), boundaries }); } }; @@ -512,7 +586,11 @@ export class EdgeSpeechTTS { if (settled) return; const data = event.data; if (typeof data === 'string') { - const { headers } = getHeadersAndData(data); + const { headers, body } = getHeadersAndData(data); + if (headers['Path'] === 'audio.metadata') { + boundaries.push(...parseAudioMetadataBody(body.trim())); + return; + } if (headers['Path'] === 'turn.end') { // Wait for any in-flight Blob decodes to complete before // deciding whether audio was received. @@ -582,12 +660,18 @@ export class EdgeSpeechTTS { }); } else { return new Promise((resolve, reject) => { - const ws = new WebSocket(url, { - headers: baseHeaders, - }); + // In browsers isomorphic-ws is the native WebSocket, whose second + // argument is a subprotocol list — passing an options object throws + // SyntaxError. Custom headers are only supported (and only needed) + // in Node, where `ws` accepts (url, options). + const ws = + typeof window === 'undefined' + ? new WebSocket(url, { headers: baseHeaders }) + : new WebSocket(url); ws.binaryType = 'arraybuffer'; let audioData = new ArrayBuffer(0); + const boundaries: TTSWordBoundary[] = []; ws.addEventListener('open', () => { ws.send(config); @@ -596,14 +680,15 @@ export class EdgeSpeechTTS { ws.addEventListener('message', (event: WebSocket.MessageEvent) => { if (typeof event.data === 'string') { - const { headers } = getHeadersAndData(event.data); - if (headers['Path'] === 'turn.end') { + const { headers, body } = getHeadersAndData(event.data); + if (headers['Path'] === 'audio.metadata') { + boundaries.push(...parseAudioMetadataBody(body.trim())); + } else if (headers['Path'] === 'turn.end') { ws.close(); if (!audioData.byteLength) { return reject(new Error('No audio data received.')); } - const res = new Response(audioData); - resolve(res); + resolve({ response: new Response(audioData), boundaries }); } } else if (event.data instanceof ArrayBuffer) { const dataView = new DataView(event.data); @@ -631,29 +716,49 @@ export class EdgeSpeechTTS { } } - async create(payload: EdgeTTSPayload): Promise { + async #fetchEdgeSpeech(payload: EdgeTTSPayload): Promise { if (this.protocol === 'https') { - return this.#fetchEdgeSpeechHttp(payload); + // The HTTPS proxy streams the audio body and carries word boundaries in + // the WORD_BOUNDARIES_HEADER response header (see /api/tts/edge route). + const response = await this.#fetchEdgeSpeechHttp(payload); + return { + response, + boundaries: parseWordBoundariesHeader(response.headers.get(WORD_BOUNDARIES_HEADER)), + }; } else { return this.#fetchEdgeSpeechWs(payload); } } - async createAudioUrl(payload: EdgeTTSPayload): Promise { + async create(payload: EdgeTTSPayload): Promise { + return (await this.#fetchEdgeSpeech(payload)).response; + } + + // Server-side helper for the /api/tts/edge route: returns the audio Response + // together with the captured word boundaries so the route can forward them. + async createWithBoundaries(payload: EdgeTTSPayload): Promise { + return this.#fetchEdgeSpeech(payload); + } + + async createAudio( + payload: EdgeTTSPayload, + ): Promise<{ url: string; boundaries: TTSWordBoundary[] }> { const cacheKey = hashPayload(payload); - if (EdgeSpeechTTS.audioUrlCache.has(cacheKey)) { - return EdgeSpeechTTS.audioUrlCache.get(cacheKey)!; - } - try { - const res = await this.create(payload); - const arrayBuffer = await res.arrayBuffer(); - const blob = new Blob([arrayBuffer], { type: 'audio/mpeg' }); - const objectUrl = URL.createObjectURL(blob); - EdgeSpeechTTS.audioCache.set(cacheKey, blob); - EdgeSpeechTTS.audioUrlCache.set(cacheKey, objectUrl); - return objectUrl; - } catch (error) { - throw error; + const cachedUrl = EdgeSpeechTTS.audioUrlCache.get(cacheKey); + if (cachedUrl) { + return { url: cachedUrl, boundaries: EdgeSpeechTTS.boundariesCache.get(cacheKey) ?? [] }; } + const { response, boundaries } = await this.#fetchEdgeSpeech(payload); + const arrayBuffer = await response.arrayBuffer(); + const blob = new Blob([arrayBuffer], { type: 'audio/mpeg' }); + const objectUrl = URL.createObjectURL(blob); + EdgeSpeechTTS.audioCache.set(cacheKey, blob); + EdgeSpeechTTS.audioUrlCache.set(cacheKey, objectUrl); + EdgeSpeechTTS.boundariesCache.set(cacheKey, boundaries); + return { url: objectUrl, boundaries }; + } + + async createAudioUrl(payload: EdgeTTSPayload): Promise { + return (await this.createAudio(payload)).url; } } diff --git a/apps/readest-app/src/services/tts/EdgeTTSClient.ts b/apps/readest-app/src/services/tts/EdgeTTSClient.ts index 219b9ae9..268fa9eb 100644 --- a/apps/readest-app/src/services/tts/EdgeTTSClient.ts +++ b/apps/readest-app/src/services/tts/EdgeTTSClient.ts @@ -1,12 +1,13 @@ import { getUserLocale } from '@/utils/misc'; import { isSameLang } from '@/utils/lang'; import { TTSClient, TTSMessageEvent } from './TTSClient'; -import { EdgeSpeechTTS, EdgeTTSPayload, EDGE_TTS_PROTOCOL } from '@/libs/edgeTTS'; +import { EdgeSpeechTTS, EdgeTTSPayload, EDGE_TTS_PROTOCOL, TTSWordBoundary } from '@/libs/edgeTTS'; import { TTSGranularity, TTSVoice, TTSVoicesGroup } from './types'; import { AppService } from '@/types/system'; import { parseSSMLMarks } from '@/utils/ssml'; import { TTSController } from './TTSController'; import { TTSUtils } from './TTSUtils'; +import { findBoundaryIndexAtTime } from './wordHighlight'; export class EdgeTTSClient implements TTSClient { name = 'edge-tts'; @@ -27,6 +28,7 @@ export class EdgeTTSClient implements TTSClient { #pausedAt = 0; #startedAt = 0; #fadeCompensation: number | null = null; + #wordTrackingRafId: number | null = null; constructor(controller?: TTSController, appService?: AppService | null) { this.controller = controller; @@ -161,9 +163,11 @@ export class EdgeTTSClient implements TTSClient { const { language: voiceLang } = mark; const voiceId = await this.getVoiceIdFromLang(voiceLang); this.#speakingLang = voiceLang; - const audioUrl = await this.#edgeTTS?.createAudioUrl( + const audioResult = await this.#edgeTTS?.createAudio( this.getPayload(voiceLang, mark.text, voiceId), ); + const audioUrl = audioResult?.url; + const boundaries = audioResult?.boundaries ?? []; if (signal.aborted) { yield { code: 'error', message: 'Aborted' } as TTSMessageEvent; break; @@ -177,6 +181,7 @@ export class EdgeTTSClient implements TTSClient { const result = await new Promise((resolve) => { const cleanUp = () => { + this.#stopWordTracking(); audio.onended = null; audio.onerror = null; audio.src = ''; @@ -207,6 +212,7 @@ export class EdgeTTSClient implements TTSClient { }; this.#isPlaying = true; audio.src = audioUrl || ''; + this.#startWordTracking(audio, boundaries); if (!this.appService?.isLinuxApp) { audio.playbackRate = this.#rate; } @@ -243,6 +249,39 @@ export class EdgeTTSClient implements TTSClient { await this.stopInternal(); } + // Follow playback time against the word-boundary metadata of the current + // chunk and tell the controller which word is being spoken so it can + // highlight it. Word offsets are in media time, so audio.playbackRate and + // pause/resume (including the resume rewind on iOS) are handled naturally + // by polling audio.currentTime. + #startWordTracking(audio: HTMLAudioElement, boundaries: TTSWordBoundary[]) { + this.#stopWordTracking(); + const controller = this.controller; + if (!controller) return; + // Always hand the words to the controller — with boundaries it highlights + // word-by-word; with none it draws the sentence highlight that was + // suppressed at mark dispatch (see TTSController.prepareSpeakWords). + controller.prepareSpeakWords(boundaries.map((boundary) => boundary.text)); + if (!boundaries.length) return; + let lastIndex = -1; + const tick = () => { + const index = findBoundaryIndexAtTime(boundaries, audio.currentTime); + if (index !== lastIndex && index >= 0) { + lastIndex = index; + controller.dispatchSpeakWord(index); + } + this.#wordTrackingRafId = requestAnimationFrame(tick); + }; + this.#wordTrackingRafId = requestAnimationFrame(tick); + } + + #stopWordTracking() { + if (this.#wordTrackingRafId !== null) { + cancelAnimationFrame(this.#wordTrackingRafId); + this.#wordTrackingRafId = null; + } + } + async pause() { if (!this.#isPlaying || !this.#audioElement) return true; this.#pausedAt = this.#audioElement.currentTime - this.#startedAt; @@ -281,6 +320,7 @@ export class EdgeTTSClient implements TTSClient { } private async stopInternal() { + this.#stopWordTracking(); this.#isPlaying = false; this.#pausedAt = 0; this.#startedAt = 0; @@ -340,6 +380,10 @@ export class EdgeTTSClient implements TTSClient { this.#primaryLang = lang; } + supportsWordBoundaries(): boolean { + return true; + } + getGranularities(): TTSGranularity[] { return ['sentence']; } diff --git a/apps/readest-app/src/services/tts/NativeTTSClient.ts b/apps/readest-app/src/services/tts/NativeTTSClient.ts index 30974d55..d67de246 100644 --- a/apps/readest-app/src/services/tts/NativeTTSClient.ts +++ b/apps/readest-app/src/services/tts/NativeTTSClient.ts @@ -296,6 +296,10 @@ export class NativeTTSClient implements TTSClient { this.#primaryLang = lang; } + supportsWordBoundaries(): boolean { + return false; + } + getGranularities(): TTSGranularity[] { return ['sentence']; } diff --git a/apps/readest-app/src/services/tts/TTSClient.ts b/apps/readest-app/src/services/tts/TTSClient.ts index 330f91a2..f4630678 100644 --- a/apps/readest-app/src/services/tts/TTSClient.ts +++ b/apps/readest-app/src/services/tts/TTSClient.ts @@ -24,6 +24,9 @@ export interface TTSClient { getAllVoices(): Promise; getVoices(lang: string): Promise; getGranularities(): TTSGranularity[]; + // Whether this client reports word-boundary timings during playback so the + // controller can highlight word-by-word (and suppress the sentence highlight). + supportsWordBoundaries(): boolean; getVoiceId(): string; getSpeakingLang(): string; } diff --git a/apps/readest-app/src/services/tts/TTSController.ts b/apps/readest-app/src/services/tts/TTSController.ts index 031a3f57..917b46d1 100644 --- a/apps/readest-app/src/services/tts/TTSController.ts +++ b/apps/readest-app/src/services/tts/TTSController.ts @@ -10,6 +10,7 @@ import { EdgeTTSClient } from './EdgeTTSClient'; import { TTSUtils } from './TTSUtils'; import { TTSClient } from './TTSClient'; import { isValidLang } from '@/utils/lang'; +import { computeWordOffsets, getTextSubRange, TTSWordOffset } from './wordHighlight'; type TTSState = | 'stopped' @@ -35,6 +36,20 @@ export class TTSController extends EventTarget { #ttsSectionIndex: number = -1; + // Word-level highlight state for the currently spoken chunk. Armed by a + // successful dispatchSpeakMark, populated by prepareSpeakWords when a TTS + // client has word-boundary metadata for the chunk. + #speakWordsArmed = false; + #speakWordBaseRange: Range | null = null; + #speakWordOffsets: (TTSWordOffset | null)[] = []; + #speakWordRanges: (Range | null | undefined)[] = []; + #suppressMarkHighlight = false; + // True while the current chunk is highlighted word-by-word, with the most + // recently highlighted word range. Lets re-highlights (e.g. on page relocate) + // re-apply the word instead of redrawing the whole sentence over it. + #wordHighlightActive = false; + #lastSpeakWordRange: Range | null = null; + state: TTSState = 'stopped'; ttsLang: string = ''; ttsRate: number = 1.0; @@ -111,6 +126,11 @@ export class TTSController extends EventTarget { #getHighlighter() { return (range: Range) => { + // Suppress the sentence highlight that foliate's setMark draws when the + // active client highlights word-by-word. The flag is only set around the + // synchronous setMark call, so word draws (dispatchSpeakWord) and paused + // navigation still highlight normally. + if (this.#suppressMarkHighlight) return; const content = this.#getPrimaryContent(); if (!content) return; const { doc, index, overlayer } = content; @@ -576,12 +596,108 @@ export class TTSController extends EventTarget { } dispatchSpeakMark(mark?: TTSMark) { + this.#resetSpeakWords(); this.dispatchEvent(new CustomEvent('tts-speak-mark', { detail: mark || { text: '' } })); if (mark && mark.name !== '-1') { try { + // When the active client highlights word-by-word, suppress the + // sentence highlight that setMark would otherwise draw, so the page + // doesn't flash the whole sentence before the first word. The fallback + // (no boundaries) is drawn later in prepareSpeakWords. + this.#suppressMarkHighlight = this.ttsClient.supportsWordBoundaries(); const range = this.view.tts?.setMark(mark.name); + this.#suppressMarkHighlight = false; + this.#speakWordsArmed = !!range; const cfi = this.view.getCFI(this.#ttsSectionIndex, range); this.dispatchEvent(new CustomEvent('tts-highlight-mark', { detail: { cfi } })); + } catch { + this.#suppressMarkHighlight = false; + } + } + } + + #resetSpeakWords() { + this.#speakWordsArmed = false; + this.#speakWordBaseRange = null; + this.#speakWordOffsets = []; + this.#speakWordRanges = []; + this.#wordHighlightActive = false; + this.#lastSpeakWordRange = null; + } + + // Re-apply the active highlight after the view relocates (page turn, + // re-render). In word mode this re-draws the current word so the sentence + // never reappears over it; otherwise it re-draws the sentence. + reapplyCurrentHighlight() { + if (this.#wordHighlightActive && this.#lastSpeakWordRange) { + this.#getHighlighter()(this.#lastSpeakWordRange.cloneRange()); + return; + } + const range = this.view.tts?.getLastRange(); + if (range) this.#getHighlighter()(range.cloneRange()); + } + + // CFI of the currently highlighted word during word-by-word playback. Used + // for the "in view" check that drives the back-to-TTS button: when a sentence + // spans a page break, the word can be on a different page than the sentence's + // ttsLocation, so the word position is the accurate reference. Returns null + // outside word mode, where the sentence-level ttsLocation is correct. + getCurrentHighlightCfi(): string | null { + if (!this.#wordHighlightActive || !this.#lastSpeakWordRange || this.#ttsSectionIndex < 0) { + return null; + } + try { + return this.view.getCFI(this.#ttsSectionIndex, this.#lastSpeakWordRange) || null; + } catch { + return null; + } + } + + // Word-level highlighting within the chunk of the last dispatched mark, + // driven by TTS clients that report word boundaries (Edge TTS). It only + // swaps the visual highlight from the sentence to the spoken word — + // ttsLocation, media-session metadata and mark navigation keep their + // sentence-level semantics. + prepareSpeakWords(words: string[]) { + if (!this.#speakWordsArmed) return; + const range = this.view.tts?.getLastRange(); + if (!range) return; + this.#speakWordBaseRange = range; + this.#speakWordOffsets = computeWordOffsets(range.toString(), words); + this.#speakWordRanges = []; + if (words.length === 0) { + // No word boundaries for this chunk: the sentence highlight was + // suppressed at mark dispatch, so draw it now as the fallback. + this.#wordHighlightActive = false; + this.#getHighlighter()(range.cloneRange()); + } else { + // Highlight the first word immediately so the suppressed sentence + // highlight never appears before playback reaches the first boundary. + this.#wordHighlightActive = true; + this.dispatchSpeakWord(0); + } + } + + dispatchSpeakWord(index: number) { + const base = this.#speakWordBaseRange; + if (!base) return; + let range = this.#speakWordRanges[index]; + if (range === undefined) { + const offset = this.#speakWordOffsets[index]; + range = offset ? getTextSubRange(base, offset.start, offset.end) : null; + this.#speakWordRanges[index] = range; + } + if (range) { + this.#lastSpeakWordRange = range; + this.#getHighlighter()(range.cloneRange()); + // Let the view follow the spoken word so it turns the page mid-sentence + // when the word crosses a page boundary, instead of waiting for the next + // sentence's mark. + try { + const cfi = this.view.getCFI(this.#ttsSectionIndex, range); + if (cfi) { + this.dispatchEvent(new CustomEvent('tts-highlight-word', { detail: { cfi } })); + } } catch {} } } diff --git a/apps/readest-app/src/services/tts/WebSpeechClient.ts b/apps/readest-app/src/services/tts/WebSpeechClient.ts index 55e1278a..3a575906 100644 --- a/apps/readest-app/src/services/tts/WebSpeechClient.ts +++ b/apps/readest-app/src/services/tts/WebSpeechClient.ts @@ -266,6 +266,10 @@ export class WebSpeechClient implements TTSClient { return [voicesGroup]; } + supportsWordBoundaries(): boolean { + return false; + } + getGranularities(): TTSGranularity[] { // currently only support sentence boundary and disable word boundary as changing voice // in the middle of speech is not possible for different granularities diff --git a/apps/readest-app/src/services/tts/wordHighlight.ts b/apps/readest-app/src/services/tts/wordHighlight.ts new file mode 100644 index 00000000..744395d8 --- /dev/null +++ b/apps/readest-app/src/services/tts/wordHighlight.ts @@ -0,0 +1,95 @@ +// Helpers for word-level TTS highlighting driven by Edge TTS word-boundary +// metadata. Boundary words are matched sequentially against the text of the +// currently spoken sentence range; an unmatched word (e.g. rewritten by a +// TTS-only proofread rule) is skipped without advancing the search cursor so +// later words still align. + +export interface TTSWordOffset { + start: number; + end: number; +} + +// Edge TTS word-boundary offsets are in 100-nanosecond ticks. +const TICKS_PER_SECOND = 10_000_000; + +export const computeWordOffsets = (text: string, words: string[]): (TTSWordOffset | null)[] => { + const offsets: (TTSWordOffset | null)[] = []; + let cursor = 0; + for (const word of words) { + const trimmed = word.trim(); + if (!trimmed) { + offsets.push(null); + continue; + } + const index = text.indexOf(trimmed, cursor); + if (index === -1) { + offsets.push(null); + continue; + } + offsets.push({ start: index, end: index + trimmed.length }); + cursor = index + trimmed.length; + } + return offsets; +}; + +export const findBoundaryIndexAtTime = ( + boundaries: { offset: number }[], + seconds: number, +): number => { + const ticks = seconds * TICKS_PER_SECOND; + let low = 0; + let high = boundaries.length - 1; + let result = -1; + while (low <= high) { + const mid = (low + high) >> 1; + if (boundaries[mid]!.offset <= ticks) { + result = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + return result; +}; + +// Extract the sub-range covering [start, end) of base.toString(). Walks the +// text nodes intersecting the base range, slicing partially-contained +// boundary nodes the same way Range stringification does. +export const getTextSubRange = (base: Range, start: number, end: number): Range | null => { + if (start < 0 || end <= start) return null; + const root = base.commonAncestorContainer; + const doc = root.ownerDocument ?? (root as Document); + // A TreeWalker never yields its root, which is the node itself when the + // base range lies within a single text node. + const textNodes = function* () { + if (root.nodeType === Node.TEXT_NODE) { + yield root; + return; + } + const walker = doc.createTreeWalker(root, NodeFilter.SHOW_TEXT); + for (let node = walker.nextNode(); node; node = walker.nextNode()) yield node; + }; + let pos = 0; + let startNode: Text | null = null; + let startOffset = 0; + for (const node of textNodes()) { + if (!base.intersectsNode(node)) continue; + const text = node as Text; + const from = node === base.startContainer ? base.startOffset : 0; + const to = node === base.endContainer ? base.endOffset : text.data.length; + if (to <= from) continue; + const len = to - from; + if (!startNode && pos + len > start) { + startNode = text; + startOffset = from + (start - pos); + } + if (startNode && pos + len >= end) { + const range = doc.createRange(); + range.setStart(startNode, startOffset); + range.setEnd(text, from + (end - pos)); + return range; + } + pos += len; + } + return null; +};