diff --git a/apps/readest-app/TODOS.md b/apps/readest-app/TODOS.md new file mode 100644 index 00000000..2852bfa3 --- /dev/null +++ b/apps/readest-app/TODOS.md @@ -0,0 +1,31 @@ +# TODOS + +Deferred items from the Edge TTS Web Audio plan review (/autoplan, 2026-07-04). +Each was explicitly deferred, not forgotten — see the Decision Audit Trail in +`.agents/plans/2026-07-03-edge-tts-webaudio.md`. + +## TTS listening engine follow-ups + +- [ ] Cross-section gapless playback: preload and schedule the next section's first + sentence so chapter boundaries are as seamless as sentence boundaries. (M) +- [ ] Buffer-ahead indicator in the TTS scrubber (show the prefetched region, + YouTube-style). (S) +- [ ] Lock-screen ±10s seek offsets in addition to prev/next sentence. (S) +- [ ] Persist measured sentence durations per book so a reopened chapter starts + with an exact timeline instead of estimates. (S) +- [ ] Worker offload for decode + WSOLA if device profiling shows main-thread jank + on low-end Android. (S) +- [ ] Sticky TTSBar scrubber (panel + lock screen only in the first release; + recorded as deliberate in decision #24). (S) +- [ ] Provider-agnostic voice source hedge: local neural TTS (e.g. Piper/Kokoro + WASM) plugging into WebAudioPlayer/SectionTimeline — the engine is designed + for this; see "Strategic framing" in the plan. (L) +- [ ] Background chapter prefetch (convert timeline estimates to exact durations + ahead of playback). (M) +- [ ] Background TTS: decouple session ownership from the reader view via an + app-level TTSSessionManager so closing the book keeps TTS playing + (headless text supply via section.createDocument(), CFI re-anchoring for + highlights on reattach, library now-playing pill). Decided matrix: close + book = keep playing; reopen same book = seamless reattach (adopt session, + redispatchPosition, lazy doc swap at next section); open a DIFFERENT + book = TTS stops; explicit stop / sleep timer = stops. (M) diff --git a/apps/readest-app/src/__tests__/hooks/useTTSControl.test.tsx b/apps/readest-app/src/__tests__/hooks/useTTSControl.test.tsx index f9e02c7c..5b4bc972 100644 --- a/apps/readest-app/src/__tests__/hooks/useTTSControl.test.tsx +++ b/apps/readest-app/src/__tests__/hooks/useTTSControl.test.tsx @@ -148,6 +148,9 @@ vi.mock('@/services/tts', () => ({ getVoices: vi.fn().mockResolvedValue([]), getVoiceId: vi.fn().mockReturnValue(''), redispatchPosition: vi.fn(), + ensureTimeline: vi.fn().mockResolvedValue(null), + getPlaybackInfo: vi.fn().mockReturnValue(null), + seekToTime: vi.fn().mockResolvedValue(undefined), state: 'idle', addEventListener: vi.fn(), removeEventListener: vi.fn(), @@ -155,12 +158,17 @@ vi.mock('@/services/tts', () => ({ }); ttsControllerInstances.push(this); }), + ensureSharedAudioContext: vi.fn().mockResolvedValue(undefined), })); vi.mock('@/libs/mediaSession', () => ({ TauriMediaSession: class {}, })); +const { mockMediaSessionRef } = vi.hoisted(() => ({ + mockMediaSessionRef: { current: null as unknown }, +})); + vi.mock('@/utils/ssml', () => ({ genSSMLRaw: vi.fn((s: string) => `${s}`), parseSSMLLang: vi.fn(() => 'en'), @@ -205,7 +213,7 @@ const { mockDeinitMediaSession } = vi.hoisted(() => ({ vi.mock('@/app/reader/hooks/useTTSMediaSession', () => ({ useTTSMediaSession: () => ({ - mediaSessionRef: { current: null }, + mediaSessionRef: mockMediaSessionRef, unblockAudio: vi.fn(), releaseUnblockAudio: vi.fn(), initMediaSession: vi.fn().mockResolvedValue(undefined), @@ -479,3 +487,103 @@ describe('useTTSControl handleHighlightMark cross-section navigation', () => { expect(mockView.goTo).not.toHaveBeenCalled(); }); }); + +describe('useTTSControl media-session position and seek', () => { + interface FakeWebMediaSession { + setActionHandler: ReturnType; + setPositionState: ReturnType; + handlers: Map void>; + metadata: unknown; + playbackState: string; + } + + const makeFakeMediaSession = (): FakeWebMediaSession => { + const handlers = new Map void>(); + return { + handlers, + metadata: null, + playbackState: 'none', + setActionHandler: vi.fn((action: string, cb: (d: MediaSessionActionDetails) => void) => { + handlers.set(action, cb); + }), + setPositionState: vi.fn(), + }; + }; + + type ControllerMock = { + ensureTimeline: ReturnType; + getPlaybackInfo: ReturnType; + seekToTime: ReturnType; + addEventListener: ReturnType; + }; + + beforeEach(() => { + ttsControllerInstances.length = 0; + pendingInitResolvers.length = 0; + mockMediaSessionRef.current = makeFakeMediaSession(); + }); + + afterEach(() => { + mockMediaSessionRef.current = null; + cleanup(); + }); + + const startTTS = async () => { + render(); + await act(async () => { + const p = eventDispatcher.dispatch('tts-speak', { bookKey: 'book-1' }); + for (let i = 0; i < 10; i++) await Promise.resolve(); + while (pendingInitResolvers.length > 0) pendingInitResolvers.shift()!(); + await p; + }); + return ttsControllerInstances[0] as ControllerMock; + }; + + it('registers a seekto handler that seeks the controller in seconds', async () => { + const controller = await startTTS(); + const fake = mockMediaSessionRef.current as FakeWebMediaSession; + expect(fake.handlers.has('seekto')).toBe(true); + await act(async () => { + fake.handlers.get('seekto')!({ seekTime: 42 } as MediaSessionActionDetails); + for (let i = 0; i < 5; i++) await Promise.resolve(); + }); + expect(controller.seekToTime).toHaveBeenCalledWith(42); + }); + + it('pushes clamped position state on tts-speak-mark', async () => { + const controller = await startTTS(); + controller.getPlaybackInfo.mockReturnValue({ + position: 20, + duration: 10, + measuredFraction: 1, + }); + const markListener = controller.addEventListener.mock.calls.find( + ([type]) => type === 'tts-speak-mark', + )?.[1] as (e: Event) => void; + expect(markListener).toBeDefined(); + await act(async () => { + markListener(new CustomEvent('tts-speak-mark', { detail: { text: 'hi', name: '0' } })); + for (let i = 0; i < 5; i++) await Promise.resolve(); + }); + const fake = mockMediaSessionRef.current as FakeWebMediaSession; + expect(fake.setPositionState).toHaveBeenCalledWith({ + duration: 10, + position: 10, // clamped to duration, never skipped + playbackRate: 1, + }); + }); + + it('does not push position state while the timeline is unavailable', async () => { + const controller = await startTTS(); + controller.getPlaybackInfo.mockReturnValue(null); + const markListener = controller.addEventListener.mock.calls.find( + ([type]) => type === 'tts-speak-mark', + )?.[1] as (e: Event) => void; + await act(async () => { + markListener(new CustomEvent('tts-speak-mark', { detail: { text: 'hi', name: '0' } })); + for (let i = 0; i < 5; i++) await Promise.resolve(); + }); + const fake = mockMediaSessionRef.current as FakeWebMediaSession; + expect(fake.setPositionState).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/edgeTTS-audio-data.test.ts b/apps/readest-app/src/__tests__/libs/edgeTTS-audio-data.test.ts new file mode 100644 index 00000000..f181d27e --- /dev/null +++ b/apps/readest-app/src/__tests__/libs/edgeTTS-audio-data.test.ts @@ -0,0 +1,112 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const fetchWithAuthMock = vi.fn(); + +vi.mock('@/utils/fetch', () => ({ + fetchWithAuth: (...args: unknown[]) => fetchWithAuthMock(...args), +})); + +vi.mock('@/services/environment', () => ({ + getAPIBaseUrl: () => 'http://api.test', + isTauriAppPlatform: () => false, +})); + +import { + EdgeSpeechTTS, + type EdgeTTSPayload, + hashTTSPayload, + serializeWordBoundaries, + WORD_BOUNDARIES_HEADER, +} from '@/libs/edgeTTS'; + +const makePayload = (text: string): EdgeTTSPayload => ({ + lang: 'en', + text, + voice: 'en-US-AriaNeural', + rate: 1.0, + pitch: 1.0, +}); + +const makeResponse = () => + new Response(new Uint8Array([1, 2, 3, 4]).buffer, { + status: 200, + headers: { + [WORD_BOUNDARIES_HEADER]: serializeWordBoundaries([ + { offset: 1_000_000, duration: 4_000_000, text: 'hello' }, + ]), + }, + }); + +describe('hashTTSPayload', () => { + test('is stable for equal payload content', () => { + expect(hashTTSPayload(makePayload('abc'))).toBe(hashTTSPayload(makePayload('abc'))); + }); + + test('differs when payload fields differ', () => { + expect(hashTTSPayload(makePayload('abc'))).not.toBe(hashTTSPayload(makePayload('abd'))); + expect(hashTTSPayload({ ...makePayload('abc'), pitch: 1.2 })).not.toBe( + hashTTSPayload(makePayload('abc')), + ); + }); +}); + +describe('createAudioData', () => { + beforeEach(() => { + fetchWithAuthMock.mockReset(); + fetchWithAuthMock.mockImplementation(async () => makeResponse()); + }); + + test('returns audio bytes and boundaries from the network on first call', async () => { + const tts = new EdgeSpeechTTS('https'); + const { data, boundaries } = await tts.createAudioData(makePayload('first call text')); + expect(new Uint8Array(data)).toEqual(new Uint8Array([1, 2, 3, 4])); + expect(boundaries).toHaveLength(1); + expect(boundaries[0]!.text).toBe('hello'); + expect(fetchWithAuthMock).toHaveBeenCalledTimes(1); + }); + + test('serves the second call from cache with a fresh, non-detached buffer', async () => { + const tts = new EdgeSpeechTTS('https'); + const payload = makePayload('cache hit text'); + const first = await tts.createAudioData(payload); + const second = await tts.createAudioData(payload); + expect(fetchWithAuthMock).toHaveBeenCalledTimes(1); + // WebKit's decodeAudioData detaches its input; every call must get its + // own copy so replay from cache cannot hand out a detached buffer. + expect(second.data).not.toBe(first.data); + expect(first.data.byteLength).toBe(4); + expect(second.data.byteLength).toBe(4); + expect(second.boundaries).toHaveLength(1); + }); + + test('deduplicates concurrent in-flight fetches for the same payload', async () => { + let release: (() => void) | undefined; + fetchWithAuthMock.mockImplementation( + () => + new Promise((resolve) => { + release = () => resolve(makeResponse()); + }), + ); + const tts = new EdgeSpeechTTS('https'); + const payload = makePayload('concurrent text'); + const p1 = tts.createAudioData(payload); + const p2 = tts.createAudioData(payload); + await Promise.resolve(); + release!(); + const [r1, r2] = await Promise.all([p1, p2]); + expect(fetchWithAuthMock).toHaveBeenCalledTimes(1); + expect(new Uint8Array(r1.data)).toEqual(new Uint8Array(r2.data)); + }); + + test('a failed fetch is not cached; the next call retries', async () => { + fetchWithAuthMock.mockImplementationOnce(async () => { + throw new Error('network down'); + }); + const tts = new EdgeSpeechTTS('https'); + const payload = makePayload('retry after failure'); + await expect(tts.createAudioData(payload)).rejects.toThrow('network down'); + const { data } = await tts.createAudioData(payload); + expect(data.byteLength).toBe(4); + expect(fetchWithAuthMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/apps/readest-app/src/__tests__/libs/edgeTTS-word-boundaries.test.ts b/apps/readest-app/src/__tests__/libs/edgeTTS-word-boundaries.test.ts index 779bc949..6ac5f548 100644 --- a/apps/readest-app/src/__tests__/libs/edgeTTS-word-boundaries.test.ts +++ b/apps/readest-app/src/__tests__/libs/edgeTTS-word-boundaries.test.ts @@ -78,7 +78,7 @@ const makeMetadataFrame = (text: string, offset: number, duration: number) => ], }); -describe('EdgeSpeechTTS.createAudio word boundaries (browser WebSocket path)', () => { +describe('EdgeSpeechTTS.createAudioData word boundaries (browser WebSocket path)', () => { beforeEach(() => { wsState.instances.length = 0; (URL as unknown as { createObjectURL?: (blob: Blob) => string }).createObjectURL = vi.fn( @@ -97,7 +97,7 @@ describe('EdgeSpeechTTS.createAudio word boundaries (browser WebSocket path)', ( pitch: 1.0, }; - const promise = tts.createAudio(payload); + const promise = tts.createAudioData(payload); await vi.waitFor(() => expect(wsState.instances.length).toBe(1)); const ws = wsState.instances[0]!; // In a browser (jsdom has `window`), the WebSocket constructor must be @@ -110,8 +110,8 @@ describe('EdgeSpeechTTS.createAudio word boundaries (browser WebSocket path)', ( ws.emit('message', { data: makeMetadataFrame('brave', 6000000, 4000000) }); ws.emit('message', { data: 'Path:turn.end\r\n\r\n' }); - const { url, boundaries } = await promise; - expect(url).toBe('blob:mock-object-url'); + const { data, boundaries } = await promise; + expect(new Uint8Array(data)).toEqual(new Uint8Array([1, 2, 3, 4])); expect(boundaries).toEqual([ { offset: 1000000, duration: 4000000, text: 'Hello' }, { offset: 6000000, duration: 4000000, text: 'brave' }, @@ -119,9 +119,9 @@ describe('EdgeSpeechTTS.createAudio word boundaries (browser WebSocket path)', ( // A second call for the same payload is served from the cache: no new // WebSocket connection, same boundaries. - const cached = await tts.createAudio(payload); + const cached = await tts.createAudioData(payload); expect(wsState.instances.length).toBe(1); - expect(cached.url).toBe('blob:mock-object-url'); + expect(new Uint8Array(cached.data)).toEqual(new Uint8Array([1, 2, 3, 4])); expect(cached.boundaries).toEqual(boundaries); }); @@ -129,7 +129,7 @@ describe('EdgeSpeechTTS.createAudio word boundaries (browser WebSocket path)', ( const { EdgeSpeechTTS } = await import('@/libs/edgeTTS'); const tts = new EdgeSpeechTTS('wss'); - const promise = tts.createAudio({ + const promise = tts.createAudioData({ lang: 'en', text: 'No metadata here', voice: 'en-US-AriaNeural', @@ -171,7 +171,7 @@ describe('word-boundary header (de)serialization', () => { }); }); -describe('EdgeSpeechTTS.createAudio over the HTTPS proxy (word boundaries via header)', () => { +describe('EdgeSpeechTTS.createAudioData over the HTTPS proxy (word boundaries via header)', () => { beforeEach(() => { httpState.headers = {}; httpState.body = new Uint8Array([1, 2, 3]); @@ -191,7 +191,7 @@ describe('EdgeSpeechTTS.createAudio over the HTTPS proxy (word boundaries via he httpState.headers = { [WORD_BOUNDARIES_HEADER]: serializeWordBoundaries(boundaries) }; const tts = new EdgeSpeechTTS('https'); - const result = await tts.createAudio({ + const result = await tts.createAudioData({ lang: 'en', text: 'Hello world https', voice: 'en-US-AriaNeural', @@ -204,7 +204,7 @@ describe('EdgeSpeechTTS.createAudio over the HTTPS proxy (word boundaries via he test('returns empty boundaries when the proxy omits the header', async () => { const { EdgeSpeechTTS } = await import('@/libs/edgeTTS'); const tts = new EdgeSpeechTTS('https'); - const result = await tts.createAudio({ + const result = await tts.createAudioData({ lang: 'en', text: 'No header https', voice: 'en-US-AriaNeural', diff --git a/apps/readest-app/src/__tests__/services/edge-tts-client-playback.test.ts b/apps/readest-app/src/__tests__/services/edge-tts-client-playback.test.ts new file mode 100644 index 00000000..9a4fecd5 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/edge-tts-client-playback.test.ts @@ -0,0 +1,302 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +import type { TTSMessageEvent } from '@/services/tts/TTSClient'; +import type { TTSController } from '@/services/tts/TTSController'; +import { FakeAudioContext } from './tts-fake-audio'; + +// Mock control shared with the hoisted module mock. +type MockAudioData = { + data: ArrayBuffer; + boundaries: Array<{ offset: number; duration: number; text: string }>; +}; +let createAudioDataBehavior: (payloadText: string) => Promise; +let parsedMarks: Array<{ name: string; text: string; language: string }> = []; + +vi.mock('@/libs/edgeTTS', () => { + const voices = [{ id: 'en-US-AriaNeural', name: 'Aria', lang: 'en-US' }]; + return { + EdgeSpeechTTS: class MockEdgeSpeechTTS { + static voices = voices; + create = vi.fn().mockResolvedValue(undefined); + createAudioData = vi + .fn() + .mockImplementation((payload: { text: string }) => createAudioDataBehavior(payload.text)); + }, + EDGE_TTS_PROTOCOL: 'wss', + }; +}); + +vi.mock('@/utils/ssml', () => ({ + parseSSMLMarks: vi.fn(() => ({ marks: parsedMarks })), +})); + +vi.mock('@/utils/misc', () => ({ + getUserLocale: vi.fn((lang: string) => (lang === 'en' ? 'en-US' : lang)), +})); + +vi.mock('@/services/tts/TTSUtils', () => ({ + TTSUtils: { + getPreferredVoice: vi.fn(() => null), + sortVoicesPreferLocaleFunc: () => () => 0, + }, +})); + +const consoleSpy = { + warn: vi.spyOn(console, 'warn').mockImplementation(() => {}), + log: vi.spyOn(console, 'log').mockImplementation(() => {}), +}; +void consoleSpy; + +// One second of fake audio bytes: the fake decoder maps 1 byte -> 1 sample at +// 24kHz, and all-zero samples make findSpeechBounds keep the full range. +const audioOf = (seconds: number): MockAudioData => ({ + data: new ArrayBuffer(Math.round(seconds * 24000)), + boundaries: [], +}); + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + +let rafCallbacks: Map; +let rafId = 0; +const runRaf = () => { + const cbs = [...rafCallbacks.values()]; + rafCallbacks.clear(); + for (const cb of cbs) cb(0); +}; + +interface MockController { + dispatchSpeakMark: ReturnType; + prepareSpeakWords: ReturnType; + dispatchSpeakWord: ReturnType; +} + +type EdgeClientClass = typeof import('@/services/tts/EdgeTTSClient').EdgeTTSClient; + +describe('EdgeTTSClient Web Audio playback', () => { + let EdgeTTSClient: EdgeClientClass; + let controller: MockController; + + beforeEach(async () => { + vi.resetModules(); + FakeAudioContext.instances = []; + rafCallbacks = new Map(); + vi.stubGlobal('AudioContext', FakeAudioContext); + vi.stubGlobal('requestAnimationFrame', (cb: FrameRequestCallback) => { + rafCallbacks.set(++rafId, cb); + return rafId; + }); + vi.stubGlobal('cancelAnimationFrame', (id: number) => { + rafCallbacks.delete(id); + }); + createAudioDataBehavior = async () => audioOf(1); + parsedMarks = [ + { name: '0', text: 'First sentence.', language: 'en' }, + { name: '1', text: 'Second sentence.', language: 'en' }, + ]; + controller = { + dispatchSpeakMark: vi.fn(), + prepareSpeakWords: vi.fn(), + dispatchSpeakWord: vi.fn(), + }; + ({ EdgeTTSClient } = await import('@/services/tts/EdgeTTSClient')); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + const startClient = async () => { + const client = new EdgeTTSClient(controller as unknown as TTSController); + await client.init(); + return client; + }; + + const collectSpeak = (client: InstanceType, signal: AbortSignal) => { + const events: TTSMessageEvent[] = []; + const done = (async () => { + for await (const event of client.speak('', signal)) { + events.push(event); + } + })(); + return { events, done }; + }; + + const ctx = () => FakeAudioContext.instances[0]!; + + test('plays marks gaplessly: boundary per audible chunk, one final end', async () => { + const client = await startClient(); + const { events, done } = collectSpeak(client, new AbortController().signal); + await flush(); + await flush(); + + // Both chunks scheduled ahead, but only chunk 0 is audible: exactly one + // mark dispatched so foliate's cursor tracks the voice, not the fetcher. + expect(ctx().sources.length).toBe(2); + expect(controller.dispatchSpeakMark).toHaveBeenCalledTimes(1); + expect(events.filter((e) => e.code === 'boundary')).toHaveLength(1); + + await ctx().advanceTo(1.1); // chunk 0 ends (starts at 0.03, 1s long) + await flush(); + expect(controller.dispatchSpeakMark).toHaveBeenCalledTimes(2); + + await ctx().advanceTo(3); // chunk 1 ends + await done; + expect(events.map((e) => e.code)).toEqual(['boundary', 'boundary', 'end']); + expect(events[0]!.mark).toBe('0'); + expect(events[1]!.mark).toBe('1'); + }); + + test('chunks are scheduled with a rate-scaled gap and no element restarts', async () => { + const client = await startClient(); + await client.setRate(1); // gap = 0.15 / 1 + const { done } = collectSpeak(client, new AbortController().signal); + await flush(); + await flush(); + const [first, second] = ctx().sources; + expect(second!.startedAt! - first!.endTime).toBeCloseTo(0.15, 5); + await ctx().advanceTo(5); + await done; + }); + + test('word tracking follows the audio clock and survives pause/resume', async () => { + createAudioDataBehavior = async () => ({ + data: new ArrayBuffer(48000), // 2s + boundaries: [ + { offset: 1_000_000, duration: 4_000_000, text: 'Hello' }, // 0.1s + { offset: 6_000_000, duration: 4_000_000, text: 'brave' }, // 0.6s + { offset: 11_000_000, duration: 4_000_000, text: 'world' }, // 1.1s + ], + }); + parsedMarks = [{ name: '0', text: 'Hello brave world', language: 'en' }]; + const client = await startClient(); + const { done } = collectSpeak(client, new AbortController().signal); + await flush(); + await flush(); + + expect(controller.prepareSpeakWords).toHaveBeenCalledWith(['Hello', 'brave', 'world']); + ctx().currentTime = 0.03 + 0.15; + runRaf(); + expect(controller.dispatchSpeakWord).toHaveBeenLastCalledWith(0); + + await client.pause(); + expect(ctx().state).toBe('suspended'); + await client.resume(); + expect(ctx().state).toBe('running'); + + ctx().currentTime = 0.03 + 0.7; + runRaf(); + expect(controller.dispatchSpeakWord).toHaveBeenLastCalledWith(1); + + // Same index is not re-dispatched. + const calls = controller.dispatchSpeakWord.mock.calls.length; + runRaf(); + expect(controller.dispatchSpeakWord.mock.calls.length).toBe(calls); + + await ctx().advanceTo(5); + await done; + }); + + test('abort mid-stream yields Aborted and stops all sources', async () => { + const client = await startClient(); + const abortController = new AbortController(); + const { events, done } = collectSpeak(client, abortController.signal); + await flush(); + await flush(); + abortController.abort(); + await done; + expect(events.at(-1)).toMatchObject({ code: 'error', message: 'Aborted' }); + expect(ctx().sources.every((s) => s.stopped)).toBe(true); + }); + + test('a no-audio mark is skipped and the session continues', async () => { + createAudioDataBehavior = async (text: string) => { + if (text === 'First sentence.') throw new Error('No audio data received.'); + return audioOf(1); + }; + const client = await startClient(); + const { events, done } = collectSpeak(client, new AbortController().signal); + await flush(); + await flush(); + await ctx().advanceTo(5); + await done; + const codes = events.map((e) => e.code); + expect(codes.filter((c) => c === 'boundary')).toHaveLength(1); + expect(codes.at(-1)).toBe('end'); + }); + + test('a decode failure is treated like no-audio: warn, skip, continue', async () => { + // The context exists once the scheduler's first fetch runs (ensureContext + // precedes it), so the first fetch installs a decoder that fails exactly + // once — the first mark's decode dies, the second succeeds. + let installed = false; + createAudioDataBehavior = async () => { + if (!installed) { + installed = true; + const context = FakeAudioContext.instances[0]!; + const original = context.decodeImpl; + let failed = false; + context.decodeImpl = async (data) => { + if (!failed) { + failed = true; + throw new Error('bad mp3'); + } + return original(data); + }; + } + return audioOf(1); + }; + const client = await startClient(); + const { events, done } = collectSpeak(client, new AbortController().signal); + await flush(); + await flush(); + await ctx().advanceTo(10); + await done; + const codes = events.map((e) => e.code); + expect(codes.filter((c) => c === 'boundary')).toHaveLength(1); // only mark 1 played + expect(codes.at(-1)).toBe('end'); + }); + + test('all marks failing still ends the session with end (no wedge)', async () => { + createAudioDataBehavior = async () => { + throw new Error('No audio data received.'); + }; + const client = await startClient(); + const { events, done } = collectSpeak(client, new AbortController().signal); + await done; // zero chunks scheduled; session-end fires synchronously + const codes = events.map((e) => e.code); + expect(codes.at(-1)).toBe('end'); + expect(codes).not.toContain('boundary'); + }); + + test('a hard fetch error yields error and terminates', async () => { + createAudioDataBehavior = async () => { + throw new Error('network exploded'); + }; + const client = await startClient(); + const { events, done } = collectSpeak(client, new AbortController().signal); + await done; + expect(events.at(-1)).toMatchObject({ code: 'error', message: 'network exploded' }); + }, 10000); + + test('pause without a session is a no-op returning true', async () => { + const client = await startClient(); + expect(await client.pause()).toBe(true); + expect(await client.resume()).toBe(true); + }); + + test('getChunkPosition reports trim-relative clamped seconds', async () => { + const client = await startClient(); + parsedMarks = [{ name: '0', text: 'Only sentence.', language: 'en' }]; + const { done } = collectSpeak(client, new AbortController().signal); + await flush(); + await flush(); + ctx().currentTime = 0.03 + 0.4; + const pos = client.getChunkPosition(); + expect(pos).not.toBeNull(); + expect(pos!).toBeGreaterThan(0.3); + expect(pos!).toBeLessThanOrEqual(1); + await ctx().advanceTo(5); + await done; + expect(client.getChunkPosition()).toBeNull(); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts b/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts index 6a86f559..ce567a94 100644 --- a/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts +++ b/apps/readest-app/src/__tests__/services/edge-tts-client.test.ts @@ -3,14 +3,13 @@ import { describe, test, expect, vi, beforeEach, afterEach } from 'vitest'; // Shared mock control: tests can override createBehavior to change how create() behaves let createBehavior: () => Promise = () => Promise.resolve(undefined); -// Shared mock control for createAudioUrl() and parsed SSML marks -let createAudioUrlBehavior = vi.fn<() => Promise>(() => Promise.resolve('blob:mock-url')); -type MockAudioResult = { - url: string; +// Shared mock control for createAudioData() and parsed SSML marks +type MockAudioData = { + data: ArrayBuffer; boundaries: Array<{ offset: number; duration: number; text: string }>; }; -let createAudioBehavior = vi.fn<() => Promise>(() => - Promise.resolve({ url: 'blob:mock-url', boundaries: [] }), +let createAudioDataBehavior = vi.fn<() => Promise>(() => + Promise.resolve({ data: new ArrayBuffer(8), boundaries: [] }), ); let parsedMarks: Array<{ name: string; text: string; language: string }> = []; @@ -27,8 +26,7 @@ vi.mock('@/libs/edgeTTS', () => { EdgeSpeechTTS: class MockEdgeSpeechTTS { static voices = voices; create = vi.fn().mockImplementation(() => createBehavior()); - createAudioUrl = vi.fn().mockImplementation(() => createAudioUrlBehavior()); - createAudio = vi.fn().mockImplementation(() => createAudioBehavior()); + createAudioData = vi.fn().mockImplementation(() => createAudioDataBehavior()); }, EDGE_TTS_PROTOCOL: 'wss', }; @@ -70,9 +68,8 @@ describe('EdgeTTSClient', () => { beforeEach(() => { createBehavior = () => Promise.resolve(undefined); - createAudioUrlBehavior = vi.fn<() => Promise>(() => Promise.resolve('blob:mock-url')); - createAudioBehavior = vi.fn<() => Promise>(() => - Promise.resolve({ url: 'blob:mock-url', boundaries: [] }), + createAudioDataBehavior = vi.fn<() => Promise>(() => + Promise.resolve({ data: new ArrayBuffer(8), boundaries: [] }), ); parsedMarks = []; client = new EdgeTTSClient(); @@ -431,14 +428,14 @@ describe('EdgeTTSClient', () => { } }; - test('retries createAudioUrl up to 3 times when preload fails', async () => { + test('retries createAudioData up to 3 times when preload fails', async () => { await client.init(); parsedMarks = [{ name: 'mark-0', text: 'hello', language: 'en' }]; - createAudioUrlBehavior = vi.fn(() => Promise.reject(new Error('network error'))); + createAudioDataBehavior = vi.fn(() => Promise.reject(new Error('network error'))); await consumePreload(client, new AbortController().signal); - expect(createAudioUrlBehavior).toHaveBeenCalledTimes(3); + expect(createAudioDataBehavior).toHaveBeenCalledTimes(3); }); test('does not retry when the first preload attempt succeeds', async () => { @@ -447,37 +444,47 @@ describe('EdgeTTSClient', () => { await consumePreload(client, new AbortController().signal); - expect(createAudioUrlBehavior).toHaveBeenCalledTimes(1); + expect(createAudioDataBehavior).toHaveBeenCalledTimes(1); }); test('stops retrying once an attempt succeeds', async () => { await client.init(); parsedMarks = [{ name: 'mark-0', text: 'hello', language: 'en' }]; let calls = 0; - createAudioUrlBehavior = vi.fn(() => { + createAudioDataBehavior = vi.fn(() => { calls++; return calls < 2 ? Promise.reject(new Error('network error')) - : Promise.resolve('blob:mock-url'); + : Promise.resolve({ data: new ArrayBuffer(8), boundaries: [] }); }); await consumePreload(client, new AbortController().signal); - expect(createAudioUrlBehavior).toHaveBeenCalledTimes(2); + expect(createAudioDataBehavior).toHaveBeenCalledTimes(2); + }); + + test('does not retry a permanent no-audio failure', async () => { + await client.init(); + parsedMarks = [{ name: 'mark-0', text: 'hello', language: 'en' }]; + createAudioDataBehavior = vi.fn(() => Promise.reject(new Error('No audio data received.'))); + + await consumePreload(client, new AbortController().signal); + + expect(createAudioDataBehavior).toHaveBeenCalledTimes(1); }); test('stops retrying once the signal is aborted', async () => { await client.init(); parsedMarks = [{ name: 'mark-0', text: 'hello', language: 'en' }]; const controller = new AbortController(); - createAudioUrlBehavior = vi.fn(() => { + createAudioDataBehavior = vi.fn(() => { controller.abort(); return Promise.reject(new Error('network error')); }); await consumePreload(client, controller.signal); - expect(createAudioUrlBehavior).toHaveBeenCalledTimes(1); + expect(createAudioDataBehavior).toHaveBeenCalledTimes(1); }); }); @@ -496,137 +503,4 @@ describe('EdgeTTSClient', () => { await expect(client.stop()).resolves.toBeUndefined(); }); }); - - describe('word boundary tracking during playback', () => { - class MockAudio { - static instances: MockAudio[] = []; - src = ''; - currentTime = 0; - preload = ''; - playbackRate = 1; - onended: ((e?: Event) => void) | null = null; - onerror: ((e?: unknown) => void) | null = null; - constructor() { - MockAudio.instances.push(this); - } - setAttribute() {} - play() { - return Promise.resolve(); - } - pause() {} - } - - let rafCallbacks: Map; - let rafId = 0; - const runRaf = () => { - const cbs = [...rafCallbacks.values()]; - rafCallbacks.clear(); - for (const cb of cbs) cb(0); - }; - const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); - - let mockController: { - dispatchSpeakMark: ReturnType; - prepareSpeakWords: ReturnType; - dispatchSpeakWord: ReturnType; - }; - - beforeEach(() => { - MockAudio.instances = []; - rafCallbacks = new Map(); - vi.stubGlobal('Audio', MockAudio); - vi.stubGlobal('requestAnimationFrame', (cb: FrameRequestCallback) => { - rafCallbacks.set(++rafId, cb); - return rafId; - }); - vi.stubGlobal('cancelAnimationFrame', (id: number) => { - rafCallbacks.delete(id); - }); - mockController = { - dispatchSpeakMark: vi.fn(), - prepareSpeakWords: vi.fn(), - dispatchSpeakWord: vi.fn(), - }; - client = new EdgeTTSClient(mockController as unknown as TTSController); - parsedMarks = [{ name: '0', text: 'Hello brave world', language: 'en' }]; - createAudioBehavior = vi.fn(() => - Promise.resolve({ - url: 'blob:mock-url', - boundaries: [ - { offset: 1_000_000, duration: 4_000_000, text: 'Hello' }, - { offset: 6_000_000, duration: 4_000_000, text: 'brave' }, - { offset: 11_000_000, duration: 4_000_000, text: 'world' }, - ], - }), - ); - }); - - afterEach(() => { - vi.unstubAllGlobals(); - }); - - const startSpeak = async () => { - await client.init(); - const it = client.speak('', new AbortController().signal); - const first = await it.next(); - expect((first.value as { code: string }).code).toBe('boundary'); - const resultPromise = it.next(); - await flush(); - const audio = MockAudio.instances.at(-1)!; - return { it, resultPromise, audio }; - }; - - test('prepares speak words and dispatches word indexes as playback advances', async () => { - const { resultPromise, audio } = await startSpeak(); - - expect(mockController.prepareSpeakWords).toHaveBeenCalledWith(['Hello', 'brave', 'world']); - - audio.currentTime = 0.11; - runRaf(); - expect(mockController.dispatchSpeakWord).toHaveBeenCalledWith(0); - - audio.currentTime = 0.65; - runRaf(); - expect(mockController.dispatchSpeakWord).toHaveBeenLastCalledWith(1); - - // Same word index is not re-dispatched on subsequent frames. - const callCount = mockController.dispatchSpeakWord.mock.calls.length; - runRaf(); - expect(mockController.dispatchSpeakWord.mock.calls.length).toBe(callCount); - - audio.onended?.(); - const result = await resultPromise; - expect((result.value as { code: string }).code).toBe('end'); - }); - - test('stops dispatching after the chunk ends', async () => { - const { resultPromise, audio } = await startSpeak(); - - audio.currentTime = 0.11; - runRaf(); - const callCount = mockController.dispatchSpeakWord.mock.calls.length; - - audio.onended?.(); - await resultPromise; - - audio.currentTime = 1.2; - runRaf(); - expect(mockController.dispatchSpeakWord.mock.calls.length).toBe(callCount); - }); - - test('hands empty words to the controller and does not track when no boundaries', async () => { - createAudioBehavior = vi.fn(() => Promise.resolve({ url: 'blob:mock-url', boundaries: [] })); - const { resultPromise, audio } = await startSpeak(); - - // Empty words are still forwarded so the controller can draw the - // sentence-highlight fallback; no per-word tracking is started. - expect(mockController.prepareSpeakWords).toHaveBeenCalledWith([]); - audio.currentTime = 0.5; - runRaf(); - expect(mockController.dispatchSpeakWord).not.toHaveBeenCalled(); - - audio.onended?.(); - await resultPromise; - }); - }); }); diff --git a/apps/readest-app/src/__tests__/services/tts-controller-timeline.test.ts b/apps/readest-app/src/__tests__/services/tts-controller-timeline.test.ts new file mode 100644 index 00000000..5867d67d --- /dev/null +++ b/apps/readest-app/src/__tests__/services/tts-controller-timeline.test.ts @@ -0,0 +1,232 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { TTSController } from '@/services/tts/TTSController'; +import { TTSClient, TTSMessageEvent } from '@/services/tts/TTSClient'; +import { recordMeasuredDuration } from '@/services/tts/ttsDuration'; +import { FoliateView } from '@/types/view'; + +// --- Heavy clients replaced with light fakes (same pattern as the main +// controller suite); foliate tts.js provides a REAL-shaped getSentences fake +// that yields ranges from a jsdom document. + +const makeMockClient = (name: string): TTSClient => ({ + name, + initialized: true, + init: vi.fn().mockResolvedValue(true), + shutdown: vi.fn().mockResolvedValue(undefined), + speak: vi.fn().mockImplementation(async function* (): AsyncIterable { + yield { code: 'end', message: 'done' }; + }), + pause: vi.fn().mockResolvedValue(true), + resume: vi.fn().mockResolvedValue(true), + stop: vi.fn().mockResolvedValue(undefined), + setPrimaryLang: vi.fn(), + setRate: vi.fn().mockResolvedValue(undefined), + setPitch: vi.fn().mockResolvedValue(undefined), + setVoice: vi.fn().mockResolvedValue(undefined), + getAllVoices: vi.fn().mockResolvedValue([]), + getVoices: vi.fn().mockResolvedValue([]), + getGranularities: vi.fn().mockReturnValue(['sentence']), + supportsWordBoundaries: vi.fn().mockReturnValue(true), + getVoiceId: vi.fn().mockReturnValue('timeline-ctrl-voice'), + getSpeakingLang: vi.fn().mockReturnValue('en'), + getChunkPosition: vi.fn().mockReturnValue(0.5), +}); + +vi.mock('@/services/tts/WebSpeechClient', () => ({ + WebSpeechClient: vi.fn().mockImplementation(function (this: Record) { + Object.assign(this, makeMockClient('web-speech')); + }), +})); + +vi.mock('@/services/tts/EdgeTTSClient', () => ({ + EdgeTTSClient: vi.fn().mockImplementation(function (this: Record) { + Object.assign(this, makeMockClient('edge-tts')); + }), +})); + +vi.mock('@/services/tts/NativeTTSClient', () => ({ + NativeTTSClient: vi.fn().mockImplementation(function (this: Record) { + Object.assign(this, makeMockClient('native')); + }), +})); + +vi.mock('@/services/tts/TTSUtils', () => ({ + TTSUtils: { + getPreferredClient: vi.fn().mockReturnValue('edge-tts'), + setPreferredClient: vi.fn(), + setPreferredVoice: vi.fn(), + getPreferredVoice: vi.fn().mockReturnValue(null), + }, +})); + +vi.mock('foliate-js/overlayer.js', () => ({ + Overlayer: { highlight: 'highlightFn' }, +})); + +// Sentences of the fake section; ends avoid foliate's short-word +// abbreviation merge so counts stay predictable. +const S0 = 'The opening sentence of the chapter reads aloud smoothly.'; +const S1 = 'A following sentence continues the passage without pause.'; +const S2 = 'The final sentence wraps the paragraph completely together.'; + +let sectionDoc: Document; +let sentenceRanges: Range[] = []; + +const buildSectionDoc = () => { + const parser = new DOMParser(); + sectionDoc = parser.parseFromString( + `

${S0} ${S1} ${S2}

`, + 'text/html', + ); + const textNode = sectionDoc.getElementById('p')!.firstChild!; + const full = textNode.textContent!; + sentenceRanges = [S0, S1, S2].map((s) => { + const start = full.indexOf(s); + const range = sectionDoc.createRange(); + range.setStart(textNode, start); + range.setEnd(textNode, start + s.length); + return range; + }); +}; + +vi.mock('foliate-js/tts.js', () => ({ + TTS: vi.fn().mockImplementation(function (this: Record) { + Object.assign(this, { + start: vi.fn().mockReturnValue('hello'), + resume: vi.fn().mockReturnValue('hello'), + // End-of-section after one paragraph so auto-advance terminates instead + // of looping the controller forever in tests. + next: vi.fn().mockReturnValue(undefined), + prev: vi.fn().mockReturnValue(undefined), + from: vi.fn().mockReturnValue('from'), + setMark: vi.fn().mockImplementation(() => sentenceRanges[0]!.cloneRange()), + getLastRange: vi.fn().mockImplementation(() => sentenceRanges[0]!.cloneRange()), + doc: null, + }); + }), + getSentences: vi.fn().mockImplementation(function* () { + for (let i = 0; i < sentenceRanges.length; i++) { + yield { blockIndex: 0, markName: String(i), range: sentenceRanges[i]! }; + } + }), +})); + +vi.mock('foliate-js/text-walker.js', () => ({ textWalker: vi.fn() })); + +vi.mock('@/utils/ssml', () => ({ + filterSSMLWithLang: vi.fn((ssml: string) => ssml), + parseSSMLMarks: vi.fn(() => ({ + plainText: 'hello', + marks: [{ offset: 0, name: '0', text: 'hello', language: 'en' }], + })), +})); + +vi.mock('@/utils/node', () => ({ createRejectFilter: vi.fn(() => () => 1) })); +vi.mock('@/utils/lang', () => ({ + isValidLang: vi.fn(() => true), + isCJKLang: vi.fn(() => false), +})); + +const makeView = () => { + const contents = { + doc: sectionDoc, + index: 0, + overlayer: { add: vi.fn(), remove: vi.fn() }, + }; + return { + book: { sections: [{ createDocument: vi.fn().mockResolvedValue(sectionDoc) }] }, + renderer: { getContents: () => [contents], primaryIndex: 0 }, + language: { isCJK: false }, + getCFI: vi.fn().mockReturnValue('epubcfi(/6/2!/4/2)'), + resolveCFI: vi.fn().mockReturnValue({ anchor: () => sentenceRanges[0] }), + tts: null, + } as unknown as FoliateView; +}; + +describe('TTSController section timeline', () => { + let controller: TTSController; + + beforeEach(async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + buildSectionDoc(); + controller = new TTSController(null, makeView()); + await controller.init(); + await controller.initViewTTS(0); + }); + + test('ensureTimeline builds lazily for the edge client and caches per section', async () => { + const timeline = await controller.ensureTimeline(); + expect(timeline).not.toBeNull(); + expect(timeline!.length).toBe(3); + expect(await controller.ensureTimeline()).toBe(timeline); + }); + + test('getPlaybackInfo composes sentence position with the chunk clock', async () => { + recordMeasuredDuration('timeline-ctrl-voice', S0, 4); + recordMeasuredDuration('timeline-ctrl-voice', S1, 6); + await controller.ensureTimeline(); + // getLastRange resolves to sentence 0; client chunk position is 0.5s. + const info = controller.getPlaybackInfo(); + expect(info).not.toBeNull(); + expect(info!.position).toBeCloseTo(0.5, 5); + expect(info!.duration).toBeGreaterThan(10); + expect(info!.measuredFraction).toBeGreaterThan(0); + }); + + test('getPlaybackInfo is null before the timeline is built (reserved-slot state)', () => { + expect(controller.getPlaybackInfo()).toBeNull(); + }); + + test('getPlaybackInfo is null for non-edge clients', async () => { + await controller.setVoice('', 'en'); // empty voice id: falls through to web client + controller.ttsClient = controller.ttsWebClient; + expect(await controller.ensureTimeline()).toBeNull(); + expect(controller.getPlaybackInfo()).toBeNull(); + }); + + test('seekToTime snaps to the sentence and speaks from its range while playing', async () => { + recordMeasuredDuration('timeline-ctrl-voice', S0, 4); + recordMeasuredDuration('timeline-ctrl-voice', S1, 6); + await controller.ensureTimeline(); + controller.state = 'playing'; + await controller.seekToTime(5); // inside sentence 1 + const tts = controller.view.tts as unknown as { from: ReturnType }; + expect(tts.from).toHaveBeenCalledTimes(1); + const arg = tts.from.mock.calls[0]![0] as Range; + expect(arg.toString()).toBe(S1); + }); + + test('seekToTime while paused stays paused and still navigates', async () => { + await controller.ensureTimeline(); + controller.state = 'paused'; + await controller.seekToTime(0); + expect(controller.state).toBe('forward-paused'); + const tts = controller.view.tts as unknown as { from: ReturnType }; + expect(tts.from).toHaveBeenCalled(); + }); + + test('seekToTime past the end clamps to the last sentence', async () => { + await controller.ensureTimeline(); + controller.state = 'playing'; + await controller.seekToTime(9999); + const tts = controller.view.tts as unknown as { from: ReturnType }; + const arg = tts.from.mock.calls[0]![0] as Range; + expect(arg.toString()).toBe(S2); + }); + + test('setRate rescales the timeline', async () => { + recordMeasuredDuration('timeline-ctrl-voice', S0, 4); + recordMeasuredDuration('timeline-ctrl-voice', S1, 4); + recordMeasuredDuration('timeline-ctrl-voice', S2, 4); + const timeline = await controller.ensureTimeline(); + const before = timeline!.getDuration(); + await controller.setRate(2); + expect(timeline!.getDuration()).toBeCloseTo(before / 2, 5); + }); + + test('shutdown drops the timeline', async () => { + await controller.ensureTimeline(); + await controller.shutdown(); + expect(controller.getPlaybackInfo()).toBeNull(); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/tts-duration.test.ts b/apps/readest-app/src/__tests__/services/tts-duration.test.ts new file mode 100644 index 00000000..da966632 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/tts-duration.test.ts @@ -0,0 +1,120 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; + +import { + calibrateVoiceRate, + defaultCharsPerSecond, + estimateSentenceSeconds, + getMeasuredDuration, + recordMeasuredDuration, + recordProvisionalDuration, +} from '@/services/tts/ttsDuration'; + +const VOICE = 'en-US-AriaNeural'; + +afterEach(() => { + localStorage.clear(); + vi.unstubAllGlobals(); +}); + +describe('measured duration store', () => { + test('records and retrieves a duration by voice and text', () => { + recordMeasuredDuration(VOICE, 'The quick brown fox jumps.', 2.4); + expect(getMeasuredDuration(VOICE, 'The quick brown fox jumps.')).toBe(2.4); + }); + + test('punctuation and case variants hit the same key', () => { + recordMeasuredDuration(VOICE, 'Hello—world! Again…', 1.7); + expect(getMeasuredDuration(VOICE, 'hello world again')).toBe(1.7); + expect(getMeasuredDuration(VOICE, 'Hello, world; AGAIN')).toBe(1.7); + }); + + test('different voices do not share durations', () => { + recordMeasuredDuration(VOICE, 'Shared sentence text here.', 2.0); + expect(getMeasuredDuration('fr-FR-DeniseNeural', 'Shared sentence text here.')).toBeUndefined(); + }); + + test('provisional never overwrites measured; measured overwrites provisional', () => { + recordProvisionalDuration(VOICE, 'A provisional sentence sample.', 3.0); + expect(getMeasuredDuration(VOICE, 'A provisional sentence sample.')).toBe(3.0); + recordMeasuredDuration(VOICE, 'A provisional sentence sample.', 2.5); + expect(getMeasuredDuration(VOICE, 'A provisional sentence sample.')).toBe(2.5); + recordProvisionalDuration(VOICE, 'A provisional sentence sample.', 9.9); + expect(getMeasuredDuration(VOICE, 'A provisional sentence sample.')).toBe(2.5); + }); +}); + +describe('defaultCharsPerSecond', () => { + test('CJK languages are far denser per character than Latin', () => { + expect(defaultCharsPerSecond('zh-CN')).toBeLessThan(defaultCharsPerSecond('en-US')); + expect(defaultCharsPerSecond('ja')).toBeLessThan(defaultCharsPerSecond('fr')); + expect(defaultCharsPerSecond('ko')).toBeLessThan(defaultCharsPerSecond('de-DE')); + }); +}); + +describe('estimateSentenceSeconds', () => { + test('uses the measured duration when available', () => { + recordMeasuredDuration(VOICE, 'A measured sentence for estimates.', 4.2); + expect(estimateSentenceSeconds('A measured sentence for estimates.', 'en', VOICE)).toBe(4.2); + }); + + test('falls back to the script default without calibration', () => { + const text = 'x'.repeat(150); + const est = estimateSentenceSeconds(text, 'en', 'uncalibrated-voice'); + expect(est).toBeCloseTo(150 / defaultCharsPerSecond('en'), 3); + }); + + test('prefers the calibrated per-voice rate over the default', () => { + const voice = 'calibration-voice'; + // Voice speaks 20 normalized chars per second. + calibrateVoiceRate(voice, 'a'.repeat(40), 2); + calibrateVoiceRate(voice, 'b'.repeat(60), 3); + const est = estimateSentenceSeconds('c'.repeat(100), 'en', voice); + expect(est).toBeGreaterThan(100 / 25); + expect(est).toBeLessThan(100 / 15); + }); + + test('calibration converges toward the observed rate', () => { + const voice = 'converging-voice'; + for (let i = 0; i < 20; i++) { + calibrateVoiceRate(voice, 'a'.repeat(50), 5); // 10 cps observed + } + const est = estimateSentenceSeconds('d'.repeat(100), 'en', voice); + expect(est).toBeGreaterThan(100 / 12); + expect(est).toBeLessThan(100 / 8); + }); + + test('short texts are skipped for calibration', () => { + const voice = 'short-skip-voice'; + calibrateVoiceRate(voice, 'hi', 60); // absurd sample must be ignored + const est = estimateSentenceSeconds('e'.repeat(150), 'en', voice); + expect(est).toBeCloseTo(150 / defaultCharsPerSecond('en'), 3); + }); + + test('empty text estimates zero', () => { + expect(estimateSentenceSeconds('', 'en', VOICE)).toBe(0); + }); +}); + +describe('persistence', () => { + test('per-voice calibration survives a module reload via localStorage', async () => { + calibrateVoiceRate('persisted-voice', 'a'.repeat(50), 5); + vi.resetModules(); + const fresh = await import('@/services/tts/ttsDuration'); + const est = fresh.estimateSentenceSeconds('f'.repeat(100), 'en', 'persisted-voice'); + expect(est).toBeGreaterThan(100 / 16); + }); + + test('does not throw when localStorage is unavailable', () => { + vi.stubGlobal('localStorage', { + getItem: () => { + throw new Error('denied'); + }, + setItem: () => { + throw new Error('denied'); + }, + clear: () => {}, + }); + expect(() => calibrateVoiceRate('blocked-voice', 'a'.repeat(50), 5)).not.toThrow(); + expect(() => estimateSentenceSeconds('g'.repeat(100), 'en', 'blocked-voice')).not.toThrow(); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/tts-fake-audio.ts b/apps/readest-app/src/__tests__/services/tts-fake-audio.ts new file mode 100644 index 00000000..e8809f56 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/tts-fake-audio.ts @@ -0,0 +1,138 @@ +// Fake Web Audio primitives for jsdom tests. The fake clock is manual: +// advanceTo(t) fires due onended callbacks in end-time order, awaiting +// microtasks between each so schedulers and waiters can react, matching how +// the real audio clock interleaves with the JS task queue. + +import type { + TTSAudioBuffer, + TTSAudioBufferSourceNode, + TTSAudioContext, +} from '@/services/tts/WebAudioPlayer'; + +export class FakeAudioBuffer implements TTSAudioBuffer { + constructor( + public samples: Float32Array, + public readonly sampleRate: number, + ) {} + get length() { + return this.samples.length; + } + get duration() { + return this.samples.length / this.sampleRate; + } + getChannelData(_channel: number): Float32Array { + return this.samples; + } + copyToChannel(source: Float32Array, _channel: number): void { + this.samples.set(source.subarray(0, this.samples.length)); + } +} + +export class FakeSourceNode implements TTSAudioBufferSourceNode { + buffer: TTSAudioBuffer | null = null; + onended: (() => void) | null = null; + startedAt: number | null = null; + stopped = false; + connected = false; + endedFired = false; + + constructor(private ctx: FakeAudioContext) {} + + connect(_destination: unknown): void { + this.connected = true; + } + disconnect(): void { + this.connected = false; + } + start(when = 0): void { + this.startedAt = Math.max(when, this.ctx.currentTime); + this.ctx.sources.push(this); + } + stop(_when?: number): void { + this.stopped = true; + this.onended?.(); + } + get endTime(): number { + return (this.startedAt ?? 0) + (this.buffer?.duration ?? 0); + } +} + +export class FakeAudioContext implements TTSAudioContext { + // Constructed instances, newest last — client tests stub the global + // AudioContext with this class and need a handle on the context the shared + // singleton created internally. + static instances: FakeAudioContext[] = []; + + currentTime = 0; + state = 'running'; + destination = {}; + onstatechange: (() => void) | null = null; + sources: FakeSourceNode[] = []; + resumeCalls = 0; + suspendCalls = 0; + closeCalls = 0; + sampleRate: number; + // Overridable resume behavior for interruption-refusal tests. + resumeImpl: (() => void) | null = null; + decodeImpl: (data: ArrayBuffer) => Promise = async (data) => + new FakeAudioBuffer(new Float32Array(data.byteLength), this.sampleRate); + + constructor(sampleRate = 24000) { + this.sampleRate = sampleRate; + FakeAudioContext.instances.push(this); + } + + async resume(): Promise { + this.resumeCalls++; + if (this.resumeImpl) { + this.resumeImpl(); + } else { + this.state = 'running'; + } + this.onstatechange?.(); + } + async suspend(): Promise { + this.suspendCalls++; + this.state = 'suspended'; + this.onstatechange?.(); + } + async close(): Promise { + this.closeCalls++; + this.state = 'closed'; + this.onstatechange?.(); + } + createBufferSource(): FakeSourceNode { + return new FakeSourceNode(this); + } + createBuffer(_channels: number, length: number, sampleRate: number): FakeAudioBuffer { + return new FakeAudioBuffer(new Float32Array(length), sampleRate); + } + decodeAudioData(data: ArrayBuffer): Promise { + return this.decodeImpl(data); + } + + setState(state: string): void { + this.state = state; + this.onstatechange?.(); + } + + async advanceTo(t: number): Promise { + for (;;) { + const due = this.sources + .filter((s) => s.startedAt !== null && !s.stopped && !s.endedFired && s.endTime <= t) + .sort((a, b) => a.endTime - b.endTime)[0]; + if (!due) break; + this.currentTime = Math.max(this.currentTime, due.endTime); + due.endedFired = true; + due.onended?.(); + await Promise.resolve(); + await Promise.resolve(); + } + this.currentTime = Math.max(this.currentTime, t); + await Promise.resolve(); + await Promise.resolve(); + } +} + +export const makeBuffer = (seconds: number, sampleRate = 24000): FakeAudioBuffer => + new FakeAudioBuffer(new Float32Array(Math.round(seconds * sampleRate)), sampleRate); diff --git a/apps/readest-app/src/__tests__/services/tts-pcm.test.ts b/apps/readest-app/src/__tests__/services/tts-pcm.test.ts new file mode 100644 index 00000000..314efdac --- /dev/null +++ b/apps/readest-app/src/__tests__/services/tts-pcm.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, test } from 'vitest'; + +import { findSpeechBounds } from '@/services/tts/pcm'; + +const SR = 24000; + +const makeSignal = ( + leadingSilenceSec: number, + speechSec: number, + trailingSilenceSec: number, + noiseFloor = 0, +) => { + const total = Math.round((leadingSilenceSec + speechSec + trailingSilenceSec) * SR); + const samples = new Float32Array(total); + const speechStart = Math.round(leadingSilenceSec * SR); + const speechEnd = speechStart + Math.round(speechSec * SR); + for (let i = 0; i < total; i++) { + if (i >= speechStart && i < speechEnd) { + samples[i] = 0.3 * Math.sin((2 * Math.PI * 440 * i) / SR); + } else if (noiseFloor > 0) { + // Deterministic pseudo-noise below the detection threshold, emulating + // MP3 decoder dither/ringing in "silent" passages. + samples[i] = noiseFloor * Math.sin((2 * Math.PI * 1731 * i) / SR + i * 0.7); + } + } + return samples; +}; + +describe('findSpeechBounds', () => { + test('trims leading and trailing silence with head/tail pads', () => { + const samples = makeSignal(0.5, 1.0, 0.8); + const { startSec, endSec } = findSpeechBounds(samples, SR); + expect(startSec).toBeGreaterThan(0.47 - 1e-6); + expect(startSec).toBeLessThan(0.53); + expect(endSec).toBeGreaterThan(1.44); + expect(endSec).toBeLessThan(1.56 + 1e-6); + expect(endSec).toBeGreaterThan(startSec); + }); + + test('ignores a realistic decoder noise floor in silent passages', () => { + const samples = makeSignal(0.5, 1.0, 0.8, 0.0008); + const { startSec, endSec } = findSpeechBounds(samples, SR); + expect(startSec).toBeGreaterThan(0.4); + expect(startSec).toBeLessThan(0.53); + expect(endSec).toBeGreaterThan(1.44); + expect(endSec).toBeLessThan(1.6); + }); + + test('all-silence input returns the full range', () => { + const samples = new Float32Array(SR); // 1s of zeros + const { startSec, endSec } = findSpeechBounds(samples, SR); + expect(startSec).toBe(0); + expect(endSec).toBeCloseTo(1, 5); + }); + + test('empty input returns zero bounds', () => { + const { startSec, endSec } = findSpeechBounds(new Float32Array(0), SR); + expect(startSec).toBe(0); + expect(endSec).toBe(0); + }); + + test('speech reaching the buffer edges clamps to the buffer', () => { + const samples = makeSignal(0, 0.5, 0); + const { startSec, endSec } = findSpeechBounds(samples, SR); + expect(startSec).toBe(0); + expect(endSec).toBeCloseTo(0.5, 2); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/tts-section-timeline.test.ts b/apps/readest-app/src/__tests__/services/tts-section-timeline.test.ts new file mode 100644 index 00000000..20bc509d --- /dev/null +++ b/apps/readest-app/src/__tests__/services/tts-section-timeline.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, test } from 'vitest'; +import { textWalker } from 'foliate-js/text-walker.js'; +import { getSentences } from 'foliate-js/tts.js'; + +import { SectionTimeline, type TimelineSentence } from '@/services/tts/SectionTimeline'; +import { recordMeasuredDuration } from '@/services/tts/ttsDuration'; + +// Note: foliate's segmenter merges sentences ending in short (<=3 letter) +// words as suspected abbreviations, so every fixture sentence ends long. +const SENTENCE_A0 = 'The quick brown fox jumps over the lazy hound.'; +const SENTENCE_A1 = 'A second sentence follows the first one closely.'; +const SENTENCE_B0 = 'Another paragraph starts a new block of text.'; +const SENTENCE_B1 = 'It also carries a couple of sentences inside.'; + +const makeDoc = (): Document => { + const parser = new DOMParser(); + return parser.parseFromString( + `` + + `

${SENTENCE_A0} ${SENTENCE_A1}

` + + `

${SENTENCE_B0} ${SENTENCE_B1}

` + + ``, + 'text/html', + ); +}; + +const enumerate = (doc: Document): TimelineSentence[] => { + const sentences: TimelineSentence[] = []; + for (const { blockIndex, markName, range } of getSentences(doc, textWalker, null, 'sentence')) { + sentences.push({ blockIndex, markName, range, text: range.toString() }); + } + return sentences; +}; + +describe('SectionTimeline', () => { + test('enumerates sentences via foliate getSentences with block-scoped marks', () => { + const sentences = enumerate(makeDoc()); + expect(sentences).toHaveLength(4); + expect(sentences[0]!.blockIndex).toBe(0); + expect(sentences[2]!.blockIndex).toBe(1); + // Mark names restart per block, matching foliate's per-block mark naming. + expect(sentences[0]!.markName).toBe(sentences[2]!.markName); + expect(sentences[0]!.text.trim()).toBe(SENTENCE_A0); + expect(sentences[3]!.text.trim()).toBe(SENTENCE_B1); + }); + + test('mixes measured and estimated durations after refresh', () => { + const voice = 'timeline-voice-mixed'; + const sentences = enumerate(makeDoc()); + const timeline = new SectionTimeline(sentences, 'en', voice); + const estimatedTotal = timeline.getDuration(); + expect(estimatedTotal).toBeGreaterThan(0); + + recordMeasuredDuration(voice, SENTENCE_A0, 10); + timeline.refresh(); + const mixedTotal = timeline.getDuration(); + // Sentence A0's estimate (~3s at 15cps) was replaced by the 10s measured value. + expect(mixedTotal).toBeGreaterThan(estimatedTotal + 5); + expect(timeline.getMeasuredFraction()).toBeGreaterThan(0); + expect(timeline.getMeasuredFraction()).toBeLessThan(1); + }); + + test('positionAt sums prior durations plus the within-sentence offset', () => { + const voice = 'timeline-voice-position'; + const sentences = enumerate(makeDoc()); + recordMeasuredDuration(voice, SENTENCE_A0, 4); + recordMeasuredDuration(voice, SENTENCE_A1, 6); + const timeline = new SectionTimeline(sentences, 'en', voice); + expect(timeline.positionAt(0, 1.5)).toBeCloseTo(1.5, 5); + expect(timeline.positionAt(1, 2)).toBeCloseTo(4 + 2, 5); + // Clamps out-of-range indexes. + expect(timeline.positionAt(-1, 0)).toBe(0); + expect(timeline.positionAt(99, 0)).toBeCloseTo(timeline.getDuration(), 5); + }); + + test('rate rescales duration and position without touching stored data', () => { + const voice = 'timeline-voice-rate'; + const sentences = enumerate(makeDoc()); + recordMeasuredDuration(voice, SENTENCE_A0, 4); + const timeline = new SectionTimeline(sentences, 'en', voice); + const at1 = timeline.getDuration(); + timeline.setRate(2); + expect(timeline.getDuration()).toBeCloseTo(at1 / 2, 5); + expect(timeline.positionAt(1, 0)).toBeCloseTo(4 / 2, 5); + timeline.setRate(0.5); + expect(timeline.getDuration()).toBeCloseTo(at1 * 2, 5); + }); + + test('sentenceAtTime maps seconds to the sentence, clamping past the end', () => { + const voice = 'timeline-voice-seek'; + const sentences = enumerate(makeDoc()); + for (const [text, dur] of [ + [SENTENCE_A0, 4], + [SENTENCE_A1, 6], + [SENTENCE_B0, 5], + [SENTENCE_B1, 5], + ] as const) { + recordMeasuredDuration(voice, text, dur); + } + const timeline = new SectionTimeline(sentences, 'en', voice); + expect(timeline.sentenceAtTime(0)?.index).toBe(0); + expect(timeline.sentenceAtTime(4.5)?.index).toBe(1); + expect(timeline.sentenceAtTime(11)?.index).toBe(2); + // Past the (possibly over-estimated) end: clamp to the last sentence so a + // lock-screen scrub past the real end is not a dead gesture. + expect(timeline.sentenceAtTime(999)?.index).toBe(3); + expect(timeline.sentenceAtTime(-1)?.index).toBe(0); + // Rate applies: at 2x, 5.5 real seconds is 11 media seconds = sentence 2. + timeline.setRate(2); + expect(timeline.sentenceAtTime(5.5)?.index).toBe(2); + }); + + test('sentenceAtTime returns null for an empty timeline', () => { + const timeline = new SectionTimeline([], 'en', 'timeline-voice-empty'); + expect(timeline.sentenceAtTime(0)).toBeNull(); + expect(timeline.getDuration()).toBe(0); + }); + + test('indexOfRange finds exact ranges, sub-ranges, and rejects foreign docs', () => { + const doc = makeDoc(); + const sentences = enumerate(doc); + const timeline = new SectionTimeline(sentences, 'en', 'timeline-voice-range'); + expect(timeline.indexOfRange(sentences[2]!.range)).toBe(2); + + // A collapsed range inside sentence 3 still resolves to sentence 3. + const sub = sentences[3]!.range.cloneRange(); + sub.collapse(false); + expect(timeline.indexOfRange(sub)).toBe(3); + + const foreign = makeDoc().createRange(); + expect(timeline.indexOfRange(foreign)).toBe(-1); + }); + + test('setVoice re-estimates unmeasured entries under the new voice', () => { + const voiceA = 'timeline-voice-a'; + const voiceB = 'timeline-voice-b'; + const sentences = enumerate(makeDoc()); + recordMeasuredDuration(voiceA, SENTENCE_A0, 20); + const timeline = new SectionTimeline(sentences, 'en', voiceA); + const withMeasured = timeline.getDuration(); + timeline.setVoice(voiceB); + // Voice B has no measured data: everything estimates again. + expect(timeline.getDuration()).toBeLessThan(withMeasured); + expect(timeline.getMeasuredFraction()).toBe(0); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/tts-time-stretch.test.ts b/apps/readest-app/src/__tests__/services/tts-time-stretch.test.ts new file mode 100644 index 00000000..0f28b164 --- /dev/null +++ b/apps/readest-app/src/__tests__/services/tts-time-stretch.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, test } from 'vitest'; + +import { timeStretch } from '@/services/tts/timeStretch'; + +const SR = 24000; + +const makeSpeechLike = (seconds: number) => { + const n = Math.round(seconds * SR); + const out = new Float32Array(n); + for (let i = 0; i < n; i++) { + const t = i / SR; + // Multi-sine with slow amplitude modulation, roughly speech-shaped. + const env = 0.5 + 0.5 * Math.sin(2 * Math.PI * 3 * t); + out[i] = + env * + (0.3 * Math.sin(2 * Math.PI * 220 * t) + + 0.2 * Math.sin(2 * Math.PI * 470 * t) + + 0.1 * Math.sin(2 * Math.PI * 910 * t)); + } + return out; +}; + +const makeSine = (seconds: number, freq: number) => { + const n = Math.round(seconds * SR); + const out = new Float32Array(n); + for (let i = 0; i < n; i++) { + out[i] = 0.5 * Math.sin((2 * Math.PI * freq * i) / SR); + } + return out; +}; + +const zeroCrossingsPerSec = (samples: Float32Array) => { + let crossings = 0; + for (let i = 1; i < samples.length; i++) { + if (samples[i - 1]! <= 0 !== samples[i]! <= 0) crossings++; + } + return crossings / (samples.length / SR); +}; + +describe('timeStretch', () => { + test('tempo 1.5 shortens output to ~1/1.5 of input length', () => { + const input = makeSpeechLike(3); + const output = timeStretch(input, SR, 1.5); + const ratio = output.length / input.length; + expect(ratio).toBeGreaterThan((1 / 1.5) * 0.95); + expect(ratio).toBeLessThan((1 / 1.5) * 1.05); + }); + + test('tempo 0.75 lengthens output to ~1/0.75 of input length', () => { + const input = makeSpeechLike(3); + const output = timeStretch(input, SR, 0.75); + const ratio = output.length / input.length; + expect(ratio).toBeGreaterThan((1 / 0.75) * 0.95); + expect(ratio).toBeLessThan((1 / 0.75) * 1.05); + }); + + test('preserves pitch: zero-crossing rate unchanged at tempo 1.5', () => { + const input = makeSine(1, 440); + const output = timeStretch(input, SR, 1.5); + const inputZps = zeroCrossingsPerSec(input); // ~880 + const outputZps = zeroCrossingsPerSec(output); + // A resampler would land near 1320; WSOLA must stay near the input rate. + expect(outputZps).toBeGreaterThan(inputZps * 0.97); + expect(outputZps).toBeLessThan(inputZps * 1.03); + }); + + test('tempo 1 returns an equal-length copy, not the same reference', () => { + const input = makeSpeechLike(1); + const output = timeStretch(input, SR, 1); + expect(output).not.toBe(input); + expect(output.length).toBe(input.length); + expect(output[1234]).toBe(input[1234]); + }); + + test('all-zero input produces finite output (zero-energy correlation guard)', () => { + const input = new Float32Array(SR); // 1s of digital silence + const output = timeStretch(input, SR, 1.5); + expect(output.length).toBeGreaterThan(0); + expect(output.every((s) => Number.isFinite(s))).toBe(true); + }); + + test('slow tempo 0.2 produces ~5x length with finite samples', () => { + const input = makeSpeechLike(1); + const output = timeStretch(input, SR, 0.2); + const ratio = output.length / input.length; + expect(ratio).toBeGreaterThan(5 * 0.95); + expect(ratio).toBeLessThan(5 * 1.05); + expect(output.every((s) => Number.isFinite(s))).toBe(true); + }); + + test('very short input is returned as a copy (below two frames)', () => { + const input = makeSine(0.05, 440); // 50ms < 2 x 40ms frames + const output = timeStretch(input, SR, 2); + expect(output.length).toBe(input.length); + }); + + test('does not mutate its input (input may be a subarray view)', () => { + const input = makeSpeechLike(1); + const snapshot = input.slice(); + timeStretch(input, SR, 1.5); + expect(input).toEqual(snapshot); + }); +}); diff --git a/apps/readest-app/src/__tests__/services/tts-web-audio-player.test.ts b/apps/readest-app/src/__tests__/services/tts-web-audio-player.test.ts new file mode 100644 index 00000000..a278feeb --- /dev/null +++ b/apps/readest-app/src/__tests__/services/tts-web-audio-player.test.ts @@ -0,0 +1,312 @@ +import { afterEach, describe, expect, test } from 'vitest'; + +import { WebAudioPlayer, type WebAudioPlayerEvent } from '@/services/tts/WebAudioPlayer'; +import { FakeAudioContext, makeBuffer } from './tts-fake-audio'; + +const SAFETY = 0.03; + +const setVisibility = (value: 'visible' | 'hidden') => { + Object.defineProperty(document, 'visibilityState', { + configurable: true, + get: () => value, + }); +}; + +afterEach(() => { + setVisibility('visible'); +}); + +const setup = () => { + const ctx = new FakeAudioContext(); + const player = new WebAudioPlayer(() => ctx); + const events: WebAudioPlayerEvent[] = []; + return { ctx, player, events, onEvent: (e: WebAudioPlayerEvent) => events.push(e) }; +}; + +describe('WebAudioPlayer scheduling', () => { + test('chunks are scheduled contiguously with the requested gap', async () => { + const { ctx, player, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.scheduleChunk(gen, makeBuffer(2), { trimStartSec: 0, mediaScale: 1, gapSec: 0.5 }); + player.scheduleChunk(gen, makeBuffer(3), { trimStartSec: 0, mediaScale: 1, gapSec: 0.5 }); + expect(ctx.sources[0]!.startedAt).toBeCloseTo(SAFETY, 5); + expect(ctx.sources[1]!.startedAt).toBeCloseTo(SAFETY + 2 + 0.5, 5); + }); + + test('chunk-start fires at schedule for index 0 and on prior onended after', async () => { + const { ctx, player, events, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.scheduleChunk(gen, makeBuffer(2), { trimStartSec: 0, mediaScale: 1, gapSec: 0.2 }); + expect(events).toEqual([{ type: 'chunk-start', chunkIndex: 0 }]); + player.scheduleChunk(gen, makeBuffer(1), { trimStartSec: 0, mediaScale: 1, gapSec: 0.2 }); + expect(events).toHaveLength(1); + await ctx.advanceTo(SAFETY + 2); + expect(events).toEqual([ + { type: 'chunk-start', chunkIndex: 0 }, + { type: 'chunk-start', chunkIndex: 1 }, + ]); + }); + + test('stale-generation scheduleChunk is a no-op', async () => { + const { ctx, player, events, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.startSession(() => {}); + player.scheduleChunk(gen, makeBuffer(1), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + expect(ctx.sources).toHaveLength(0); + expect(events).toHaveLength(0); + }); +}); + +describe('WebAudioPlayer backpressure', () => { + test('third chunk waits until the first finishes while visible', async () => { + const { ctx, player, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.scheduleChunk(gen, makeBuffer(2), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + player.scheduleChunk(gen, makeBuffer(2), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + let resolved: boolean | null = null; + const wait = player.waitUntilReady(gen).then((r) => { + resolved = r; + return r; + }); + await Promise.resolve(); + expect(resolved).toBeNull(); + await ctx.advanceTo(SAFETY + 2); + expect(await wait).toBe(true); + }); + + test('hidden visibility deepens the pending-chunk budget to 5', async () => { + setVisibility('hidden'); + const { player, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + for (let i = 0; i < 4; i++) { + player.scheduleChunk(gen, makeBuffer(1), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + } + expect(await player.waitUntilReady(gen)).toBe(true); + }); + + test('seconds cap blocks scheduling far ahead even under the chunk budget', async () => { + setVisibility('hidden'); + const { ctx, player, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.scheduleChunk(gen, makeBuffer(30), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + player.scheduleChunk(gen, makeBuffer(31), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + let resolved: boolean | null = null; + const wait = player.waitUntilReady(gen).then((r) => { + resolved = r; + return r; + }); + await Promise.resolve(); + expect(resolved).toBeNull(); // 61s ahead > 60s cap, though only 2 < 5 chunks + await ctx.advanceTo(SAFETY + 30); + expect(await wait).toBe(true); + }); + + test('waitUntilReady returns false for a stale generation', async () => { + const { player, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.startSession(() => {}); + expect(await player.waitUntilReady(gen)).toBe(false); + }); +}); + +describe('WebAudioPlayer session end', () => { + test('session-end fires after endSession + last onended', async () => { + const { ctx, player, events, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.scheduleChunk(gen, makeBuffer(1), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + player.endSession(gen); + expect(events.filter((e) => e.type === 'session-end')).toHaveLength(0); + await ctx.advanceTo(SAFETY + 1); + expect(events.filter((e) => e.type === 'session-end')).toHaveLength(1); + }); + + test('endSession with zero scheduled chunks fires session-end synchronously', async () => { + const { player, events, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.endSession(gen); + expect(events).toEqual([{ type: 'session-end' }]); + }); + + test('endSession after the last onended already fired still ends the session', async () => { + const { ctx, player, events, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.scheduleChunk(gen, makeBuffer(1), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + await ctx.advanceTo(SAFETY + 1); + expect(events.filter((e) => e.type === 'session-end')).toHaveLength(0); + player.endSession(gen); + expect(events.filter((e) => e.type === 'session-end')).toHaveLength(1); + }); + + test('session-end is emitted exactly once', async () => { + const { ctx, player, events, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.scheduleChunk(gen, makeBuffer(1), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + player.endSession(gen); + player.endSession(gen); + await ctx.advanceTo(SAFETY + 1); + await ctx.advanceTo(SAFETY + 2); + expect(events.filter((e) => e.type === 'session-end')).toHaveLength(1); + }); +}); + +describe('WebAudioPlayer abort', () => { + test('abort stops pending sources, resolves waiters false, silences events', async () => { + const { ctx, player, events, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + for (let i = 0; i < 3; i++) { + player.scheduleChunk(gen, makeBuffer(2), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + } + const wait = player.waitUntilReady(gen); + player.abortSession(); + expect(await wait).toBe(false); + expect(ctx.sources.every((s) => s.stopped)).toBe(true); + const countBefore = events.length; + await ctx.advanceTo(100); + expect(events).toHaveLength(countBefore); + }); + + test('double abortSession is idempotent', async () => { + const { player, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.scheduleChunk(gen, makeBuffer(1), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + player.abortSession(); + expect(() => player.abortSession()).not.toThrow(); + }); + + test('abort storm: 10 rapid startSession calls leave one live session, no stale events', async () => { + const { ctx, player, onEvent } = setup(); + await player.ensureContext(); + const gens: number[] = []; + const staleEvents: WebAudioPlayerEvent[] = []; + for (let i = 0; i < 10; i++) { + const gen = player.startSession(i === 9 ? onEvent : (e) => staleEvents.push(e)); + player.scheduleChunk(gen, makeBuffer(1), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + gens.push(gen); + } + for (const gen of gens.slice(0, 9)) { + player.scheduleChunk(gen, makeBuffer(1), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + expect(await player.waitUntilReady(gen)).toBe(false); + } + // Stale sessions emitted only their own synchronous chunk-start 0 before + // being aborted; nothing new after the storm. + const staleCount = staleEvents.length; + await ctx.advanceTo(50); + expect(staleEvents).toHaveLength(staleCount); + expect(await player.waitUntilReady(gens[9]!)).toBe(true); + }); +}); + +describe('WebAudioPlayer pause/resume and interruption', () => { + test('pauseContext suspends; resumeContext resumes', async () => { + const { ctx, player } = setup(); + await player.ensureContext(); + player.startSession(() => {}); + await player.pauseContext(); + expect(ctx.state).toBe('suspended'); + await player.resumeContext(); + expect(ctx.state).toBe('running'); + }); + + test('auto-resumes on unexpected suspension while a session is live', async () => { + const { ctx, player, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.scheduleChunk(gen, makeBuffer(2), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + const before = ctx.resumeCalls; + ctx.setState('interrupted'); + await Promise.resolve(); + expect(ctx.resumeCalls).toBeGreaterThan(before); + }); + + test('user pause suppresses auto-resume', async () => { + const { ctx, player, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.scheduleChunk(gen, makeBuffer(2), { trimStartSec: 0, mediaScale: 1, gapSec: 0 }); + await player.pauseContext(); + const before = ctx.resumeCalls; + ctx.setState('interrupted'); + await Promise.resolve(); + expect(ctx.resumeCalls).toBe(before); + }); + + test('resumeContext throws when the context refuses to run again', async () => { + const { ctx, player } = setup(); + await player.ensureContext(); + await player.pauseContext(); + ctx.resumeImpl = () => { + ctx.state = 'interrupted'; + }; + await expect(player.resumeContext()).rejects.toThrow(); + }); +}); + +describe('WebAudioPlayer playback position', () => { + test('maps playback time through trim offset and media scale', async () => { + const { ctx, player, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + // 2s output chunk covering original media [0.3, 3.3) stretched 1.5x. + player.scheduleChunk(gen, makeBuffer(2), { trimStartSec: 0.3, mediaScale: 1.5, gapSec: 0.2 }); + await ctx.advanceTo(SAFETY + 1); + const pos = player.getPlaybackPosition(gen); + expect(pos?.chunkIndex).toBe(0); + expect(pos?.mediaTimeSec).toBeCloseTo(0.3 + 1 * 1.5, 3); + }); + + test('clamps before the first chunk and in the inter-chunk gap', async () => { + const { ctx, player, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + player.scheduleChunk(gen, makeBuffer(1), { trimStartSec: 0.2, mediaScale: 1, gapSec: 1 }); + player.scheduleChunk(gen, makeBuffer(1), { trimStartSec: 0.1, mediaScale: 1, gapSec: 1 }); + // Before the first chunk starts. + expect(player.getPlaybackPosition(gen)?.mediaTimeSec).toBeCloseTo(0.2, 5); + // Inside the gap after chunk 0 (ends at SAFETY+1; next starts SAFETY+2). + ctx.currentTime = SAFETY + 1.5; + const pos = player.getPlaybackPosition(gen); + expect(pos?.chunkIndex).toBe(0); + expect(pos?.mediaTimeSec).toBeCloseTo(0.2 + 1, 5); + }); + + test('returns null for a stale generation or empty session', async () => { + const { player, onEvent } = setup(); + await player.ensureContext(); + const gen = player.startSession(onEvent); + expect(player.getPlaybackPosition(gen)).toBeNull(); + player.startSession(() => {}); + expect(player.getPlaybackPosition(gen)).toBeNull(); + }); +}); + +describe('WebAudioPlayer buffers', () => { + test('createMonoBuffer preserves samples and sample rate', async () => { + const { player } = setup(); + const samples = new Float32Array([0.1, -0.2, 0.3, -0.4]); + const buffer = await player.createMonoBuffer(samples, 48000); + expect(buffer.sampleRate).toBe(48000); + expect(buffer.length).toBe(4); + expect(Array.from(buffer.getChannelData(0))).toEqual(Array.from(samples)); + }); + + test('decode resolves through the context decoder at its sample rate', async () => { + const ctx = new FakeAudioContext(48000); + const player = new WebAudioPlayer(() => ctx); + const buffer = await player.decode(new ArrayBuffer(96000)); + expect(buffer.sampleRate).toBe(48000); + expect(buffer.duration).toBeCloseTo(2, 5); + }); +}); diff --git a/apps/readest-app/src/__tests__/utils/time-format.test.ts b/apps/readest-app/src/__tests__/utils/time-format.test.ts new file mode 100644 index 00000000..9643dbc1 --- /dev/null +++ b/apps/readest-app/src/__tests__/utils/time-format.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, test } from 'vitest'; + +import { formatPlaybackTime } from '@/utils/time'; + +describe('formatPlaybackTime', () => { + test('formats minutes and seconds by default', () => { + expect(formatPlaybackTime(0)).toBe('0:00'); + expect(formatPlaybackTime(5)).toBe('0:05'); + expect(formatPlaybackTime(62)).toBe('1:02'); + expect(formatPlaybackTime(600)).toBe('10:00'); + }); + + test('formats hours when the value reaches one hour', () => { + expect(formatPlaybackTime(3600)).toBe('1:00:00'); + expect(formatPlaybackTime(3600 + 61)).toBe('1:01:01'); + }); + + test('forceHours pins both labels of a row to the same layout', () => { + // Both labels use the format chosen by the TOTAL's magnitude so the row + // never re-layouts when the elapsed time crosses an hour. + expect(formatPlaybackTime(62, true)).toBe('0:01:02'); + }); + + test('clamps negative and non-finite input to zero', () => { + expect(formatPlaybackTime(-5)).toBe('0:00'); + expect(formatPlaybackTime(Number.NaN)).toBe('0:00'); + expect(formatPlaybackTime(Number.POSITIVE_INFINITY)).toBe('0:00'); + }); + + test('truncates fractional seconds', () => { + expect(formatPlaybackTime(59.9)).toBe('0:59'); + }); +}); diff --git a/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx b/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx index 56174c5e..59c3033a 100644 --- a/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx +++ b/apps/readest-app/src/app/reader/components/tts/TTSControl.tsx @@ -15,7 +15,11 @@ import TTSIcon from './TTSIcon'; import TTSBar from './TTSBar'; const POPUP_WIDTH = 282; +// The popup is fixed-height and non-scrolling: these must track the panel's +// content or rows get clipped. The taller variant fits the scrubber row that +// only timeline-capable (Edge) playback shows. const POPUP_HEIGHT = 160; +const POPUP_HEIGHT_WITH_PROGRESS = 184; const POPUP_PADDING = 10; interface TTSControlProps { @@ -46,13 +50,17 @@ const TTSControl: React.FC = ({ bookKey, gridInsets }) => { const popupPadding = useResponsiveSize(POPUP_PADDING); const maxWidth = window.innerWidth - 2 * popupPadding; const popupWidth = Math.min(maxWidth, useResponsiveSize(POPUP_WIDTH)); - const popupHeight = useResponsiveSize(POPUP_HEIGHT); + const popupBaseHeight = useResponsiveSize(POPUP_HEIGHT); + const popupProgressHeight = useResponsiveSize(POPUP_HEIGHT_WITH_PROGRESS); const tts = useTTSControl({ bookKey, onRequestHidePanel: () => setShowPanel(false), }); + const hasPlaybackInfo = tts.ttsClientsInited && tts.handleSupportsPlaybackInfo(); + const popupHeight = hasPlaybackInfo ? popupProgressHeight : popupBaseHeight; + useEffect(() => { if (tts.showBackToCurrentTTSLocation) { setShouldMountBackButton(true); @@ -230,6 +238,9 @@ const TTSControl: React.FC = ({ bookKey, gridInsets }) => { onGetVoiceId={tts.handleGetVoiceId} onSelectTimeout={tts.handleSelectTimeout} onToogleTTSBar={tts.handleToggleTTSBar} + onSeek={tts.handleSeekTo} + onGetPlaybackInfo={tts.handleGetPlaybackInfo} + hasTimeline={hasPlaybackInfo} /> )} diff --git a/apps/readest-app/src/app/reader/components/tts/TTSPanel.tsx b/apps/readest-app/src/app/reader/components/tts/TTSPanel.tsx index 9ad934fc..1ad1577a 100644 --- a/apps/readest-app/src/app/reader/components/tts/TTSPanel.tsx +++ b/apps/readest-app/src/app/reader/components/tts/TTSPanel.tsx @@ -1,5 +1,5 @@ import clsx from 'clsx'; -import { useState, ChangeEvent, useEffect } from 'react'; +import { useState, ChangeEvent, useEffect, useRef, useCallback } from 'react'; import { MdPlayCircle, MdPauseCircle, MdFastRewind, MdFastForward, MdAlarm } from 'react-icons/md'; import { TbChevronCompactDown, TbChevronCompactUp } from 'react-icons/tb'; import { RiVoiceAiFill } from 'react-icons/ri'; @@ -11,6 +11,14 @@ import { TranslationFunc, useTranslation } from '@/hooks/useTranslation'; import { useSettingsStore } from '@/store/settingsStore'; import { useDefaultIconSize, useResponsiveSize } from '@/hooks/useResponsiveSize'; import { getLanguageName } from '@/utils/lang'; +import { formatPlaybackTime } from '@/utils/time'; +import { eventDispatcher } from '@/utils/event'; + +type TTSPlaybackInfo = { + position: number; + duration: number; + measuredFraction: number; +}; type TTSPanelProps = { bookKey: string; @@ -27,6 +35,161 @@ type TTSPanelProps = { onGetVoiceId: () => string; onSelectTimeout: (bookKey: string, value: number) => void; onToogleTTSBar: () => void; + onSeek: (seconds: number) => Promise; + onGetPlaybackInfo: () => TTSPlaybackInfo | null; + hasTimeline: boolean; +}; + +// Suppress poll updates briefly after a seek so the optimistic thumb never +// snaps back while the cold-seek fetch completes. +const SEEK_SUPPRESS_MS = 2000; +// Small backward drifts come from estimate refinement, not playback — hold the +// displayed position monotonic; larger jumps are deliberate (seek, chapter). +const MONOTONIC_SLACK_SEC = 3; + +// Chapter progress row: elapsed / thin scrubber / total. Lives between the +// rate slider and the transport cluster; the thin range-xs track plus flanking +// time labels keep it visually distinct from the chunky tick-labeled rate +// slider one block up (misgrabbing THAT persists a global setting). +const TTSProgressRow = ({ + bookKey, + isEink, + onSeek, + onGetPlaybackInfo, +}: { + bookKey: string; + isEink: boolean; + onSeek: (seconds: number) => Promise; + onGetPlaybackInfo: () => TTSPlaybackInfo | null; +}) => { + const _ = useTranslation(); + const [info, setInfo] = useState(null); + const [stale, setStale] = useState(true); + const [displayTotal, setDisplayTotal] = useState(null); + const [dragValue, setDragValue] = useState(null); + const dragValueRef = useRef(null); + const suppressUntilRef = useRef(0); + const lastPositionRef = useRef(0); + const keyboardCommitRef = useRef | null>(null); + + const refresh = useCallback(() => { + if (dragValueRef.current !== null) return; + if (Date.now() < suppressUntilRef.current) return; + const next = onGetPlaybackInfo(); + if (!next) { + // Keep the last-known values rendered (disabled) across chapter + // transitions and while the lazy timeline builds — no row blink. + setStale(true); + return; + } + let position = next.position; + if ( + position < lastPositionRef.current && + lastPositionRef.current - position < MONOTONIC_SLACK_SEC + ) { + position = lastPositionRef.current; + } + lastPositionRef.current = position; + setInfo({ ...next, position }); + setStale(false); + // Quantize the displayed total: only follow estimate drift when it moves + // by more than 2%, so the chapter length reads stable, not twitchy. + setDisplayTotal((prev) => + prev === null || Math.abs(next.duration - prev) / prev > 0.02 ? next.duration : prev, + ); + }, [onGetPlaybackInfo]); + + useEffect(() => { + refresh(); + if (isEink) { + // No 1s repaints on e-ink: follow sentence-level position events only. + const handler = (event: CustomEvent) => { + const detail = event.detail as { bookKey?: string; kind?: string }; + if (detail.bookKey === bookKey && detail.kind === 'sentence') refresh(); + }; + eventDispatcher.on('tts-position', handler); + return () => eventDispatcher.off('tts-position', handler); + } + const interval = setInterval(refresh, 1000); + return () => clearInterval(interval); + }, [refresh, isEink, bookKey]); + + const commitSeek = (seconds: number) => { + dragValueRef.current = null; + setDragValue(null); + // Optimistic thumb: land on the target immediately and hold it there + // while the seek resolves; never snap back. + const previous = lastPositionRef.current; + suppressUntilRef.current = Date.now() + SEEK_SUPPRESS_MS; + lastPositionRef.current = seconds; + setInfo((prev) => (prev ? { ...prev, position: seconds } : prev)); + onSeek(seconds).catch(() => { + // A silent snap-back is the exact violation the optimistic thumb + // prevents — restore visibly and say why. + suppressUntilRef.current = 0; + lastPositionRef.current = previous; + setInfo((prev) => (prev ? { ...prev, position: previous } : prev)); + eventDispatcher.dispatch('toast', { message: _('Failed to seek'), type: 'error' }); + }); + }; + + const handleChange = (e: ChangeEvent) => { + // React fires range onChange continuously during a drag; track the value + // and commit only on pointer/key release. + const value = parseFloat(e.target.value); + dragValueRef.current = value; + setDragValue(value); + }; + + const handlePointerCommit = () => { + if (dragValueRef.current !== null) commitSeek(dragValueRef.current); + }; + + const handleKeyUp = () => { + // Holding an arrow key must not fire a network seek per press. + if (keyboardCommitRef.current) clearTimeout(keyboardCommitRef.current); + keyboardCommitRef.current = setTimeout(() => { + if (dragValueRef.current !== null) commitSeek(dragValueRef.current); + }, 500); + }; + + const total = displayTotal ?? info?.duration ?? 0; + const position = Math.min(dragValue ?? info?.position ?? 0, total); + const ready = info !== null && total > 0; + const forceHours = total >= 3600; + const step = Math.max(5, Math.round(total / 100)) || 5; + const elapsedLabel = ready ? formatPlaybackTime(position, forceHours) : '--:--'; + const remainingLabel = ready + ? `-${formatPlaybackTime(Math.max(total - position, 0), forceHours)}` + : '--:--'; + + return ( +
+ {elapsedLabel} + {/* Plain native range, matching the footer's Jump to Location slider: + thin track, small thumb, visually unmistakable for the chunky rate + slider above. */} + + {remainingLabel} +
+ ); }; const getTTSTimeoutOptions = (_: TranslationFunc) => { @@ -116,6 +279,9 @@ const TTSPanel = ({ onGetVoiceId, onSelectTimeout, onToogleTTSBar, + onSeek, + onGetPlaybackInfo, + hasTimeline, }: TTSPanelProps) => { const _ = useTranslation(); const { envConfig } = useEnv(); @@ -233,6 +399,14 @@ const TTSPanel = ({ {_('Fast')} + {hasTimeline && ( + + )}