42f9b8fe3c
* feat(tts): add PCM speech-bounds detection for sentence audio trimming Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(tts): add WSOLA time-stretch for pitch-preserved playback rate Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(tts): add sentence duration store with per-voice speaking-rate calibration Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * refactor(tts): serve edge audio as ArrayBuffer with in-flight fetch dedup Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(tts): add WebAudioPlayer with gapless chunk scheduling and backpressure Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(tts): play edge TTS through gapless Web Audio pipeline Replaces the per-sentence audio element with trimmed, time-stretched buffers scheduled on the shared AudioContext. Marks dispatch at audible time so schedule-ahead cannot run foliate's cursor past the voice; a decode failure or missing audio skips the chunk instead of wedging the session; pause and resume ride context suspend and resume with no iOS rewind hack; the object-URL cache is gone. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(tts): add section timeline with measured and estimated sentence durations Includes the foliate-js submodule bump for the getSentences export (fork branch feat/tts-get-sentences; fork PR must merge before this lands so the pinned SHA resolves). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(tts): expose section playback position and sentence-snapped seeking Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(tts): surface playback position and seek in the media session Position state is clamped, never skipped, so the lock-screen scrubber stays live when estimates overshoot; seekto units map per backend (native ms, web seconds). The AudioContext warms up in the tts-speak gesture path before any await, and the silent keep-alive element now runs on all platforms so desktop hardware media keys survive the removal of the per-sentence audio element. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(tts): add seekable chapter progress bar to the TTS panel The scrubber joins the transport cluster with a thin range-xs track and flanking tabular time labels so it cannot be misgrabbed for the chunky rate slider (which persists a global setting). States: reserved disabled slot until the lazy timeline lands, persists across chapter transitions, optimistic thumb with failure toast, monotonic position, tilde-prefixed estimated totals, sentence-event updates under e-ink. The popup grows only when a timeline-capable client is active. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * docs: record deferred TTS listening-engine follow-ups Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * docs: record background TTS decoupling design decisions Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(tts): slim the panel scrubber to a native track with remaining time Match the footer Jump to Location slider (plain native range: thin track, small thumb) instead of the chunky daisyUI pill, show remaining time with a minus prefix on the right, and drop the This chapter caption. Popup height shrinks accordingly. Verified live in Chrome. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * chore: pin foliate-js to merged main with getSentences export Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(tts): catch autoplay rejection from the keep-alive element Running the silent keep-alive on all platforms exposed an un-awaited play() that headless Chromium rejects without a user gesture, failing CI on unhandled rejections while every test passed. The keep-alive is best-effort; the production path is gesture-qualified. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
127 lines
4.4 KiB
TypeScript
127 lines
4.4 KiB
TypeScript
import { useRef } from 'react';
|
|
import { useSettingsStore } from '@/store/settingsStore';
|
|
import { useBookDataStore } from '@/store/bookDataStore';
|
|
import { useReaderStore } from '@/store/readerStore';
|
|
import { buildTTSMediaMetadata } from '@/utils/ttsMetadata';
|
|
import { useTranslation } from '@/hooks/useTranslation';
|
|
import { SILENCE_DATA } from '@/services/tts';
|
|
import { getMediaSession, TauriMediaSession } from '@/libs/mediaSession';
|
|
import { fetchImageAsBase64 } from '@/utils/image';
|
|
|
|
interface UseTTSMediaSessionProps {
|
|
bookKey: string;
|
|
}
|
|
|
|
export const useTTSMediaSession = ({ bookKey }: UseTTSMediaSessionProps) => {
|
|
const _ = useTranslation();
|
|
const { settings } = useSettingsStore();
|
|
const { getBookData } = useBookDataStore();
|
|
const { getProgress, getViewSettings } = useReaderStore();
|
|
|
|
const mediaSessionRef = useRef<TauriMediaSession | MediaSession | null>(null);
|
|
const unblockerAudioRef = useRef<HTMLAudioElement | null>(null);
|
|
|
|
// this enables WebAudio to play even when the mute toggle switch is ON
|
|
const unblockAudio = () => {
|
|
if (unblockerAudioRef.current) return;
|
|
unblockerAudioRef.current = document.createElement('audio');
|
|
unblockerAudioRef.current.setAttribute('x-webkit-airplay', 'deny');
|
|
unblockerAudioRef.current.addEventListener('play', () => {
|
|
if ('mediaSession' in navigator) {
|
|
navigator.mediaSession.metadata = null;
|
|
}
|
|
});
|
|
unblockerAudioRef.current.preload = 'auto';
|
|
unblockerAudioRef.current.loop = true;
|
|
unblockerAudioRef.current.src = SILENCE_DATA;
|
|
unblockerAudioRef.current.play().catch((err) => {
|
|
// Autoplay policy rejects play() outside a user gesture (headless test
|
|
// runs, programmatic TTS starts). The keep-alive is best-effort: the
|
|
// production path calls this inside the tts-speak gesture handler, and
|
|
// a rejection here must not surface as an unhandled rejection.
|
|
console.warn('Keep-alive audio blocked:', err);
|
|
});
|
|
};
|
|
|
|
const releaseUnblockAudio = () => {
|
|
if (!unblockerAudioRef.current) return;
|
|
try {
|
|
unblockerAudioRef.current.pause();
|
|
unblockerAudioRef.current.currentTime = 0;
|
|
unblockerAudioRef.current.removeAttribute('src');
|
|
unblockerAudioRef.current.src = '';
|
|
unblockerAudioRef.current.load();
|
|
unblockerAudioRef.current = null;
|
|
console.log('Unblock audio released');
|
|
} catch (err) {
|
|
console.warn('Error releasing unblock audio:', err);
|
|
}
|
|
};
|
|
|
|
const initMediaSession = async () => {
|
|
const mediaSession = getMediaSession();
|
|
if (!mediaSession) return;
|
|
|
|
mediaSessionRef.current = mediaSession;
|
|
|
|
if (mediaSession instanceof TauriMediaSession) {
|
|
const bookData = getBookData(bookKey);
|
|
const progress = getProgress(bookKey);
|
|
const viewSettings = getViewSettings(bookKey);
|
|
if (!bookData || !bookData.book) return;
|
|
const { title, author, coverImageUrl } = bookData.book;
|
|
const { sectionLabel } = progress || {};
|
|
const ttsMediaMetadataMode = viewSettings?.ttsMediaMetadata ?? 'sentence';
|
|
|
|
let artworkImage = '/icon.png';
|
|
try {
|
|
artworkImage = await fetchImageAsBase64(coverImageUrl || '/icon.png');
|
|
} catch {
|
|
artworkImage = await fetchImageAsBase64('/icon.png');
|
|
}
|
|
|
|
await mediaSession.setActive({
|
|
active: true,
|
|
keepAppInForeground: settings.alwaysInForeground,
|
|
notificationTitle: _('Read Aloud'),
|
|
notificationText: _('Ready to read aloud'),
|
|
foregroundServiceTitle: _('Read Aloud'),
|
|
foregroundServiceText: _('Ready to read aloud'),
|
|
});
|
|
const metadata = buildTTSMediaMetadata({
|
|
markText: title,
|
|
markName: '0',
|
|
sectionLabel: sectionLabel || '',
|
|
title,
|
|
author,
|
|
ttsMediaMetadata: ttsMediaMetadataMode,
|
|
});
|
|
mediaSession.updateMetadata({
|
|
title: metadata.title,
|
|
artist: metadata.artist,
|
|
album: metadata.album,
|
|
artwork: artworkImage,
|
|
});
|
|
}
|
|
};
|
|
|
|
const deinitMediaSession = async () => {
|
|
if (mediaSessionRef.current && mediaSessionRef.current instanceof TauriMediaSession) {
|
|
await mediaSessionRef.current.setActive({
|
|
active: false,
|
|
keepAppInForeground: settings.alwaysInForeground,
|
|
});
|
|
}
|
|
mediaSessionRef.current = null;
|
|
};
|
|
|
|
return {
|
|
mediaSessionRef,
|
|
unblockerAudioRef,
|
|
unblockAudio,
|
|
releaseUnblockAudio,
|
|
initMediaSession,
|
|
deinitMediaSession,
|
|
};
|
|
};
|