akai-tools/readest
2
0
Fork 0
forked from akai/readest
Code Pull Requests Activity

tts: show speaking sentence and chapter info in tts media session (#1289)

Browse Source
This commit is contained in:
Huang Xin
2025-05-31 15:21:17 +08:00
committed by GitHub
parent e4d217f3aa
commit 77fa57438e
14 changed files with 148 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/readest-app/src/app/library/components/SettingsMenu.tsx
+1 -1
View File
@@ -235,6 +235,7 @@ const SettingsMenu: React.FC<SettingsMenuProps> = ({ setIsDropdownOpen }) => {
Icon={isScreenWakeLock ? MdCheck : undefined}
onClick={toggleScreenWakeLock}
/>
<MenuItem label={_('Reload Page')} onClick={handleReloadPage} />
<MenuItem
label={
themeMode === 'dark'
@@ -246,7 +247,6 @@ const SettingsMenu: React.FC<SettingsMenuProps> = ({ setIsDropdownOpen }) => {
Icon={themeMode === 'dark' ? BiMoon : themeMode === 'light' ? BiSun : TbSunMoon}
onClick={cycleThemeMode}
/>
<MenuItem label={_('Reload Page')} onClick={handleReloadPage} />
<hr className='border-base-200 my-1' />
{isWebAppPlatform() && <MenuItem label={_('Download Readest')} onClick={downloadReadest} />}
<MenuItem label={_('About Readest')} onClick={showAboutReadest} />
apps/readest-app/src/app/reader/components/FooterBar.tsx
+13 -4
View File
@@ -41,8 +41,8 @@ const FooterBar: React.FC<FooterBarProps> = ({
}) => {
const _ = useTranslation();
const { envConfig, appService } = useEnv();
const { hoveredBookKey, setHoveredBookKey, getView, getProgress, getViewSettings } =
useReaderStore();
const { hoveredBookKey, setHoveredBookKey } = useReaderStore();
const { getView, getViewState, getProgress, getViewSettings } = useReaderStore();
const { isSideBarVisible, setSideBarVisible } = useSidebarStore();
const [actionTab, setActionTab] = React.useState('');
const sliderHeight = useResponsiveSize(28);
@@ -53,6 +53,7 @@ const FooterBar: React.FC<FooterBarProps> = ({
const view = getView(bookKey);
const progress = getProgress(bookKey);
const viewSettings = getViewSettings(bookKey);
const viewState = getViewState(bookKey);
const handleProgressChange = (value: number) => {
view?.goToFraction(value / 100.0);
@@ -147,6 +148,7 @@ const FooterBar: React.FC<FooterBarProps> = ({
};
const isVisible = hoveredBookKey === bookKey;
const ttsEnabled = viewState?.ttsEnabled;
const progressInfo = bookFormat === 'PDF' ? section : pageinfo;
const progressValid = !!progressInfo;
const progressFraction =
@@ -312,7 +314,10 @@ const FooterBar: React.FC<FooterBarProps> = ({
}
onClick={() => handleSetActionTab('font')}
/>
<Button icon={<TTSIcon className='' />} onClick={() => handleSetActionTab('tts')} />
<Button
icon={<TTSIcon className={ttsEnabled ? 'text-blue-500' : ''} />}
onClick={() => handleSetActionTab('tts')}
/>
</div>
{/* Desktop footer bar */}
<div className='hidden w-full items-center gap-x-4 px-4 sm:flex'>
@@ -351,7 +356,11 @@ const FooterBar: React.FC<FooterBarProps> = ({
handleProgressChange(parseInt((e.target as HTMLInputElement).value, 10))
}
/>
<Button icon={<FaHeadphones />} onClick={handleSpeakText} tooltip={_('Speak')} />
<Button
icon={<FaHeadphones className={ttsEnabled ? 'text-blue-500' : ''} />}
onClick={handleSpeakText}
tooltip={_('Speak')}
/>
<Button
icon={viewSettings?.rtl ? <RiArrowLeftSLine /> : <RiArrowRightSLine />}
onClick={viewSettings?.rtl ? handleGoPrevPage : handleGoNextPage}
apps/readest-app/src/app/reader/components/annotator/Annotator.tsx
+11 -2
View File
@@ -91,8 +91,15 @@ const Annotator: React.FC<{ bookKey: string }> = ({ bookKey }) => {
view?.deselect();
};
const { handleScroll, handlePointerup, handleSelectionchange, handleShowPopup, handleUpToPopup } =
useTextSelector(bookKey, setSelection, handleDismissPopup);
const {
handleScroll,
handleTouchStart,
handleTouchEnd,
handlePointerup,
handleSelectionchange,
handleShowPopup,
handleUpToPopup,
} = useTextSelector(bookKey, setSelection, handleDismissPopup);
const onLoad = (event: Event) => {
const detail = (event as CustomEvent).detail;
@@ -105,7 +112,9 @@ const Annotator: React.FC<{ bookKey: string }> = ({ bookKey }) => {
};
if (bookData.book?.format !== 'PDF') {
view?.renderer?.addEventListener('scroll', handleScroll);
detail.doc?.addEventListener('touchstart', handleTouchStart);
detail.doc?.addEventListener('touchmove', handleTouchmove);
detail.doc?.addEventListener('touchend', handleTouchEnd);
detail.doc?.addEventListener('pointerup', () => handlePointerup(doc, index));
detail.doc?.addEventListener('selectionchange', () => handleSelectionchange(doc, index));
apps/readest-app/src/app/reader/components/tts/TTSControl.tsx
+47 -10
View File
@@ -5,7 +5,7 @@ import { useBookDataStore } from '@/store/bookDataStore';
import { useReaderStore } from '@/store/readerStore';
import { useTranslation } from '@/hooks/useTranslation';
import { useResponsiveSize } from '@/hooks/useResponsiveSize';
import { TTSController, SILENCE_DATA } from '@/services/tts';
import { TTSController, SILENCE_DATA, TTSMark } from '@/services/tts';
import { getPopupPosition, Position } from '@/utils/sel';
import { eventDispatcher } from '@/utils/event';
import { parseSSMLLang } from '@/utils/ssml';
@@ -24,7 +24,7 @@ const TTSControl = () => {
const _ = useTranslation();
const { appService } = useEnv();
const { getBookData } = useBookDataStore();
const { getView, getViewSettings } = useReaderStore();
const { getView, getProgress, getViewSettings, setTTSEnabled } = useReaderStore();
const [bookKey, setBookKey] = useState<string>('');
const [ttsLang, setTtsLang] = useState<string>('en');
const [isPlaying, setIsPlaying] = useState(false);
@@ -45,8 +45,10 @@ const TTSControl = () => {
const popupHeight = useResponsiveSize(POPUP_HEIGHT);
const iconRef = useRef<HTMLDivElement>(null);
const ttsControllerRef = useRef<TTSController | null>(null);
const unblockerAudioRef = useRef<HTMLAudioElement | null>(null);
const ttsControllerRef = useRef<TTSController | null>(null);
const [ttsController, setTtsController] = useState<TTSController | null>(null);
const [ttsClientsInited, setTtsClientsInitialized] = useState(false);
// this enables WebAudio to play even when the mute toggle switch is ON
const unblockAudio = () => {
@@ -95,6 +97,33 @@ const TTSControl = () => {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
useEffect(() => {
if (!ttsController || !bookKey) return;
const bookData = getBookData(bookKey);
if (!bookData || !bookData.book) return;
const { title, author, coverImageUrl } = bookData.book;
const handleSpeakMark = (e: Event) => {
const progress = getProgress(bookKey);
const { sectionLabel } = progress || {};
const mark = (e as CustomEvent<TTSMark>).detail;
if ('mediaSession' in navigator) {
navigator.mediaSession.metadata = new MediaMetadata({
title: mark.text,
artist: sectionLabel || title,
album: author,
artwork: [{ src: coverImageUrl || '/icon.png', sizes: '512x512', type: 'image/png' }],
});
}
};
ttsController.addEventListener('tts-speak-mark', handleSpeakMark);
return () => {
ttsController.removeEventListener('tts-speak-mark', handleSpeakMark);
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [ttsController, bookKey]);
const handleTTSSpeak = async (event: CustomEvent) => {
const { bookKey, range } = event.detail;
const view = getView(bookKey);
@@ -116,6 +145,8 @@ const TTSControl = () => {
ttsControllerRef.current.stop();
ttsControllerRef.current = null;
}
setTTSEnabled(bookKey, true);
setShowIndicator(true);
try {
@@ -125,6 +156,7 @@ const TTSControl = () => {
if (getOSPlatform() === 'ios' || appService?.isIOSApp) {
unblockAudio();
}
setTtsClientsInitialized(false);
const ttsController = new TTSController(view);
await ttsController.init();
await ttsController.initViewTTS();
@@ -142,7 +174,9 @@ const TTSControl = () => {
ttsController.setRate(viewSettings.ttsRate);
ttsController.speak(ssml);
ttsControllerRef.current = ttsController;
setTtsController(ttsController);
}
setTtsClientsInitialized(true);
} catch (error) {
eventDispatcher.dispatch('toast', {
message: _('TTS not supported in this device'),
@@ -152,9 +186,10 @@ const TTSControl = () => {
}
};
const handleTTSStop = async () => {
const handleTTSStop = async (event: CustomEvent) => {
const { bookKey } = event.detail;
if (ttsControllerRef.current) {
handleStop();
handleStop(bookKey);
}
};
@@ -197,11 +232,12 @@ const TTSControl = () => {
}
};
const handleStop = async () => {
const handleStop = async (bookKey: string) => {
const ttsController = ttsControllerRef.current;
if (ttsController) {
await ttsController.stop();
ttsControllerRef.current = null;
setTtsController(null);
getView(bookKey)?.deselect();
setIsPlaying(false);
setShowPanel(false);
@@ -213,6 +249,7 @@ const TTSControl = () => {
if (getOSPlatform() === 'ios' || appService?.isIOSApp) {
releaseUnblockAudio();
}
setTTSEnabled(bookKey, false);
};
// rate range: 0.5 - 3, 1.0 is normal speed
@@ -266,7 +303,7 @@ const TTSControl = () => {
return '';
};
const handleSelectTimeout = (value: number) => {
const handleSelectTimeout = (bookKey: string, value: number) => {
setTimeoutOption(value);
if (timeoutFunc) {
clearTimeout(timeoutFunc);
@@ -274,7 +311,7 @@ const TTSControl = () => {
if (value > 0) {
setTimeoutFunc(
setTimeout(() => {
handleStop();
handleStop(bookKey);
}, value * 1000),
);
setTimeoutTimestamp(Date.now() + value * 1000);
@@ -356,10 +393,10 @@ const TTSControl = () => {
: 'bottom-[70px] sm:bottom-14',
)}
>
<TTSIcon isPlaying={isPlaying} onClick={togglePopup} />
<TTSIcon isPlaying={isPlaying} ttsInited={ttsClientsInited} onClick={togglePopup} />
</div>
)}
{showPanel && panelPosition && trianglePosition && (
{showPanel && panelPosition && trianglePosition && ttsClientsInited && (
<Popup
width={popupWidth}
height={popupHeight}
apps/readest-app/src/app/reader/components/tts/TTSIcon.tsx
+11 -3
View File
@@ -1,20 +1,28 @@
import clsx from 'clsx';
import React from 'react';
type TTSIconProps = {
isPlaying: boolean;
ttsInited: boolean;
onClick: () => void;
};
const TTSIcon: React.FC<TTSIconProps> = ({ isPlaying, onClick }) => {
const TTSIcon: React.FC<TTSIconProps> = ({ isPlaying, ttsInited, onClick }) => {
const bars = [1, 2, 3, 4];
return (
<div className='relative h-full w-full cursor-pointer' onClick={onClick}>
<div
className={clsx(
'relative h-full w-full',
ttsInited ? 'cursor-pointer' : 'cursor-not-allowed',
)}
onClick={onClick}
>
<div className='absolute inset-0 overflow-hidden rounded-full bg-gradient-to-r from-blue-500 via-emerald-500 to-violet-500'>
<div
className='absolute -inset-full bg-gradient-to-r from-blue-500 via-emerald-500 to-violet-500'
style={{
animation: isPlaying ? 'moveGradient 2s alternate infinite' : 'none',
animation: isPlaying && ttsInited ? 'moveGradient 2s alternate infinite' : 'none',
}}
/>
</div>
apps/readest-app/src/app/reader/components/tts/TTSPanel.tsx
+6 -3
View File
@@ -23,7 +23,7 @@ type TTSPanelProps = {
onGetVoices: (lang: string) => Promise<TTSVoice[]>;
onSetVoice: (voice: string, lang: string) => void;
onGetVoiceId: () => string;
onSelectTimeout: (value: number) => void;
onSelectTimeout: (bookKey: string, value: number) => void;
};
const getTTSTimeoutOptions = (_: TranslationFunc) => {
@@ -155,7 +155,7 @@ const TTSPanel = ({
const updateTimeout = (timeout: number) => {
const now = Date.now();
if (timeout > 0 && timeout < now) {
onSelectTimeout(0);
onSelectTimeout(bookKey, 0);
setTimeoutCountdown('');
} else if (timeout > 0) {
setTimeoutCountdown(getCountdownTime(timeout));
@@ -256,7 +256,10 @@ const TTSPanel = ({
)}
>
{timeoutOptions.map((option, index) => (
<li key={`${index}-${option.value}`} onClick={() => onSelectTimeout(option.value)}>
<li
key={`${index}-${option.value}`}
onClick={() => onSelectTimeout(bookKey, option.value)}
>
<div className='flex items-center px-2'>
<span style={{ minWidth: `${defaultIconSize}px` }}>
{timeoutOption === option.value && <MdCheck className='text-base-content' />}
apps/readest-app/src/app/reader/hooks/useTextSelector.ts
+11 -1
View File
@@ -22,6 +22,7 @@ export const useTextSelector = (
const isPopuped = useRef(false);
const isUpToPopup = useRef(false);
const isTextSelected = useRef(false);
const isTouchStarted = useRef(false);
const selectionPosition = useRef<number | null>(null);
const isValidSelection = (sel: Selection) => {
@@ -79,7 +80,8 @@ export const useTextSelector = (
// On Android no proper events are fired to notify selection done,
// we make the popup show when the selection is changed
if (osPlatform === 'android') {
// note that selection may be initiated by a tts speak
if (osPlatform === 'android' && isTouchStarted.current) {
makeSelection(sel, index, false);
}
isUpToPopup.current = true;
@@ -96,6 +98,12 @@ export const useTextSelector = (
}
}
};
const handleTouchStart = () => {
isTouchStarted.current = true;
};
const handleTouchEnd = () => {
isTouchStarted.current = false;
};
const handleScroll = () => {
// Prevent the container from scrolling when text is selected in paginated mode
// This is a workaround for the issue #873
@@ -158,6 +166,8 @@ export const useTextSelector = (
return {
handleScroll,
handleTouchStart,
handleTouchEnd,
handlePointerup,
handleSelectionchange,
handleShowPopup,
apps/readest-app/src/services/tts/EdgeTTSClient.ts
+6 -1
View File
@@ -3,9 +3,11 @@ import { TTSClient, TTSMessageEvent, TTSVoice } from './TTSClient';
import { EdgeSpeechTTS, EdgeTTSPayload } from '@/libs/edgeTTS';
import { parseSSMLLang, parseSSMLMarks } from '@/utils/ssml';
import { TTSGranularity } from '@/types/view';
import { TTSController } from './TTSController';
import { TTSUtils } from './TTSUtils';
export class EdgeTTSClient implements TTSClient {
controller?: TTSController;
#primaryLang = 'en';
#speakingLang = '';
#rate = 1.0;
@@ -20,7 +22,8 @@ export class EdgeTTSClient implements TTSClient {
#startedAt = 0;
available = true;
constructor() {
constructor(controller?: TTSController) {
this.controller = controller;
this.#edgeTTS = new EdgeSpeechTTS();
}
@@ -129,6 +132,8 @@ export class EdgeTTSClient implements TTSClient {
audio.setAttribute('x-webkit-airplay', 'deny');
audio.preload = 'auto';
this.controller?.dispatchSpeakMark(mark);
yield {
code: 'boundary',
message: `Start chunk: ${mark.name}`,
apps/readest-app/src/services/tts/TTSController.ts
+7 -2
View File
@@ -4,6 +4,7 @@ import { parseSSMLMarks } from '@/utils/ssml';
import { WebSpeechClient } from './WebSpeechClient';
import { EdgeTTSClient } from './EdgeTTSClient';
import { TTSUtils } from './TTSUtils';
import { TTSMark } from './types';
type TTSState =
| 'stopped'
@@ -31,8 +32,8 @@ export class TTSController extends EventTarget {
constructor(view: FoliateView) {
super();
this.ttsWebClient = new WebSpeechClient();
this.ttsEdgeClient = new EdgeTTSClient();
this.ttsWebClient = new WebSpeechClient(this);
this.ttsEdgeClient = new EdgeTTSClient(this);
this.ttsClient = this.ttsWebClient;
this.view = view;
}
@@ -273,6 +274,10 @@ export class TTSController extends EventTarget {
return this.ttsClient.getSpeakingLang();
}
dispatchSpeakMark(mark: TTSMark) {
this.dispatchEvent(new CustomEvent('tts-speak-mark', { detail: mark }));
}
error(e: unknown) {
console.error(e);
this.state = 'stopped';
apps/readest-app/src/services/tts/WebSpeechClient.ts
+11 -1
View File
@@ -2,7 +2,9 @@ import { getUserLocale } from '@/utils/misc';
import { TTSClient, TTSMessageEvent, TTSVoice } from './TTSClient';
import { parseSSMLLang, parseSSMLMarks } from '@/utils/ssml';
import { TTSGranularity } from '@/types/view';
import { TTSController } from './TTSController';
import { TTSUtils } from './TTSUtils';
import { TTSMark } from './types';
const BLACKLISTED_VOICES = [
'Albert',
@@ -50,7 +52,8 @@ async function* speakWithMarks(
getRate: () => number,
getPitch: () => number,
getVoice: (lang: string) => Promise<SpeechSynthesisVoice | null>,
setSpeakingLang: (lang: string) => void = () => {},
setSpeakingLang: (lang: string) => void,
dispatchSpeakMark: (mark: TTSMark) => void,
) {
const { marks } = parseSSMLMarks(ssml);
@@ -61,6 +64,7 @@ async function* speakWithMarks(
const { language } = mark;
const voiceLang = language || defaultLang;
setSpeakingLang(voiceLang);
dispatchSpeakMark(mark);
utterance.text = mark.text;
utterance.rate = getRate();
utterance.pitch = getPitch();
@@ -99,6 +103,7 @@ async function* speakWithMarks(
}
export class WebSpeechClient implements TTSClient {
controller?: TTSController;
#primaryLang = 'en';
#speakingLang = '';
#rate = 1.0;
@@ -108,6 +113,10 @@ export class WebSpeechClient implements TTSClient {
#synth = window.speechSynthesis;
available = true;
constructor(controller?: TTSController) {
this.controller = controller;
}
async init() {
if (!this.#synth) {
this.available = false;
@@ -166,6 +175,7 @@ export class WebSpeechClient implements TTSClient {
() => this.#pitch,
this.getVoiceFromLang,
(lang) => (this.#speakingLang = lang),
(mark) => this.controller?.dispatchSpeakMark(mark),
)) {
if (signal.aborted) {
console.log('TTS aborted');
apps/readest-app/src/services/tts/index.ts
+1
View File
@@ -1,3 +1,4 @@
export * from './types';
export * from './TTSClient';
export * from './WebSpeechClient';
export * from './EdgeTTSClient';
apps/readest-app/src/services/tts/types.ts
+6
View File
@@ -0,0 +1,6 @@
export type TTSMark = {
offset: number;
name: string;
text: string;
language?: string;
};
apps/readest-app/src/store/readerStore.ts
+16
View File
@@ -19,6 +19,7 @@ interface ViewState {
error: string | null;
progress: BookProgress | null;
ribbonVisible: boolean;
ttsEnabled: boolean;
/* View settings for the view:
generally view settings have a hierarchy of global settings < book settings < view settings
view settings for primary view are saved to book config which is persisted to config file
@@ -33,6 +34,7 @@ interface ReaderStore {
setBookKeys: (keys: string[]) => void;
setHoveredBookKey: (key: string | null) => void;
setBookmarkRibbonVisibility: (key: string, visible: boolean) => void;
setTTSEnabled: (key: string, enabled: boolean) => void;
setProgress: (
key: string,
location: string,
@@ -103,6 +105,7 @@ export const useReaderStore = create<ReaderStore>((set, get) => ({
error: null,
progress: null,
ribbonVisible: false,
ttsEnabled: false,
viewSettings: null,
},
},
@@ -149,6 +152,7 @@ export const useReaderStore = create<ReaderStore>((set, get) => ({
error: null,
progress: null,
ribbonVisible: false,
ttsEnabled: false,
viewSettings: JSON.parse(JSON.stringify(configViewSettings)) as ViewSettings,
},
},
@@ -167,6 +171,7 @@ export const useReaderStore = create<ReaderStore>((set, get) => ({
error: 'Failed to load book.',
progress: null,
ribbonVisible: false,
ttsEnabled: false,
viewSettings: null,
},
},
@@ -282,4 +287,15 @@ export const useReaderStore = create<ReaderStore>((set, get) => ({
},
},
})),
setTTSEnabled: (key: string, enabled: boolean) =>
set((state) => ({
viewStates: {
...state.viewStates,
[key]: {
...state.viewStates[key]!,
ttsEnabled: enabled,
},
},
})),
}));
apps/readest-app/src/utils/ssml.ts
+1 -6
View File
@@ -1,9 +1,4 @@
type TTSMark = {
offset: number;
name: string;
text: string;
language?: string;
};
import { TTSMark } from '@/services/tts/types';
const cleanTextContent = (text: string) =>
text.replace(/\r\n/g, ' ').replace(/\r/g, ' ').replace(/\n/g, ' ').trimStart();