feat(reader): add TTS highlight granularity setting (word or sentence) (#4807)

This commit is contained in:
Huang Xin
2026-06-26 18:48:57 +08:00
committed by GitHub
parent dced42912f
commit 4874eb9ae7
42 changed files with 236 additions and 37 deletions
@@ -1935,5 +1935,8 @@
"Sort descending": "ترتيب تنازلي",
"Filter Annotations": "تصفية التعليقات التوضيحية",
"Colors": "الألوان",
"Styles": "الأنماط"
"Styles": "الأنماط",
"Word": "كلمة",
"Sentence": "جملة",
"Granularity": "الدقة"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "অধঃক্রমে সাজান",
"Filter Annotations": "টীকা ফিল্টার করুন",
"Colors": "রঙ",
"Styles": "শৈলী"
"Styles": "শৈলী",
"Word": "শব্দ",
"Sentence": "বাক্য",
"Granularity": "সূক্ষ্মতা"
}
@@ -1764,5 +1764,8 @@
"Metadata updated": "གནད་སྨིན་གོ་དོན་གསར་བསྒྱུར་བྱས།",
"Filter Annotations": "མཆན་འདེམས་སྒྲིག",
"Colors": "ཁ་དོག",
"Styles": "རྣམ་པ"
"Styles": "རྣམ་པ",
"Word": "ཚིག",
"Sentence": "ཚིག་གྲུབ།",
"Granularity": "ཞིབ་ཕྲའི་ཚད"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "Absteigend sortieren",
"Filter Annotations": "Anmerkungen filtern",
"Colors": "Farben",
"Styles": "Stile"
"Styles": "Stile",
"Word": "Wort",
"Sentence": "Satz",
"Granularity": "Granularität"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "Φθίνουσα ταξινόμηση",
"Filter Annotations": "Φιλτράρισμα σχολιασμών",
"Colors": "Χρώματα",
"Styles": "Στυλ"
"Styles": "Στυλ",
"Word": "Λέξη",
"Sentence": "Πρόταση",
"Granularity": "Διαβάθμιση"
}
@@ -1836,5 +1836,8 @@
"Sort descending": "Orden descendente",
"Filter Annotations": "Filtrar anotaciones",
"Colors": "Colores",
"Styles": "Estilos"
"Styles": "Estilos",
"Word": "Palabra",
"Sentence": "Oración",
"Granularity": "Granularidad"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "مرتب‌سازی نزولی",
"Filter Annotations": "فیلتر یادداشت‌ها",
"Colors": "رنگ‌ها",
"Styles": "سبک‌ها"
"Styles": "سبک‌ها",
"Word": "کلمه",
"Sentence": "جمله",
"Granularity": "دانه‌بندی"
}
@@ -1836,5 +1836,8 @@
"Sort descending": "Tri décroissant",
"Filter Annotations": "Filtrer les annotations",
"Colors": "Couleurs",
"Styles": "Styles"
"Styles": "Styles",
"Word": "Mot",
"Sentence": "Phrase",
"Granularity": "Granularité"
}
@@ -1836,5 +1836,8 @@
"Sort descending": "מיון יורד",
"Filter Annotations": "סינון הערות",
"Colors": "צבעים",
"Styles": "סגנונות"
"Styles": "סגנונות",
"Word": "מילה",
"Sentence": "משפט",
"Granularity": "רמת פירוט"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "अवरोही क्रम",
"Filter Annotations": "टिप्पणियाँ फ़िल्टर करें",
"Colors": "रंग",
"Styles": "शैलियाँ"
"Styles": "शैलियाँ",
"Word": "शब्द",
"Sentence": "वाक्य",
"Granularity": "सूक्ष्मता"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "Csökkenő sorrend",
"Filter Annotations": "Jegyzetek szűrése",
"Colors": "Színek",
"Styles": "Stílusok"
"Styles": "Stílusok",
"Word": "Szó",
"Sentence": "Mondat",
"Granularity": "Részletesség"
}
@@ -1770,5 +1770,8 @@
"Sort descending": "Urutkan menurun",
"Filter Annotations": "Filter Anotasi",
"Colors": "Warna",
"Styles": "Gaya"
"Styles": "Gaya",
"Word": "Kata",
"Sentence": "Kalimat",
"Granularity": "Granularitas"
}
@@ -1836,5 +1836,8 @@
"Sort descending": "Ordine decrescente",
"Filter Annotations": "Filtra annotazioni",
"Colors": "Colori",
"Styles": "Stili"
"Styles": "Stili",
"Word": "Parola",
"Sentence": "Frase",
"Granularity": "Granularità"
}
@@ -1770,5 +1770,8 @@
"Sort descending": "降順で並べ替え",
"Filter Annotations": "注釈をフィルター",
"Colors": "色",
"Styles": "スタイル"
"Styles": "スタイル",
"Word": "単語",
"Sentence": "文",
"Granularity": "粒度"
}
@@ -1770,5 +1770,8 @@
"Sort descending": "내림차순 정렬",
"Filter Annotations": "주석 필터링",
"Colors": "색상",
"Styles": "스타일"
"Styles": "스타일",
"Word": "단어",
"Sentence": "문장",
"Granularity": "단위"
}
@@ -1770,5 +1770,8 @@
"Sort descending": "Isih menurun",
"Filter Annotations": "Tapis Anotasi",
"Colors": "Warna",
"Styles": "Gaya"
"Styles": "Gaya",
"Word": "Perkataan",
"Sentence": "Ayat",
"Granularity": "Granulariti"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "Aflopend sorteren",
"Filter Annotations": "Aantekeningen filteren",
"Colors": "Kleuren",
"Styles": "Stijlen"
"Styles": "Stijlen",
"Word": "Woord",
"Sentence": "Zin",
"Granularity": "Granulariteit"
}
@@ -1869,5 +1869,8 @@
"Sort descending": "Sortuj malejąco",
"Filter Annotations": "Filtruj adnotacje",
"Colors": "Kolory",
"Styles": "Style"
"Styles": "Style",
"Word": "Słowo",
"Sentence": "Zdanie",
"Granularity": "Ziarnistość"
}
@@ -1836,5 +1836,8 @@
"Sort descending": "Ordem decrescente",
"Filter Annotations": "Filtrar anotações",
"Colors": "Cores",
"Styles": "Estilos"
"Styles": "Estilos",
"Word": "Palavra",
"Sentence": "Frase",
"Granularity": "Granularidade"
}
@@ -1836,5 +1836,8 @@
"Sort descending": "Ordem decrescente",
"Filter Annotations": "Filtrar anotações",
"Colors": "Cores",
"Styles": "Estilos"
"Styles": "Estilos",
"Word": "Palavra",
"Sentence": "Frase",
"Granularity": "Granularidade"
}
@@ -1836,5 +1836,8 @@
"Sort descending": "Sortare descrescătoare",
"Filter Annotations": "Filtrează adnotările",
"Colors": "Culori",
"Styles": "Stiluri"
"Styles": "Stiluri",
"Word": "Cuvânt",
"Sentence": "Propoziție",
"Granularity": "Granularitate"
}
@@ -1869,5 +1869,8 @@
"Sort descending": "По убыванию",
"Filter Annotations": "Фильтровать аннотации",
"Colors": "Цвета",
"Styles": "Стили"
"Styles": "Стили",
"Word": "Слово",
"Sentence": "Предложение",
"Granularity": "Детализация"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "අවරෝහණව සකසන්න",
"Filter Annotations": "සටහන් පෙරහන් කරන්න",
"Colors": "වර්ණ",
"Styles": "මෝස්තර"
"Styles": "මෝස්තර",
"Word": "වචනය",
"Sentence": "වාක්‍යය",
"Granularity": "විස්තර මට්ටම"
}
@@ -1869,5 +1869,8 @@
"Sort descending": "Padajoče razvrščanje",
"Filter Annotations": "Filtriraj opombe",
"Colors": "Barve",
"Styles": "Slogi"
"Styles": "Slogi",
"Word": "Beseda",
"Sentence": "Stavek",
"Granularity": "Granularnost"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "Fallande sortering",
"Filter Annotations": "Filtrera anteckningar",
"Colors": "Färger",
"Styles": "Stilar"
"Styles": "Stilar",
"Word": "Ord",
"Sentence": "Mening",
"Granularity": "Detaljnivå"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "இறங்குவரிசை",
"Filter Annotations": "சிறுகுறிப்புகளை வடிகட்டு",
"Colors": "நிறங்கள்",
"Styles": "பாணிகள்"
"Styles": "பாணிகள்",
"Word": "சொல்",
"Sentence": "வாக்கியம்",
"Granularity": "நுண்மை"
}
@@ -1770,5 +1770,8 @@
"Sort descending": "เรียงจากมากไปน้อย",
"Filter Annotations": "กรองคำอธิบายประกอบ",
"Colors": "สี",
"Styles": "สไตล์"
"Styles": "สไตล์",
"Word": "คำ",
"Sentence": "ประโยค",
"Granularity": "ระดับความละเอียด"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "Azalan sıralama",
"Filter Annotations": "Açıklamaları Filtrele",
"Colors": "Renkler",
"Styles": "Stiller"
"Styles": "Stiller",
"Word": "Kelime",
"Sentence": "Cümle",
"Granularity": "Ayrıntı düzeyi"
}
@@ -1869,5 +1869,8 @@
"Sort descending": "За спаданням",
"Filter Annotations": "Фільтрувати анотації",
"Colors": "Кольори",
"Styles": "Стилі"
"Styles": "Стилі",
"Word": "Слово",
"Sentence": "Речення",
"Granularity": "Деталізація"
}
@@ -1803,5 +1803,8 @@
"Sort descending": "Kamayish boʻyicha",
"Filter Annotations": "Izohlarni filtrlash",
"Colors": "Ranglar",
"Styles": "Uslublar"
"Styles": "Uslublar",
"Word": "Soz",
"Sentence": "Gap",
"Granularity": "Tafsilot darajasi"
}
@@ -1770,5 +1770,8 @@
"Sort descending": "Sắp xếp giảm dần",
"Filter Annotations": "Lọc chú thích",
"Colors": "Màu sắc",
"Styles": "Kiểu"
"Styles": "Kiểu",
"Word": "Từ",
"Sentence": "Câu",
"Granularity": "Mức độ chi tiết"
}
@@ -1770,5 +1770,8 @@
"Sort descending": "降序排列",
"Filter Annotations": "筛选笔记",
"Colors": "颜色",
"Styles": "样式"
"Styles": "样式",
"Word": "单词",
"Sentence": "句子",
"Granularity": "粒度"
}
@@ -1770,5 +1770,8 @@
"Sort descending": "降序排列",
"Filter Annotations": "篩選筆記",
"Colors": "顏色",
"Styles": "樣式"
"Styles": "樣式",
"Word": "單字",
"Sentence": "句子",
"Granularity": "粒度"
}
@@ -133,6 +133,7 @@ vi.mock('@/services/tts', () => ({
),
initViewTTS: vi.fn().mockResolvedValue(undefined),
updateHighlightOptions: vi.fn(),
setHighlightGranularity: vi.fn(),
setLang: vi.fn(),
setRate: vi.fn(),
setVoice: vi.fn(),
@@ -744,6 +744,29 @@ describe('TTSController', () => {
controller.prepareSpeakWords([]);
expect(controller.getCurrentHighlightCfi()).toBeNull();
});
test('granularity "sentence" forces sentence highlighting (skips word-by-word)', async () => {
controller.setHighlightGranularity('sentence');
await armWithSentence(makeSentenceRange());
getOverlayer().add.mockClear();
// Even when the client reports word boundaries and calls prepareSpeakWords,
// the user's "sentence" choice keeps highlighting at the sentence level: the
// sentence highlight was already drawn at mark dispatch (not suppressed), so
// prepareSpeakWords is a no-op and word mode never engages.
controller.prepareSpeakWords(['Hello', 'brave', 'world']);
expect(getOverlayer().add).not.toHaveBeenCalled();
expect(controller.getCurrentHighlightCfi()).toBeNull();
});
test('granularity "word" (default) still highlights word-by-word', async () => {
controller.setHighlightGranularity('word');
await armWithSentence(makeSentenceRange());
getOverlayer().add.mockClear();
controller.prepareSpeakWords(['Hello', 'brave', 'world']);
// First word highlighted immediately and word mode engaged.
expect(getOverlayer().add).toHaveBeenCalledTimes(1);
expect(controller.getCurrentHighlightCfi()).not.toBeNull();
});
});
describe('tts-position event', () => {
@@ -531,6 +531,12 @@ export const useTTSControl = ({ bookKey, onRequestHidePanel }: UseTTSControlProp
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [viewSettings?.ttsHighlightOptions, viewSettings?.isEink, getTTSHighlightOptions]);
useEffect(() => {
if (ttsControllerRef.current && viewSettings?.ttsHighlightGranularity) {
ttsControllerRef.current.setHighlightGranularity(viewSettings.ttsHighlightGranularity);
}
}, [viewSettings?.ttsHighlightGranularity]);
// handleStop (defined before handleTTSSpeak/handleTTSStop which reference it)
const handleStop = useCallback(
async (bookKey: string) => {
@@ -659,6 +665,7 @@ export const useTTSControl = ({ bookKey, onRequestHidePanel }: UseTTSControlProp
ttsController.updateHighlightOptions(
getTTSHighlightOptions(viewSettings.ttsHighlightOptions, viewSettings.isEink),
);
ttsController.setHighlightGranularity(viewSettings.ttsHighlightGranularity ?? 'word');
const ssml =
oneTime && ttsSpeakRange
? genSSMLRaw(ttsSpeakRange.toString().trim())
@@ -6,7 +6,7 @@ import { useResetViewSettings } from '@/hooks/useResetSettings';
import { useTranslation } from '@/hooks/useTranslation';
import { saveViewSettings } from '@/helpers/settings';
import { SettingsPanelPanelProp } from './SettingsDialog';
import { TTSMediaMetadataMode } from '@/services/tts/types';
import { TTSHighlightGranularity, TTSMediaMetadataMode } from '@/services/tts/types';
import { BoxedList, SettingsRow, SettingsSelect } from './primitives';
import TTSHighlightStyleEditor, { TTSHighlightStyle } from './color/TTSHighlightStyleEditor';
@@ -20,6 +20,9 @@ const TTSPanel: React.FC<SettingsPanelPanelProp> = ({ bookKey, onRegisterReset }
const [ttsMediaMetadata, setTtsMediaMetadata] = useState<TTSMediaMetadataMode>(
viewSettings.ttsMediaMetadata ?? 'sentence',
);
const [ttsHighlightGranularity, setTtsHighlightGranularity] = useState<TTSHighlightGranularity>(
viewSettings.ttsHighlightGranularity ?? 'word',
);
const [ttsHighlightStyle, setTtsHighlightStyle] = useState(
viewSettings.ttsHighlightOptions.style,
);
@@ -35,6 +38,9 @@ const TTSPanel: React.FC<SettingsPanelPanelProp> = ({ bookKey, onRegisterReset }
const handleReset = () => {
resetToDefaults({
ttsMediaMetadata: setTtsMediaMetadata as React.Dispatch<React.SetStateAction<string>>,
ttsHighlightGranularity: setTtsHighlightGranularity as React.Dispatch<
React.SetStateAction<string>
>,
});
};
@@ -49,6 +55,19 @@ const TTSPanel: React.FC<SettingsPanelPanelProp> = ({ bookKey, onRegisterReset }
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [ttsMediaMetadata]);
useEffect(() => {
if (ttsHighlightGranularity === viewSettings.ttsHighlightGranularity) return;
saveViewSettings(
envConfig,
bookKey,
'ttsHighlightGranularity',
ttsHighlightGranularity,
false,
false,
);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [ttsHighlightGranularity]);
const handleTTSStyleChange = (style: TTSHighlightStyle) => {
setTtsHighlightStyle(style);
saveViewSettings(envConfig, bookKey, 'ttsHighlightOptions', {
@@ -76,12 +95,18 @@ const TTSPanel: React.FC<SettingsPanelPanelProp> = ({ bookKey, onRegisterReset }
setTtsMediaMetadata(event.target.value as TTSMediaMetadataMode);
};
const handleTTSGranularityChange = (granularity: TTSHighlightGranularity) => {
setTtsHighlightGranularity(granularity);
};
return (
<div className='my-4 w-full space-y-6'>
<TTSHighlightStyleEditor
granularity={ttsHighlightGranularity}
style={ttsHighlightStyle}
color={ttsHighlightColor}
customColors={customTtsHighlightColors}
onGranularityChange={handleTTSGranularityChange}
onStyleChange={handleTTSStyleChange}
onColorChange={handleTTSColorChange}
onCustomColorsChange={handleCustomTtsColorsChange}
@@ -1,6 +1,7 @@
import React from 'react';
import { MdClose } from 'react-icons/md';
import { useTranslation } from '@/hooks/useTranslation';
import { TTSHighlightGranularity } from '@/services/tts/types';
import { BoxedList, SettingsRow, SettingsSelect } from '../primitives';
import ColorInput from './ColorInput';
@@ -12,18 +13,22 @@ export type TTSHighlightStyle =
| 'outline';
interface TTSHighlightStyleEditorProps {
granularity: TTSHighlightGranularity;
style: TTSHighlightStyle;
color: string;
customColors: string[];
onGranularityChange: (granularity: TTSHighlightGranularity) => void;
onStyleChange: (style: TTSHighlightStyle) => void;
onColorChange: (color: string) => void;
onCustomColorsChange: (colors: string[]) => void;
}
const TTSHighlightStyleEditor: React.FC<TTSHighlightStyleEditorProps> = ({
granularity,
style,
color,
customColors,
onGranularityChange,
onStyleChange,
onColorChange,
onCustomColorsChange,
@@ -60,6 +65,18 @@ const TTSHighlightStyleEditor: React.FC<TTSHighlightStyleEditorProps> = ({
return (
<BoxedList title={_('TTS Highlighting')}>
<SettingsRow label={_('Granularity')}>
<SettingsSelect
value={granularity}
onChange={(e) => onGranularityChange(e.target.value as TTSHighlightGranularity)}
ariaLabel={_('Granularity')}
options={[
{ value: 'word', label: _('Word') },
{ value: 'sentence', label: _('Sentence') },
]}
/>
</SettingsRow>
<SettingsRow label={_('Style')}>
<SettingsSelect
value={style}
@@ -372,6 +372,7 @@ export const DEFAULT_TTS_CONFIG: TTSConfig = {
ttsLocation: '',
showTTSBar: false,
ttsHighlightOptions: { style: 'highlight', color: '#808080' },
ttsHighlightGranularity: 'word',
ttsMediaMetadata: 'sentence',
};
@@ -2,7 +2,13 @@ import { FoliateView } from '@/types/view';
import { AppService } from '@/types/system';
import { filterSSMLWithLang, parseSSMLMarks } from '@/utils/ssml';
import { Overlayer } from 'foliate-js/overlayer.js';
import { TTSGranularity, TTSHighlightOptions, TTSMark, TTSVoice } from './types';
import {
TTSGranularity,
TTSHighlightGranularity,
TTSHighlightOptions,
TTSMark,
TTSVoice,
} from './types';
import { createRejectFilter } from '@/utils/node';
import { WebSpeechClient } from './WebSpeechClient';
import { NativeTTSClient } from './NativeTTSClient';
@@ -78,6 +84,11 @@ export class TTSController extends EventTarget {
// re-apply the word instead of redrawing the whole sentence over it.
#wordHighlightActive = false;
#lastSpeakWordRange: Range | null = null;
// User-chosen highlight granularity. 'word' (default) highlights word-by-word
// when the active client reports word boundaries (Edge); 'sentence' keeps the
// highlight at the sentence level even then. Sentence highlighting is assumed
// supported by every client, so 'word' falls back to it automatically.
#highlightGranularity: TTSHighlightGranularity = 'word';
state: TTSState = 'stopped';
ttsLang: string = '';
@@ -190,6 +201,10 @@ export class TTSController extends EventTarget {
this.options.color = options.color;
}
setHighlightGranularity(granularity: TTSHighlightGranularity) {
this.#highlightGranularity = granularity;
}
async initViewTTS(index?: number) {
if (this.#ttsSectionIndex === -1) {
const fromSectionIndex = (index || this.#getPrimaryContent()?.index) ?? 0;
@@ -677,8 +692,11 @@ export class TTSController extends EventTarget {
// When the active client highlights word-by-word, suppress the
// sentence highlight that setMark would otherwise draw, so the page
// doesn't flash the whole sentence before the first word. The fallback
// (no boundaries) is drawn later in prepareSpeakWords.
this.#suppressMarkHighlight = this.ttsClient.supportsWordBoundaries();
// (no boundaries) is drawn later in prepareSpeakWords. When the user
// forces sentence granularity we keep the sentence highlight, so don't
// suppress it.
this.#suppressMarkHighlight =
this.ttsClient.supportsWordBoundaries() && this.#highlightGranularity === 'word';
const range = this.view.tts?.setMark(mark.name);
this.#suppressMarkHighlight = false;
this.#speakWordsArmed = !!range;
@@ -760,6 +778,10 @@ export class TTSController extends EventTarget {
// sentence-level semantics.
prepareSpeakWords(words: string[]) {
if (!this.#speakWordsArmed) return;
// User forced sentence-level highlighting: the sentence highlight was drawn
// at mark dispatch (not suppressed), so there's nothing to do here — leave
// word mode off even though the client reported word boundaries.
if (this.#highlightGranularity === 'sentence') return;
const range = this.view.tts?.getLastRange();
if (!range) return;
this.#speakWordBaseRange = range;
@@ -1,5 +1,7 @@
export type TTSGranularity = 'sentence' | 'word';
export type TTSHighlightGranularity = 'word' | 'sentence';
export type TTSMediaMetadataMode = 'sentence' | 'paragraph' | 'chapter';
export type TTSHighlightOptions = {
+2
View File
@@ -1,5 +1,6 @@
import { BookMetadata } from '@/libs/document';
import { TTSHighlightOptions } from '@/services/tts/types';
import { TTSHighlightGranularity } from '@/services/tts/types';
import { TTSMediaMetadataMode } from '@/services/tts/types';
import type { AnnotationLinkType } from '@/utils/deeplink';
import { AnnotationToolType } from './annotator';
@@ -314,6 +315,7 @@ export interface TTSConfig {
ttsLocation: string;
showTTSBar: boolean;
ttsHighlightOptions: TTSHighlightOptions;
ttsHighlightGranularity: TTSHighlightGranularity;
ttsMediaMetadata: TTSMediaMetadataMode;
}