Initial implementation of TTS with Web Speech API (#120)

* Initial implementation of TTS with Web Speech API * Add translations for annotator tools
2025-01-07 21:57:55 +01:00
parent a59a3b633b
commit 07b04b82e6
27 changed files with 748 additions and 13 deletions
@@ -2,6 +2,7 @@ import React from 'react';

 import { useSettingsStore } from '@/store/settingsStore';
 import { useReaderStore } from '@/store/readerStore';
+import { useBookDataStore } from '@/store/bookDataStore';
 import FoliateViewer from './FoliateViewer';
 import getGridTemplate from '@/utils/grid';
 import SectionInfo from './SectionInfo';
@@ -11,7 +12,6 @@ import PageInfoView from './PageInfo';
 import Ribbon from './Ribbon';
 import SettingsDialog from './settings/SettingsDialog';
 import Annotator from './annotator/Annotator';
-import { useBookDataStore } from '@/store/bookDataStore';
 import FootnotePopup from './FootnotePopup';

 interface BooksGridProps {
@@ -49,7 +49,7 @@ const BooksGrid: React.FC<BooksGridProps> = ({ bookKeys, onCloseBook }) => {
          <div
            id={`gridcell-${bookKey}`}
            key={bookKey}
-            className='relative h-full w-full rounded-window overflow-hidden'
+            className='rounded-window relative h-full w-full overflow-hidden'
          >
            {isBookmarked && <Ribbon width={marginGap} />}
            <HeaderBar
@@ -26,6 +26,7 @@ import Spinner from '@/components/Spinner';
 import SideBar from './sidebar/SideBar';
 import Notebook from './notebook/Notebook';
 import BooksGrid from './BooksGrid';
+import TTSControl from './tts/TTSControl';

 const ReaderContent: React.FC<{ ids?: string; settings: SystemSettings }> = ({ ids, settings }) => {
  const router = useRouter();
@@ -154,6 +155,7 @@ const ReaderContent: React.FC<{ ids?: string; settings: SystemSettings }> = ({ i
    <div className='flex h-screen'>
      <SideBar onGoToLibrary={handleCloseBooksToLibrary} />
      <BooksGrid bookKeys={bookKeys} onCloseBook={handleCloseBook} />
+      <TTSControl />
      <Notebook />
      {showDetailsBook && (
        <BookDetailModal
@@ -6,7 +6,8 @@ import { FaWikipediaW } from 'react-icons/fa';
 import { BsPencilSquare } from 'react-icons/bs';
 import { RiDeleteBinLine } from 'react-icons/ri';
 import { BsTranslate } from 'react-icons/bs';
-import { SiDeepl } from 'react-icons/si';
+import { TbHexagonLetterD } from 'react-icons/tb';
+import { FaHeadphones } from 'react-icons/fa6';

 import * as CFI from 'foliate-js/epubcfi.js';
 import { Overlayer } from 'foliate-js/overlayer.js';
@@ -17,6 +18,7 @@ import { useBookDataStore } from '@/store/bookDataStore';
 import { useSettingsStore } from '@/store/settingsStore';
 import { useReaderStore } from '@/store/readerStore';
 import { useNotebookStore } from '@/store/notebookStore';
+import { useTranslation } from '@/hooks/useTranslation';
 import { useFoliateEvents } from '../../hooks/useFoliateEvents';
 import { useNotesSync } from '../../hooks/useNotesSync';
 import { getPopupPosition, getPosition, Position, TextSelection } from '@/utils/sel';
@@ -28,6 +30,7 @@ import WikipediaPopup from './WikipediaPopup';
 import DeepLPopup from './DeepLPopup';

 const Annotator: React.FC<{ bookKey: string }> = ({ bookKey }) => {
+  const _ = useTranslation();
  const { envConfig } = useEnv();
  const { settings } = useSettingsStore();
  const { getConfig, saveConfig, getBookData, updateBooknotes } = useBookDataStore();
@@ -222,7 +225,7 @@ const Annotator: React.FC<{ bookKey: string }> = ({ bookKey }) => {

  const handleCopy = () => {
    if (!selection || !selection.text) return;
-    setToastMessage('Copied to notebook');
+    setToastMessage(_('Copied to notebook'));

    const { booknotes: annotations = [] } = config;
    if (selection) navigator.clipboard.writeText(selection.text);
@@ -334,19 +337,29 @@ const Annotator: React.FC<{ bookKey: string }> = ({ bookKey }) => {
    setShowDeepLPopup(true);
  };

+  const handleSpeakText = async () => {
+    if (!selection || !selection.text) return;
+    setShowAnnotPopup(false);
+
+    await view!.initTTS();
+    const ssml = view!.tts.from(selection.range)!;
+    eventDispatcher.dispatch('speak', { bookKey, ssml });
+  };
+
  const selectionAnnotated = selection?.annotated;
  const buttons = [
-    { tooltipText: 'Copy', Icon: FiCopy, onClick: handleCopy },
+    { tooltipText: _('Copy'), Icon: FiCopy, onClick: handleCopy },
    {
-      tooltipText: selectionAnnotated ? 'Delete Highlight' : 'Highlight',
+      tooltipText: selectionAnnotated ? _('Delete Highlight') : _('Highlight'),
      Icon: selectionAnnotated ? RiDeleteBinLine : PiHighlighterFill,
      onClick: handleHighlight,
    },
-    { tooltipText: 'Annotate', Icon: BsPencilSquare, onClick: handleAnnotate },
-    { tooltipText: 'Search', Icon: FiSearch, onClick: handleSearch },
-    { tooltipText: 'Dictionary', Icon: BsTranslate, onClick: handleDictionary },
-    { tooltipText: 'Translate', Icon: SiDeepl, onClick: handleTranslation },
-    { tooltipText: 'Wikipedia', Icon: FaWikipediaW, onClick: handleWikipedia },
+    { tooltipText: _('Annotate'), Icon: BsPencilSquare, onClick: handleAnnotate },
+    { tooltipText: _('Search'), Icon: FiSearch, onClick: handleSearch },
+    { tooltipText: _('Dictionary'), Icon: TbHexagonLetterD, onClick: handleDictionary },
+    { tooltipText: _('Translate'), Icon: BsTranslate, onClick: handleTranslation },
+    { tooltipText: _('Wikipedia'), Icon: FaWikipediaW, onClick: handleWikipedia },
+    { tooltipText: _('Speak'), Icon: FaHeadphones, onClick: handleSpeakText },
  ];

  return (
@@ -51,7 +51,8 @@ const HighlightOptions: React.FC<HighlightOptionsProps> = ({
          >
            <div
              className={clsx(
-                'h-4 w-4 p-0 text-center leading-none',
+                'w-4 p-0 text-center leading-none',
+                style === 'highlight' ? 'h-4' : 'h-5',
                style === 'highlight' &&
                  (selectedStyle === 'highlight' ? `bg-${selectedColor}-400` : `bg-gray-300`),
                (style === 'underline' || style === 'squiggly') &&
@@ -0,0 +1,147 @@
+import React, { useState, useRef, useEffect } from 'react';
+import { TTSController } from '@/services/tts/TTSController';
+import { WebSpeechClient } from '@/services/tts';
+import { getPopupPosition, Position } from '@/utils/sel';
+import { eventDispatcher } from '@/utils/event';
+import { useReaderStore } from '@/store/readerStore';
+import Popup from '@/components/Popup';
+import TTSPanel from './TTSPanel';
+import TTSIcon from './TTSIcon';
+
+const POPUP_WIDTH = 260;
+const POPUP_HEIGHT = 80;
+const POPUP_PADDING = 10;
+
+const TTSControl = () => {
+  const { getView } = useReaderStore();
+  const [isPlaying, setIsPlaying] = useState(false);
+  const [showPanel, setShowPanel] = useState(false);
+  const [panelPosition, setPanelPosition] = useState<Position>();
+  const [trianglePosition, setTrianglePosition] = useState<Position>();
+
+  const iconRef = useRef<HTMLDivElement>(null);
+  const ttsControllerRef = useRef<TTSController | null>(null);
+
+  useEffect(() => {
+    return () => {
+      if (ttsControllerRef.current) {
+        ttsControllerRef.current.kill();
+        ttsControllerRef.current = null;
+      }
+    };
+  }, []);
+
+  useEffect(() => {
+    eventDispatcher.on('speak', handleSpeak);
+    return () => {
+      eventDispatcher.off('speak', handleSpeak);
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  const handleSpeak = async (event: CustomEvent) => {
+    const { bookKey, ssml } = event.detail;
+    const view = getView(bookKey);
+    if (!view) return;
+
+    const ttsClient = new WebSpeechClient();
+    const ttsController = new TTSController(ttsClient, view);
+    ttsControllerRef.current = ttsController;
+    ttsControllerRef.current.speak(ssml);
+    setIsPlaying(true);
+  };
+
+  const handleTogglePlay = async () => {
+    const ttsController = ttsControllerRef.current;
+    if (!ttsController) return;
+
+    if (isPlaying) {
+      ttsController.pause();
+      setIsPlaying(false);
+    } else {
+      await ttsController.start();
+      setIsPlaying(true);
+    }
+  };
+
+  const handleBackward = async () => {
+    const ttsController = ttsControllerRef.current;
+    if (ttsController) {
+      await ttsController.backward();
+    }
+  };
+
+  const handleForward = async () => {
+    const ttsController = ttsControllerRef.current;
+    if (ttsController) {
+      await ttsController.forward();
+    }
+  };
+
+  const handleStop = async () => {
+    const ttsController = ttsControllerRef.current;
+    if (ttsController) {
+      ttsController.stop();
+      ttsControllerRef.current = null;
+      setIsPlaying(false);
+      setShowPanel(false);
+    }
+  };
+
+  const updatePanelPosition = () => {
+    if (iconRef.current) {
+      const rect = iconRef.current.getBoundingClientRect();
+      const windowRect = document.documentElement.getBoundingClientRect();
+
+      const trianglePos = {
+        dir: 'up',
+        point: { x: rect.left + rect.width / 2, y: rect.top - 12 },
+      } as Position;
+
+      const popupPos = getPopupPosition(
+        trianglePos,
+        windowRect,
+        POPUP_WIDTH,
+        POPUP_HEIGHT,
+        POPUP_PADDING,
+      );
+
+      setPanelPosition(popupPos);
+      setTrianglePosition(trianglePos);
+    }
+  };
+
+  const togglePopup = () => {
+    updatePanelPosition();
+    setShowPanel((prev) => !prev);
+  };
+
+  return (
+    <div>
+      {ttsControllerRef.current && (
+        <div ref={iconRef} className='absolute bottom-12 right-6 h-12 w-12'>
+          <TTSIcon isPlaying={isPlaying} onClick={togglePopup} />
+        </div>
+      )}
+      {showPanel && panelPosition && trianglePosition && (
+        <Popup
+          width={POPUP_WIDTH}
+          height={POPUP_HEIGHT}
+          position={panelPosition}
+          trianglePosition={trianglePosition}
+          className='bg-base-200 absolute flex shadow-lg'
+        >
+          <TTSPanel
+            isPlaying={isPlaying}
+            onTogglePlay={handleTogglePlay}
+            onBackward={handleBackward}
+            onForward={handleForward}
+            onStop={handleStop}
+          />
+        </Popup>
+      )}
+    </div>
+  );
+};
+
+export default TTSControl;
@@ -0,0 +1,54 @@
+import React from 'react';
+
+type TTSIconProps = {
+  isPlaying: boolean;
+  onClick: () => void;
+};
+
+const TTSIcon: React.FC<TTSIconProps> = ({ isPlaying, onClick }) => {
+  const bars = [1, 2, 3, 4];
+
+  return (
+    <div className='relative h-full w-full cursor-pointer' onClick={onClick}>
+      <div className='absolute inset-0 overflow-hidden rounded-full bg-gradient-to-r from-blue-500 via-emerald-500 to-violet-500'>
+        <div
+          className='absolute -inset-full bg-gradient-to-r from-blue-500 via-emerald-500 to-violet-500'
+          style={{
+            animation: isPlaying ? 'moveGradient 2s alternate infinite' : 'none',
+          }}
+        />
+      </div>
+
+      <div className='absolute inset-0 flex items-center justify-center'>
+        <style>{`
+          @keyframes moveGradient {
+            0% { transform: translate(0, 0); }
+            100% { transform: translate(25%, 25%); }
+          }
+          @keyframes bounce {
+            0%, 100% { transform: scaleY(1); }
+            50% { transform: scaleY(0.6); }
+          }
+        `}</style>
+        <div className='flex items-end space-x-1'>
+          {bars.map((bar) => (
+            <div
+              key={bar}
+              className='w-1 rounded-t bg-white'
+              style={{
+                height: '16px',
+                animationName: isPlaying ? 'bounce' : 'none',
+                animationDuration: isPlaying ? `${1 + bar * 0.1}s` : '0s',
+                animationTimingFunction: 'ease-in-out',
+                animationIterationCount: 'infinite',
+                animationDelay: `${bar * 0.1}s`,
+              }}
+            />
+          ))}
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default TTSIcon;
@@ -0,0 +1,37 @@
+import React from 'react';
+import { MdPlayCircle, MdPauseCircle, MdFastRewind, MdFastForward, MdStop } from 'react-icons/md';
+
+type TTSPanelProps = {
+  isPlaying: boolean;
+  onTogglePlay: () => void;
+  onBackward: () => void;
+  onForward: () => void;
+  onStop: () => void;
+};
+
+const TTSPanel = ({ isPlaying, onTogglePlay, onBackward, onForward, onStop }: TTSPanelProps) => {
+  return (
+    <div className='flex w-full items-center justify-center rounded-2xl p-4'>
+      <div className='flex items-center justify-between space-x-4'>
+        <button onClick={onBackward} className='hover:bg-base-200/75 rounded-full p-1'>
+          <MdFastRewind size={32} />
+        </button>
+        <button onClick={onTogglePlay} className='hover:bg-base-200/75 rounded-full p-1'>
+          {isPlaying ? (
+            <MdPauseCircle size={48} className='fill-primary' />
+          ) : (
+            <MdPlayCircle size={48} className='fill-primary' />
+          )}
+        </button>
+        <button onClick={onForward} className='hover:bg-base-200/75 rounded-full p-1'>
+          <MdFastForward size={32} />
+        </button>
+        <button onClick={onStop} className='hover:bg-base-200/75 rounded-full p-1'>
+          <MdStop size={32} />
+        </button>
+      </div>
+    </div>
+  );
+};
+
+export default TTSPanel;