feat: supported background TTS with media session controls, closes #2099 and closes #964 (#2138)

2025-09-30 01:45:17 +08:00
parent 25b44176b9
commit 0a1e0212e2
49 changed files with 1105 additions and 57 deletions
@@ -2,19 +2,22 @@ import clsx from 'clsx';
 import React, { useState, useRef, useEffect, useCallback } from 'react';
 import { useEnv } from '@/context/EnvContext';
 import { useThemeStore } from '@/store/themeStore';
+import { useSettingsStore } from '@/store/settingsStore';
 import { useBookDataStore } from '@/store/bookDataStore';
 import { useReaderStore } from '@/store/readerStore';
 import { useTranslation } from '@/hooks/useTranslation';
 import { useResponsiveSize } from '@/hooks/useResponsiveSize';
 import { TTSController, SILENCE_DATA, TTSMark } from '@/services/tts';
+import { getMediaSession, TauriMediaSession } from '@/libs/mediaSession';
 import { getPopupPosition, Position } from '@/utils/sel';
 import { eventDispatcher } from '@/utils/event';
 import { parseSSMLLang } from '@/utils/ssml';
 import { throttle } from '@/utils/throttle';
-import { invokeUseBackgroundAudio } from '@/utils/bridge';
 import { CFI } from '@/libs/document';
 import { Insets } from '@/types/misc';
 import { Overlay } from '@/components/Overlay';
+import { fetchImageAsBase64 } from '@/utils/image';
+import { invokeUseBackgroundAudio } from '@/utils/bridge';
 import Popup from '@/components/Popup';
 import TTSPanel from './TTSPanel';
 import TTSIcon from './TTSIcon';
@@ -33,6 +36,7 @@ const TTSControl: React.FC<TTSControlProps> = ({ bookKey, gridInsets }) => {
  const _ = useTranslation();
  const { appService } = useEnv();
  const { safeAreaInsets } = useThemeStore();
+  const { settings } = useSettingsStore();
  const { getBookData } = useBookDataStore();
  const { hoveredBookKey, getView, getProgress, getViewSettings } = useReaderStore();
  const { setViewSettings, setTTSEnabled } = useReaderStore();
@@ -58,6 +62,7 @@ const TTSControl: React.FC<TTSControlProps> = ({ bookKey, gridInsets }) => {
  const iconRef = useRef<HTMLDivElement>(null);
  const unblockerAudioRef = useRef<HTMLAudioElement | null>(null);
  const ttsControllerRef = useRef<TTSController | null>(null);
+  const mediaSessionRef = useRef<TauriMediaSession | MediaSession | null>(null);
  const [ttsController, setTtsController] = useState<TTSController | null>(null);
  const [ttsClientsInited, setTtsClientsInitialized] = useState(false);

@@ -92,6 +97,43 @@ const TTSControl: React.FC<TTSControlProps> = ({ bookKey, gridInsets }) => {
    }
  };

+  const initMediaSession = async () => {
+    const mediaSession = getMediaSession();
+    if (!mediaSession) return;
+
+    mediaSessionRef.current = mediaSession;
+
+    if (mediaSession instanceof TauriMediaSession) {
+      const bookData = getBookData(bookKey);
+      const progress = getProgress(bookKey);
+      if (!bookData || !bookData.book) return;
+      const { title, author, coverImageUrl } = bookData.book;
+      const { sectionLabel } = progress || {};
+
+      let artworkImage = '/icon.png';
+      try {
+        artworkImage = await fetchImageAsBase64(coverImageUrl || '/icon.png');
+      } catch {
+        artworkImage = await fetchImageAsBase64('/icon.png');
+      }
+
+      await mediaSession.setActive(true, settings.alwaysInForeground);
+      mediaSession.updateMetadata({
+        title: title,
+        artist: sectionLabel || title,
+        album: author,
+        artwork: artworkImage,
+      });
+    }
+  };
+
+  const deinitMediaSession = async () => {
+    if (mediaSessionRef.current && mediaSessionRef.current instanceof TauriMediaSession) {
+      await mediaSessionRef.current.setActive(false);
+    }
+    mediaSessionRef.current = null;
+  };
+
  useEffect(() => {
    return () => {
      if (ttsControllerRef.current) {
@@ -121,13 +163,24 @@ const TTSControl: React.FC<TTSControlProps> = ({ bookKey, gridInsets }) => {
      const progress = getProgress(bookKey);
      const { sectionLabel } = progress || {};
      const mark = (e as CustomEvent<TTSMark>).detail;
-      if ('mediaSession' in navigator) {
-        navigator.mediaSession.metadata = new MediaMetadata({
-          title: mark?.text || '',
-          artist: sectionLabel || title,
-          album: author,
-          artwork: [{ src: coverImageUrl || '/icon.png', sizes: '512x512', type: 'image/png' }],
-        });
+
+      if (mediaSessionRef.current) {
+        const mediaSession = mediaSessionRef.current;
+        if (mediaSession instanceof TauriMediaSession) {
+          mediaSession.updateMetadata({
+            title: mark?.text || '',
+            artist: sectionLabel || title,
+            album: author,
+            artwork: '',
+          });
+        } else {
+          mediaSession.metadata = new MediaMetadata({
+            title: mark?.text || '',
+            artist: sectionLabel || title,
+            album: author,
+            artwork: [{ src: coverImageUrl || '/icon.png', sizes: '512x512', type: 'image/png' }],
+          });
+        }
      }
    };

@@ -205,7 +258,10 @@ const TTSControl: React.FC<TTSControlProps> = ({ bookKey, gridInsets }) => {
      if (appService?.isMobile) {
        unblockAudio();
      }
+      await initMediaSession();
      setTtsClientsInitialized(false);
+
+      setShowIndicator(true);
      const ttsController = new TTSController(appService, view);
      await ttsController.init();
      await ttsController.initViewTTS();
@@ -223,7 +279,6 @@ const TTSControl: React.FC<TTSControlProps> = ({ bookKey, gridInsets }) => {
      }
      setTtsClientsInitialized(true);
      setTTSEnabled(bookKey, true);
-      setShowIndicator(true);
    } catch (error) {
      eventDispatcher.dispatch('toast', {
        message: _('TTS not supported for this document'),
@@ -302,6 +357,7 @@ const TTSControl: React.FC<TTSControlProps> = ({ bookKey, gridInsets }) => {
      if (appService?.isMobile) {
        releaseUnblockAudio();
      }
+      await deinitMediaSession();
      setTTSEnabled(bookKey, false);
    },
    // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -425,32 +481,33 @@ const TTSControl: React.FC<TTSControlProps> = ({ bookKey, gridInsets }) => {
  };

  useEffect(() => {
-    if ('mediaSession' in navigator) {
-      navigator.mediaSession.setActionHandler('play', () => {
+    const { current: mediaSession } = mediaSessionRef;
+    if (mediaSession) {
+      mediaSession.setActionHandler('play', () => {
        handleTogglePlay();
      });

-      navigator.mediaSession.setActionHandler('pause', () => {
+      mediaSession.setActionHandler('pause', () => {
        handleTogglePlay();
      });

-      navigator.mediaSession.setActionHandler('stop', () => {
+      mediaSession.setActionHandler('stop', () => {
        handlePause();
      });

-      navigator.mediaSession.setActionHandler('seekforward', () => {
+      mediaSession.setActionHandler('seekforward', () => {
        handleForward(true);
      });

-      navigator.mediaSession.setActionHandler('seekbackward', () => {
+      mediaSession.setActionHandler('seekbackward', () => {
        handleBackward(true);
      });

-      navigator.mediaSession.setActionHandler('nexttrack', () => {
+      mediaSession.setActionHandler('nexttrack', () => {
        handleForward();
      });

-      navigator.mediaSession.setActionHandler('previoustrack', () => {
+      mediaSession.setActionHandler('previoustrack', () => {
        handleBackward();
      });
    }