fix: tts now works in background in iOS, closes #547 (#822)

To enable background playback in Android, go to Settings > Apps & Notifications > Readest > Battery > Battery Optimization, and disable battery optimization for Readest.

apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/build.rs

@@ -2,6 +2,7 @@ const COMMANDS: &[&str] = &[
     "auth_with_safari",
     "auth_with_custom_tab",
     "copy_uri_to_path",
+    "use_background_audio",
 ];
 
 fn main() {

apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/ios/Sources/NativeBridgePlugin.swift

@@ -1,4 +1,6 @@
 import AuthenticationServices
+import AVFoundation
+import MediaPlayer
 import SwiftRs
 import Tauri
 import UIKit
@@ -8,9 +10,33 @@ class SafariAuthRequestArgs: Decodable {
   let authUrl: String
 }
 
+class UseBackgroundAudioRequestArgs: Decodable {
+  let enabled: Bool
+}
+
 class NativeBridgePlugin: Plugin {
   private var authSession: ASWebAuthenticationSession?
 
+  @objc public func use_background_audio(_ invoke: Invoke) {
+    do {
+      let args = try invoke.parseArgs(UseBackgroundAudioRequestArgs.self)
+      let enabled = args.enabled
+      let session = AVAudioSession.sharedInstance()
+      if enabled {
+        try session.setCategory(.playback, mode: .default, options: [.mixWithOthers])
+        try session.setActive(true)
+        print("AVAudioSession activated")
+      } else {
+        try session.setActive(false)
+        MPNowPlayingInfoCenter.default().nowPlayingInfo = nil
+        print("AVAudioSession deactivated")
+      }
+      invoke.resolve()
+    } catch {
+      print("Failed to set up audio session:", error)
+    }
+  }
+
   @objc public func auth_with_safari(_ invoke: Invoke) throws {
     let args = try invoke.parseArgs(SafariAuthRequestArgs.self)
     let authUrl = URL(string: args.authUrl)!

apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/permissions/autogenerated/commands/use_background_audio.toml

@@ -0,0 +1,13 @@
+# Automatically generated - DO NOT EDIT!
+
+"$schema" = "../../schemas/schema.json"
+
+[[permission]]
+identifier = "allow-use-background-audio"
+description = "Enables the use_background_audio command without any pre-configured scope."
+commands.allow = ["use_background_audio"]
+
+[[permission]]
+identifier = "deny-use-background-audio"
+description = "Denies the use_background_audio command without any pre-configured scope."
+commands.deny = ["use_background_audio"]

apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/permissions/autogenerated/reference.md

@@ -7,6 +7,7 @@ Default permissions for the plugin
 - `allow-auth-with-safari`
 - `allow-auth-with-custom-tab`
 - `allow-copy-uri-to-path`
+- `allow-use-background-audio`
 
 ## Permission Table
 
@@ -92,6 +93,32 @@ Enables the copy_uri_to_path command without any pre-configured scope.
 
 Denies the copy_uri_to_path command without any pre-configured scope.
 
+</td>
+</tr>
+
+<tr>
+<td>
+
+`native-bridge:allow-use-background-audio`
+
+</td>
+<td>
+
+Enables the use_background_audio command without any pre-configured scope.
+
+</td>
+</tr>
+
+<tr>
+<td>
+
+`native-bridge:deny-use-background-audio`
+
+</td>
+<td>
+
+Denies the use_background_audio command without any pre-configured scope.
+
 </td>
 </tr>
 </table>

apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/permissions/default.toml

@@ -1,3 +1,3 @@
 [default]
 description = "Default permissions for the plugin"
-permissions = ["allow-auth-with-safari", "allow-auth-with-custom-tab", "allow-copy-uri-to-path"]
+permissions = ["allow-auth-with-safari", "allow-auth-with-custom-tab", "allow-copy-uri-to-path", "allow-use-background-audio"]

apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/permissions/schemas/schema.json

@@ -331,10 +331,22 @@
           "markdownDescription": "Denies the copy_uri_to_path command without any pre-configured scope."
         },
         {
-          "description": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-auth-with-safari`\n- `allow-auth-with-custom-tab`\n- `allow-copy-uri-to-path`",
+          "description": "Enables the use_background_audio command without any pre-configured scope.",
+          "type": "string",
+          "const": "allow-use-background-audio",
+          "markdownDescription": "Enables the use_background_audio command without any pre-configured scope."
+        },
+        {
+          "description": "Denies the use_background_audio command without any pre-configured scope.",
+          "type": "string",
+          "const": "deny-use-background-audio",
+          "markdownDescription": "Denies the use_background_audio command without any pre-configured scope."
+        },
+        {
+          "description": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-auth-with-safari`\n- `allow-auth-with-custom-tab`\n- `allow-copy-uri-to-path`\n- `allow-use-background-audio`",
           "type": "string",
           "const": "default",
-          "markdownDescription": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-auth-with-safari`\n- `allow-auth-with-custom-tab`\n- `allow-copy-uri-to-path`"
+          "markdownDescription": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-auth-with-safari`\n- `allow-auth-with-custom-tab`\n- `allow-copy-uri-to-path`\n- `allow-use-background-audio`"
         }
       ]
     }

apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/src/commands.rs

@@ -27,3 +27,11 @@ pub(crate) async fn copy_uri_to_path<R: Runtime>(
 ) -> Result<CopyURIResponse> {
     app.native_bridge().copy_uri_to_path(payload)
 }
+
+#[command]
+pub(crate) async fn use_background_audio<R: Runtime>(
+    app: AppHandle<R>,
+    payload: UseBackgroundAudioRequest,
+) -> Result<()> {
+    app.native_bridge().use_background_audio(payload)
+}

apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/src/desktop.rs

@@ -25,4 +25,8 @@ impl<R: Runtime> NativeBridge<R> {
     pub fn copy_uri_to_path(&self, _payload: CopyURIRequest) -> crate::Result<CopyURIResponse> {
         Err(crate::Error::UnsupportedPlatformError)
     }
+
+    pub fn use_background_audio(&self, _payload: UseBackgroundAudioRequest) -> crate::Result<()> {
+        Err(crate::Error::UnsupportedPlatformError)
+    }
 }

apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/src/lib.rs

@@ -40,6 +40,7 @@ pub fn init<R: Runtime>() -> TauriPlugin<R> {
             commands::auth_with_safari,
             commands::auth_with_custom_tab,
             commands::copy_uri_to_path,
+            commands::use_background_audio,
         ])
         .setup(|app, api| {
             #[cfg(mobile)]

apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/src/mobile.rs

@@ -47,3 +47,11 @@ impl<R: Runtime> NativeBridge<R> {
             .map_err(Into::into)
     }
 }
+
+impl<R: Runtime> NativeBridge<R> {
+    pub fn use_background_audio(&self, payload: UseBackgroundAudioRequest) -> crate::Result<()> {
+        self.0
+            .run_mobile_plugin("use_background_audio", payload)
+            .map_err(Into::into)
+    }
+}

apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/src/models.rs

@@ -25,3 +25,9 @@ pub struct CopyURIResponse {
     pub success: bool,
     pub error: Option<String>,
 }
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct UseBackgroundAudioRequest {
+    pub enabled: bool,
+}

apps/readest-app/src/app/reader/components/FoliateViewer.tsx

@@ -58,9 +58,9 @@ const FoliateViewer: React.FC<{
     const { detail } = event as CustomEvent;
     detail.data = Promise.resolve(detail.data)
       .then((data) => {
-        const viewSettings = getViewSettings(bookKey)!;
+        const viewSettings = getViewSettings(bookKey);
         if (detail.type === 'text/css') return transformStylesheet(data);
-        if (detail.type === 'application/xhtml+xml') {
+        if (viewSettings && detail.type === 'application/xhtml+xml') {
           const ctx = {
             bookKey,
             viewSettings,

apps/readest-app/src/app/reader/components/tts/TTSControl.tsx

@@ -11,6 +11,7 @@ import { eventDispatcher } from '@/utils/event';
 import { parseSSMLLang } from '@/utils/ssml';
 import { getOSPlatform } from '@/utils/misc';
 import { throttle } from '@/utils/throttle';
+import { invokeUseBackgroundAudio } from '@/utils/bridge';
 import Popup from '@/components/Popup';
 import TTSPanel from './TTSPanel';
 import TTSIcon from './TTSIcon';
@@ -39,15 +40,32 @@ const TTSControl = () => {
 
   const iconRef = useRef<HTMLDivElement>(null);
   const ttsControllerRef = useRef<TTSController | null>(null);
+  const unblockerAudioRef = useRef<HTMLAudioElement | null>(null);
 
   // this enables WebAudio to play even when the mute toggle switch is ON
   const unblockAudio = () => {
-    const audio = document.createElement('audio');
-    audio.setAttribute('x-webkit-airplay', 'deny');
-    audio.preload = 'auto';
-    audio.loop = true;
-    audio.src = SILENCE_DATA;
-    audio.play();
+    if (unblockerAudioRef.current) return;
+    unblockerAudioRef.current = document.createElement('audio');
+    unblockerAudioRef.current.setAttribute('x-webkit-airplay', 'deny');
+    unblockerAudioRef.current.preload = 'auto';
+    unblockerAudioRef.current.loop = true;
+    unblockerAudioRef.current.src = SILENCE_DATA;
+    unblockerAudioRef.current.play();
+  };
+
+  const releaseUnblockAudio = () => {
+    if (!unblockerAudioRef.current) return;
+    try {
+      unblockerAudioRef.current.pause();
+      unblockerAudioRef.current.currentTime = 0;
+      unblockerAudioRef.current.removeAttribute('src');
+      unblockerAudioRef.current.src = '';
+      unblockerAudioRef.current.load();
+      unblockerAudioRef.current = null;
+      console.log('Unblock audio released');
+    } catch (err) {
+      console.warn('Error releasing unblock audio:', err);
+    }
   };
 
   useEffect(() => {
@@ -94,6 +112,9 @@ const TTSControl = () => {
     setShowIndicator(true);
 
     try {
+      if (appService?.isIOSApp) {
+        await invokeUseBackgroundAudio({ enabled: true });
+      }
       if (getOSPlatform() === 'ios' || appService?.isIOSApp) {
         unblockAudio();
       }
@@ -177,6 +198,12 @@ const TTSControl = () => {
       setShowPanel(false);
       setShowIndicator(false);
     }
+    if (appService?.isIOSApp) {
+      await invokeUseBackgroundAudio({ enabled: false });
+    }
+    if (getOSPlatform() === 'ios' || appService?.isIOSApp) {
+      releaseUnblockAudio();
+    }
   };
 
   // rate range: 0.5 - 3, 1.0 is normal speed

apps/readest-app/src/libs/edgeTTS.ts

@@ -143,8 +143,7 @@ const hashPayload = (payload: EdgeTTSPayload): string => {
 
 export class EdgeSpeechTTS {
   static voices = genVoiceList(EDGE_TTS_VOICES);
-  private static audioCache = new LRUCache<string, AudioBuffer>(200);
-  private audioContext = new AudioContext();
+  private static audioCache = new LRUCache<string, ArrayBuffer>(200);
 
   constructor() {}
 
@@ -267,17 +266,16 @@ export class EdgeSpeechTTS {
     return this.#fetchEdgeSpeechWs(payload);
   }
 
-  async createAudio(payload: EdgeTTSPayload): Promise<AudioBuffer> {
+  async createAudio(payload: EdgeTTSPayload): Promise<Blob> {
     const cacheKey = hashPayload(payload);
     if (EdgeSpeechTTS.audioCache.has(cacheKey)) {
-      return EdgeSpeechTTS.audioCache.get(cacheKey)!;
+      return new Blob([EdgeSpeechTTS.audioCache.get(cacheKey)!], { type: 'audio/mpeg' });
     }
     try {
       const res = await this.create(payload);
       const arrayBuffer = await res.arrayBuffer();
-      const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer.slice(0));
-      EdgeSpeechTTS.audioCache.set(cacheKey, audioBuffer);
-      return audioBuffer;
+      EdgeSpeechTTS.audioCache.set(cacheKey, arrayBuffer);
+      return new Blob([arrayBuffer], { type: 'audio/mpeg' });
     } catch (error) {
       throw error;
     }

apps/readest-app/src/services/tts/EdgeTTSClient.ts

@@ -13,31 +13,19 @@ export class EdgeTTSClient implements TTSClient {
   #voices: TTSVoice[] = [];
   #edgeTTS: EdgeSpeechTTS;
 
-  static #audioContext: AudioContext | null;
-  #sourceNode: AudioBufferSourceNode | null = null;
+  #audioElement: HTMLAudioElement | null = null;
   #isPlaying = false;
   #pausedAt = 0;
   #startedAt = 0;
-  #audioBuffer: AudioBuffer | null = null;
   available = true;
 
   constructor() {
     this.#edgeTTS = new EdgeSpeechTTS();
   }
 
-  async initializeAudioContext() {
-    if (!EdgeTTSClient.#audioContext) {
-      EdgeTTSClient.#audioContext = new AudioContext();
-    }
-    if (EdgeTTSClient.#audioContext.state === 'suspended') {
-      await EdgeTTSClient.#audioContext.resume();
-    }
-  }
-
   async init() {
     this.#voices = EdgeSpeechTTS.voices;
     try {
-      await this.initializeAudioContext();
       await this.#edgeTTS.create({
         lang: 'en',
         text: 'test',
@@ -115,15 +103,12 @@ export class EdgeTTSClient implements TTSClient {
         break;
       }
       try {
-        this.#audioBuffer = await this.#edgeTTS.createAudio(
-          this.getPayload(lang, mark.text, voiceId),
-        );
-        if (!EdgeTTSClient.#audioContext) {
-          EdgeTTSClient.#audioContext = new AudioContext();
-        }
-        this.#sourceNode = EdgeTTSClient.#audioContext.createBufferSource();
-        this.#sourceNode.buffer = this.#audioBuffer;
-        this.#sourceNode.connect(EdgeTTSClient.#audioContext.destination);
+        const blob = await this.#edgeTTS.createAudio(this.getPayload(lang, mark.text, voiceId));
+        const url = URL.createObjectURL(blob);
+        this.#audioElement = new Audio(url);
+        const audio = this.#audioElement;
+        audio.setAttribute('x-webkit-airplay', 'deny');
+        audio.preload = 'auto';
 
         yield {
           code: 'boundary',
@@ -132,36 +117,37 @@ export class EdgeTTSClient implements TTSClient {
         };
 
         const result = await new Promise<TTSMessageEvent>((resolve) => {
-          if (EdgeTTSClient.#audioContext === null || this.#sourceNode === null) {
-            throw new Error('Audio context or source node is null');
-          }
-          this.#sourceNode.onended = (event: Event) => {
-            // chunk finished speaking or aborted speaking
-            if (signal.aborted || event.type === 'stopped') {
-              resolve({
-                code: 'error',
-                message: 'Aborted',
-              });
-              return;
+          const cleanUp = () => {
+            audio.onended = null;
+            audio.onerror = null;
+            audio.pause();
+            audio.src = '';
+            URL.revokeObjectURL(url);
+          };
+          audio.onended = () => {
+            cleanUp();
+            if (signal.aborted) {
+              resolve({ code: 'error', message: 'Aborted' });
+            } else {
+              resolve({ code: 'end', message: `Chunk finished: ${mark.name}` });
             }
-            resolve({
-              code: 'end',
-              message: `Chunk finished: ${mark.name}`,
-            });
+          };
+          audio.onerror = (e) => {
+            cleanUp();
+            console.warn('Audio playback error:', e);
+            resolve({ code: 'error', message: 'Audio playback error' });
           };
           if (signal.aborted) {
-            resolve({
-              code: 'error',
-              message: 'Aborted',
-            });
+            cleanUp();
+            resolve({ code: 'error', message: 'Aborted' });
             return;
           }
-          if (EdgeTTSClient.#audioContext.state === 'suspended') {
-            EdgeTTSClient.#audioContext.resume();
-          }
-          this.#sourceNode.start(0);
           this.#isPlaying = true;
-          this.#startedAt = EdgeTTSClient.#audioContext.currentTime;
+          audio.play().catch((err) => {
+            cleanUp();
+            console.error('Failed to play audio:', err);
+            resolve({ code: 'error', message: 'Playback failed: ' + err.message });
+          });
         });
         yield result;
       } catch (error) {
@@ -186,17 +172,17 @@ export class EdgeTTSClient implements TTSClient {
   }
 
   async pause() {
-    if (!this.#isPlaying || !EdgeTTSClient.#audioContext) return;
-    this.#pausedAt = EdgeTTSClient.#audioContext.currentTime - this.#startedAt;
-    await EdgeTTSClient.#audioContext.suspend();
+    if (!this.#isPlaying || !this.#audioElement) return;
+    this.#pausedAt = this.#audioElement.currentTime - this.#startedAt;
+    await this.#audioElement.pause();
     this.#isPlaying = false;
   }
 
   async resume() {
-    if (this.#isPlaying || !EdgeTTSClient.#audioContext) return;
-    await EdgeTTSClient.#audioContext.resume();
+    if (this.#isPlaying || !this.#audioElement) return;
+    await this.#audioElement.play();
     this.#isPlaying = true;
-    this.#startedAt = EdgeTTSClient.#audioContext.currentTime - this.#pausedAt;
+    this.#startedAt = this.#audioElement.currentTime - this.#pausedAt;
   }
 
   async stop() {
@@ -207,21 +193,18 @@ export class EdgeTTSClient implements TTSClient {
     this.#isPlaying = false;
     this.#pausedAt = 0;
     this.#startedAt = 0;
-    if (this.#sourceNode) {
-      try {
-        this.#sourceNode.stop();
-        if (this.#sourceNode?.onended) {
-          this.#sourceNode.onended(new Event('stopped'));
-        }
-      } catch (err) {
-        if (!(err instanceof Error) || err.name !== 'InvalidStateError') {
-          console.log('Error stopping source node:', err);
-        }
+    if (this.#audioElement) {
+      this.#audioElement.pause();
+      this.#audioElement.currentTime = 0;
+      if (this.#audioElement?.onended) {
+        this.#audioElement.onended(new Event('stopped'));
       }
-      this.#sourceNode.disconnect();
-      this.#sourceNode = null;
+      if (this.#audioElement.src?.startsWith('blob:')) {
+        URL.revokeObjectURL(this.#audioElement.src);
+      }
+      this.#audioElement.src = '';
+      this.#audioElement = null;
     }
-    this.#audioBuffer = null;
   }
 
   async setRate(rate: number) {

apps/readest-app/src/utils/bridge.ts

@@ -10,6 +10,10 @@ export interface CopyURIResponse {
   error?: string;
 }
 
+export interface UseBackgroundAudioRequest {
+  enabled: boolean;
+}
+
 export async function copyURIToPath(request: CopyURIRequest): Promise<CopyURIResponse> {
   const result = await invoke<CopyURIResponse>('plugin:native-bridge|copy_uri_to_path', {
     payload: request,
@@ -17,3 +21,9 @@ export async function copyURIToPath(request: CopyURIRequest): Promise<CopyURIRes
 
   return result;
 }
+
+export async function invokeUseBackgroundAudio(request: UseBackgroundAudioRequest): Promise<void> {
+  await invoke('plugin:native-bridge|use_background_audio', {
+    payload: request,
+  });
+}