From 0a1e0212e2e1cd5ff3cd330b8389453f54a3b0f3 Mon Sep 17 00:00:00 2001 From: Huang Xin Date: Tue, 30 Sep 2025 01:45:17 +0800 Subject: [PATCH] feat: supported background TTS with media session controls, closes #2099 and closes #964 (#2138) --- .../public/locales/ar/translation.json | 3 +- .../public/locales/bn/translation.json | 3 +- .../public/locales/bo/translation.json | 3 +- .../public/locales/de/translation.json | 3 +- .../public/locales/el/translation.json | 3 +- .../public/locales/es/translation.json | 3 +- .../public/locales/fr/translation.json | 3 +- .../public/locales/hi/translation.json | 3 +- .../public/locales/id/translation.json | 3 +- .../public/locales/it/translation.json | 3 +- .../public/locales/ja/translation.json | 3 +- .../public/locales/ko/translation.json | 3 +- .../public/locales/nl/translation.json | 3 +- .../public/locales/pl/translation.json | 3 +- .../public/locales/pt/translation.json | 3 +- .../public/locales/ru/translation.json | 3 +- .../public/locales/si/translation.json | 3 +- .../public/locales/ta/translation.json | 3 +- .../public/locales/th/translation.json | 3 +- .../public/locales/tr/translation.json | 3 +- .../public/locales/uk/translation.json | 3 +- .../public/locales/vi/translation.json | 3 +- .../public/locales/zh-CN/translation.json | 3 +- .../public/locales/zh-TW/translation.json | 3 +- .../android/app/src/main/AndroidManifest.xml | 5 +- .../android/build.gradle.kts | 4 +- .../android/build.gradle.kts | 8 +- .../android/src/main/java/NativeTTSPlugin.kt | 362 +++++++++++++++++- .../plugins/tauri-plugin-native-tts/build.rs | 5 + .../commands/checkPermissions.toml | 13 + .../commands/requestPermissions.toml | 13 + .../commands/set_media_session_active.toml | 13 + .../update_media_session_metadata.toml | 13 + .../commands/update_media_session_state.toml | 13 + .../permissions/autogenerated/reference.md | 135 +++++++ .../permissions/default.toml | 5 + .../permissions/schemas/schema.json | 64 +++- .../tauri-plugin-native-tts/src/commands.rs | 24 ++ .../tauri-plugin-native-tts/src/desktop.rs | 18 + .../tauri-plugin-native-tts/src/lib.rs | 3 + .../tauri-plugin-native-tts/src/mobile.rs | 33 ++ .../tauri-plugin-native-tts/src/models.rs | 23 ++ .../app/library/components/SettingsMenu.tsx | 32 ++ .../app/reader/components/tts/TTSControl.tsx | 91 ++++- .../app/reader/hooks/useTextTranslation.ts | 2 +- apps/readest-app/src/libs/mediaSession.ts | 135 +++++++ apps/readest-app/src/services/constants.ts | 1 + apps/readest-app/src/types/settings.ts | 1 + apps/readest-app/src/utils/image.ts | 74 ++++ 49 files changed, 1105 insertions(+), 57 deletions(-) create mode 100644 apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/checkPermissions.toml create mode 100644 apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/requestPermissions.toml create mode 100644 apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/set_media_session_active.toml create mode 100644 apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/update_media_session_metadata.toml create mode 100644 apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/update_media_session_state.toml create mode 100644 apps/readest-app/src/libs/mediaSession.ts create mode 100644 apps/readest-app/src/utils/image.ts diff --git a/apps/readest-app/public/locales/ar/translation.json b/apps/readest-app/public/locales/ar/translation.json index 50704a3a..bce68e43 100644 --- a/apps/readest-app/public/locales/ar/translation.json +++ b/apps/readest-app/public/locales/ar/translation.json @@ -612,5 +612,6 @@ "Restart App": "إعادة تشغيل التطبيق", "Start Migration": "بدء الترحيل", "Advanced Settings": "إعدادات متقدمة", - "File count: {{size}}": "عدد الملفات: {{size}}" + "File count: {{size}}": "عدد الملفات: {{size}}", + "Background Read Aloud": "قراءة بصوت عالٍ في الخلفية" } diff --git a/apps/readest-app/public/locales/bn/translation.json b/apps/readest-app/public/locales/bn/translation.json index 502c60bc..04994962 100644 --- a/apps/readest-app/public/locales/bn/translation.json +++ b/apps/readest-app/public/locales/bn/translation.json @@ -596,5 +596,6 @@ "Restart App": "অ্যাপ পুনরায় চালু করুন", "Start Migration": "মাইগ্রেশন শুরু করুন", "Advanced Settings": "উন্নত সেটিংস", - "File count: {{size}}": "ফাইলের সংখ্যা: {{size}}" + "File count: {{size}}": "ফাইলের সংখ্যা: {{size}}", + "Background Read Aloud": "পটভূমিতে উচ্চস্বরে পড়ুন" } diff --git a/apps/readest-app/public/locales/bo/translation.json b/apps/readest-app/public/locales/bo/translation.json index 499d6830..920643e1 100644 --- a/apps/readest-app/public/locales/bo/translation.json +++ b/apps/readest-app/public/locales/bo/translation.json @@ -592,5 +592,6 @@ "Restart App": "དེབ་ཀྱི་སྤྱོད་བྱས་མ་ཐུབ།", "Start Migration": "མཉམ་དུ་ཀློག་འཛུལ་བྱས་མ་ཐུབ།", "Advanced Settings": "དབང་བསྐྱོད་སྒྲིག་སྟངས།", - "File count: {{size}}": "དེབ་གནས་བཅས་པའི་ཨང་། {{size}}" + "File count: {{size}}": "དེབ་གནས་བཅས་པའི་ཨང་། {{size}}", + "Background Read Aloud": "གནས་སྟངས་ཀྱིས་ཀློག་བྱེད།" } diff --git a/apps/readest-app/public/locales/de/translation.json b/apps/readest-app/public/locales/de/translation.json index 122afd4a..24ca885e 100644 --- a/apps/readest-app/public/locales/de/translation.json +++ b/apps/readest-app/public/locales/de/translation.json @@ -596,5 +596,6 @@ "Restart App": "App neu starten", "Start Migration": "Migration starten", "Advanced Settings": "Erweiterte Einstellungen", - "File count: {{size}}": "Dateianzahl: {{size}}" + "File count: {{size}}": "Dateianzahl: {{size}}", + "Background Read Aloud": "Hintergrund-Vorlesen" } diff --git a/apps/readest-app/public/locales/el/translation.json b/apps/readest-app/public/locales/el/translation.json index d0e69990..4237c0ba 100644 --- a/apps/readest-app/public/locales/el/translation.json +++ b/apps/readest-app/public/locales/el/translation.json @@ -596,5 +596,6 @@ "Restart App": "Επανεκκίνηση εφαρμογής", "Start Migration": "Έναρξη μετανάστευσης", "Advanced Settings": "Προηγμένες ρυθμίσεις", - "File count: {{size}}": "Αριθμός αρχείων: {{size}}" + "File count: {{size}}": "Αριθμός αρχείων: {{size}}", + "Background Read Aloud": "Ανάγνωση στο παρασκήνιο" } diff --git a/apps/readest-app/public/locales/es/translation.json b/apps/readest-app/public/locales/es/translation.json index 7fe4ecc8..f2ca961e 100644 --- a/apps/readest-app/public/locales/es/translation.json +++ b/apps/readest-app/public/locales/es/translation.json @@ -600,5 +600,6 @@ "Restart App": "Reiniciar Aplicación", "Start Migration": "Iniciar Migración", "Advanced Settings": "Configuraciones Avanzadas", - "File count: {{size}}": "Número de archivos: {{size}}" + "File count: {{size}}": "Número de archivos: {{size}}", + "Background Read Aloud": "Lectura en segundo plano" } diff --git a/apps/readest-app/public/locales/fr/translation.json b/apps/readest-app/public/locales/fr/translation.json index 162d6b8b..1a628615 100644 --- a/apps/readest-app/public/locales/fr/translation.json +++ b/apps/readest-app/public/locales/fr/translation.json @@ -600,5 +600,6 @@ "Restart App": "Redémarrer l'Application", "Start Migration": "Démarrer la Migration", "Advanced Settings": "Paramètres Avancés", - "File count: {{size}}": "Nombre de fichiers : {{size}}" + "File count: {{size}}": "Nombre de fichiers : {{size}}", + "Background Read Aloud": "Lecture Vocale en Arrière-Plan" } diff --git a/apps/readest-app/public/locales/hi/translation.json b/apps/readest-app/public/locales/hi/translation.json index b72958f1..4527f99a 100644 --- a/apps/readest-app/public/locales/hi/translation.json +++ b/apps/readest-app/public/locales/hi/translation.json @@ -596,5 +596,6 @@ "Restart App": "एप्लिकेशन पुनरारंभ करें", "Start Migration": "माइग्रेशन प्रारंभ करें", "Advanced Settings": "उन्नत सेटिंग्स", - "File count: {{size}}": "फ़ाइलों की संख्या: {{size}}" + "File count: {{size}}": "फ़ाइलों की संख्या: {{size}}", + "Background Read Aloud": "पृष्ठभूमि में वाचन" } diff --git a/apps/readest-app/public/locales/id/translation.json b/apps/readest-app/public/locales/id/translation.json index cac02f03..ac43b190 100644 --- a/apps/readest-app/public/locales/id/translation.json +++ b/apps/readest-app/public/locales/id/translation.json @@ -592,5 +592,6 @@ "Restart App": "Restart Aplikasi", "Start Migration": "Mulai Migrasi", "Advanced Settings": "Pengaturan Lanjutan", - "File count: {{size}}": "Jumlah file: {{size}}" + "File count: {{size}}": "Jumlah file: {{size}}", + "Background Read Aloud": "Bacakan Latar Belakang" } diff --git a/apps/readest-app/public/locales/it/translation.json b/apps/readest-app/public/locales/it/translation.json index d02ce1ce..bf90a0bc 100644 --- a/apps/readest-app/public/locales/it/translation.json +++ b/apps/readest-app/public/locales/it/translation.json @@ -600,5 +600,6 @@ "Restart App": "Riavvia App", "Start Migration": "Inizia Migrazione", "Advanced Settings": "Impostazioni Avanzate", - "File count: {{size}}": "Conteggio file: {{size}}" + "File count: {{size}}": "Conteggio file: {{size}}", + "Background Read Aloud": "Background Read Aloud" } diff --git a/apps/readest-app/public/locales/ja/translation.json b/apps/readest-app/public/locales/ja/translation.json index d1eab325..71a31549 100644 --- a/apps/readest-app/public/locales/ja/translation.json +++ b/apps/readest-app/public/locales/ja/translation.json @@ -592,5 +592,6 @@ "Restart App": "アプリを再起動", "Start Migration": "移行を開始", "Advanced Settings": "詳細設定", - "File count: {{size}}": "ファイル数: {{size}}" + "File count: {{size}}": "ファイル数: {{size}}", + "Background Read Aloud": "バックグラウンド読み上げ" } diff --git a/apps/readest-app/public/locales/ko/translation.json b/apps/readest-app/public/locales/ko/translation.json index c1a3bef7..e19ff4ec 100644 --- a/apps/readest-app/public/locales/ko/translation.json +++ b/apps/readest-app/public/locales/ko/translation.json @@ -592,5 +592,6 @@ "Restart App": "앱 재시작", "Start Migration": "마이그레이션 시작", "Advanced Settings": "고급 설정", - "File count: {{size}}": "파일 수: {{size}}" + "File count: {{size}}": "파일 수: {{size}}", + "Background Read Aloud": "백그라운드 음성 읽기" } diff --git a/apps/readest-app/public/locales/nl/translation.json b/apps/readest-app/public/locales/nl/translation.json index b183e99e..f87ccd71 100644 --- a/apps/readest-app/public/locales/nl/translation.json +++ b/apps/readest-app/public/locales/nl/translation.json @@ -596,5 +596,6 @@ "Restart App": "Herstart App", "Start Migration": "Start Migratie", "Advanced Settings": "Geavanceerde Instellingen", - "File count: {{size}}": "Bestandsaantal: {{size}}" + "File count: {{size}}": "Bestandsaantal: {{size}}", + "Background Read Aloud": "Achtergrond Voorlezen" } diff --git a/apps/readest-app/public/locales/pl/translation.json b/apps/readest-app/public/locales/pl/translation.json index de5d0a9d..5dcf8b00 100644 --- a/apps/readest-app/public/locales/pl/translation.json +++ b/apps/readest-app/public/locales/pl/translation.json @@ -604,5 +604,6 @@ "Restart App": "Zrestartuj aplikację", "Start Migration": "Rozpocznij migrację", "Advanced Settings": "Ustawienia zaawansowane", - "File count: {{size}}": "Liczba plików: {{size}}" + "File count: {{size}}": "Liczba plików: {{size}}", + "Background Read Aloud": "Czytanie na głos w tle" } diff --git a/apps/readest-app/public/locales/pt/translation.json b/apps/readest-app/public/locales/pt/translation.json index 955cfef6..9bd08b9a 100644 --- a/apps/readest-app/public/locales/pt/translation.json +++ b/apps/readest-app/public/locales/pt/translation.json @@ -600,5 +600,6 @@ "Restart App": "Reiniciar Aplicativo", "Start Migration": "Iniciar Migração", "Advanced Settings": "Configurações Avançadas", - "File count: {{size}}": "Contagem de arquivos: {{size}}" + "File count: {{size}}": "Contagem de arquivos: {{size}}", + "Background Read Aloud": "Leitura em Segundo Plano" } diff --git a/apps/readest-app/public/locales/ru/translation.json b/apps/readest-app/public/locales/ru/translation.json index 31d20817..772574e7 100644 --- a/apps/readest-app/public/locales/ru/translation.json +++ b/apps/readest-app/public/locales/ru/translation.json @@ -604,5 +604,6 @@ "Restart App": "Перезапустить приложение", "Start Migration": "Начать миграцию", "Advanced Settings": "Расширенные настройки", - "File count: {{size}}": "Количество файлов: {{size}}" + "File count: {{size}}": "Количество файлов: {{size}}", + "Background Read Aloud": "Озвучивание в фоне" } diff --git a/apps/readest-app/public/locales/si/translation.json b/apps/readest-app/public/locales/si/translation.json index 20495d2f..3feb93cb 100644 --- a/apps/readest-app/public/locales/si/translation.json +++ b/apps/readest-app/public/locales/si/translation.json @@ -596,5 +596,6 @@ "Restart App": "යෙදුම නැවත ආරම්භ කරන්න", "Start Migration": "මැතිවරණය ආරම්භ කරන්න", "Advanced Settings": "උසස් සැකසුම්", - "File count: {{size}}": "ගොනු ගණන: {{size}}" + "File count: {{size}}": "ගොනු ගණන: {{size}}", + "Background Read Aloud": "පසුබිම් පවසා කියවීම" } diff --git a/apps/readest-app/public/locales/ta/translation.json b/apps/readest-app/public/locales/ta/translation.json index 6f5cd943..ef4067f1 100644 --- a/apps/readest-app/public/locales/ta/translation.json +++ b/apps/readest-app/public/locales/ta/translation.json @@ -596,5 +596,6 @@ "Restart App": "செயலியை மீண்டும் தொடங்கவும்", "Start Migration": "மொத்தமாக்கலை தொடங்கவும்", "Advanced Settings": "மேம்பட்ட அமைப்புகள்", - "File count: {{size}}": "கோப்பு எண்ணிக்கை: {{size}}" + "File count: {{size}}": "கோப்பு எண்ணிக்கை: {{size}}", + "Background Read Aloud": "பின்னணி உரை வாசிப்பு" } diff --git a/apps/readest-app/public/locales/th/translation.json b/apps/readest-app/public/locales/th/translation.json index 8062aaa7..3a06a24d 100644 --- a/apps/readest-app/public/locales/th/translation.json +++ b/apps/readest-app/public/locales/th/translation.json @@ -592,5 +592,6 @@ "Restart App": "รีสตาร์ทแอป", "Start Migration": "เริ่มการย้ายข้อมูล", "Advanced Settings": "การตั้งค่าขั้นสูง", - "File count: {{size}}": "จำนวนไฟล์: {{size}}" + "File count: {{size}}": "จำนวนไฟล์: {{size}}", + "Background Read Aloud": "อ่านออกเสียงเบื้องหลัง" } diff --git a/apps/readest-app/public/locales/tr/translation.json b/apps/readest-app/public/locales/tr/translation.json index 4523fcb0..bf452bcd 100644 --- a/apps/readest-app/public/locales/tr/translation.json +++ b/apps/readest-app/public/locales/tr/translation.json @@ -596,5 +596,6 @@ "Restart App": "Uygulamayı Yeniden Başlat", "Start Migration": "Taşımayı Başlat", "Advanced Settings": "Gelişmiş Ayarlar", - "File count: {{size}}": "Dosya sayısı: {{size}}" + "File count: {{size}}": "Dosya sayısı: {{size}}", + "Background Read Aloud": "Arka Planda Sesli Okuma" } diff --git a/apps/readest-app/public/locales/uk/translation.json b/apps/readest-app/public/locales/uk/translation.json index c504f341..f8db0904 100644 --- a/apps/readest-app/public/locales/uk/translation.json +++ b/apps/readest-app/public/locales/uk/translation.json @@ -604,5 +604,6 @@ "Restart App": "Перезапустити програму", "Start Migration": "Почати переміщення", "Advanced Settings": "Розширені налаштування", - "File count: {{size}}": "Кількість файлів: {{size}}" + "File count: {{size}}": "Кількість файлів: {{size}}", + "Background Read Aloud": "Озвучування у фоновому режимі" } diff --git a/apps/readest-app/public/locales/vi/translation.json b/apps/readest-app/public/locales/vi/translation.json index 56ab4fd9..3a5af09e 100644 --- a/apps/readest-app/public/locales/vi/translation.json +++ b/apps/readest-app/public/locales/vi/translation.json @@ -592,5 +592,6 @@ "Restart App": "Khởi động lại ứng dụng", "Start Migration": "Bắt đầu di chuyển", "Advanced Settings": "Cài đặt nâng cao", - "File count: {{size}}": "Số lượng tệp: {{size}}" + "File count: {{size}}": "Số lượng tệp: {{size}}", + "Background Read Aloud": "Đọc To Nền" } diff --git a/apps/readest-app/public/locales/zh-CN/translation.json b/apps/readest-app/public/locales/zh-CN/translation.json index 78055472..9e9e6c87 100644 --- a/apps/readest-app/public/locales/zh-CN/translation.json +++ b/apps/readest-app/public/locales/zh-CN/translation.json @@ -592,5 +592,6 @@ "Restart App": "重启应用", "Start Migration": "开始迁移", "Advanced Settings": "高级设置", - "File count: {{size}}": "文件数量: {{size}}" + "File count: {{size}}": "文件数量: {{size}}", + "Background Read Aloud": "后台语音朗读" } diff --git a/apps/readest-app/public/locales/zh-TW/translation.json b/apps/readest-app/public/locales/zh-TW/translation.json index 6565715f..313f0d86 100644 --- a/apps/readest-app/public/locales/zh-TW/translation.json +++ b/apps/readest-app/public/locales/zh-TW/translation.json @@ -592,5 +592,6 @@ "Restart App": "重新啟動應用", "Start Migration": "開始遷移", "Advanced Settings": "高級設置", - "File count: {{size}}": "文件數量: {{size}}" + "File count: {{size}}": "文件數量: {{size}}", + "Background Read Aloud": "背景朗讀" } diff --git a/apps/readest-app/src-tauri/gen/android/app/src/main/AndroidManifest.xml b/apps/readest-app/src-tauri/gen/android/app/src/main/AndroidManifest.xml index fd3917f7..376251a8 100644 --- a/apps/readest-app/src-tauri/gen/android/app/src/main/AndroidManifest.xml +++ b/apps/readest-app/src-tauri/gen/android/app/src/main/AndroidManifest.xml @@ -1,6 +1,9 @@ - + + + + diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/android/build.gradle.kts b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/android/build.gradle.kts index e0fe1e91..ad6ed780 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/android/build.gradle.kts +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-bridge/android/build.gradle.kts @@ -33,8 +33,8 @@ android { } dependencies { - implementation("androidx.core:core-ktx:1.9.0") - implementation("androidx.appcompat:appcompat:1.6.0") + implementation("androidx.core:core-ktx:1.12.0") + implementation("androidx.appcompat:appcompat:1.6.1") implementation("androidx.browser:browser:1.8.0") implementation("com.google.android.material:material:1.7.0") testImplementation("junit:junit:4.13.2") diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/android/build.gradle.kts b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/android/build.gradle.kts index 65c829dd..1a40c60f 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/android/build.gradle.kts +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/android/build.gradle.kts @@ -33,9 +33,11 @@ android { } dependencies { - - implementation("androidx.core:core-ktx:1.9.0") - implementation("androidx.appcompat:appcompat:1.6.0") + implementation("androidx.media3:media3-common:1.2.0") + implementation("androidx.media3:media3-session:1.2.0") + implementation("androidx.media3:media3-exoplayer:1.2.0") + implementation("androidx.core:core-ktx:1.12.0") + implementation("androidx.appcompat:appcompat:1.6.1") implementation("com.google.android.material:material:1.7.0") testImplementation("junit:junit:4.13.2") androidTestImplementation("androidx.test.ext:junit:1.1.5") diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/android/src/main/java/NativeTTSPlugin.kt b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/android/src/main/java/NativeTTSPlugin.kt index af9b5c09..ac65d1e5 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/android/src/main/java/NativeTTSPlugin.kt +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/android/src/main/java/NativeTTSPlugin.kt @@ -1,5 +1,6 @@ package com.readest.native_tts +import android.Manifest import android.os.Bundle import android.app.Activity import android.content.Context @@ -8,8 +9,18 @@ import android.speech.tts.TextToSpeech import android.speech.tts.UtteranceProgressListener import android.speech.tts.Voice import android.util.Log +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import androidx.media3.common.MediaMetadata +import androidx.media3.common.Player +import androidx.media3.common.MediaItem +import androidx.media3.common.util.UnstableApi +import androidx.media3.common.ForwardingPlayer +import androidx.media3.session.MediaSession +import androidx.media3.exoplayer.ExoPlayer import app.tauri.annotation.Command import app.tauri.annotation.InvokeArg +import app.tauri.annotation.Permission import app.tauri.annotation.TauriPlugin import app.tauri.plugin.Invoke import app.tauri.plugin.JSObject @@ -25,6 +36,16 @@ import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference import java.util.* +import java.net.URL + +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.Service +import android.content.Intent +import android.os.Build +import android.os.IBinder +import androidx.core.app.NotificationCompat +import androidx.core.app.NotificationManagerCompat data class TTSVoiceData( val id: String, @@ -60,7 +81,103 @@ class SetVoiceArgs( val voice: String? = null ) -@TauriPlugin +@InvokeArg +class UpdateMediaSessionMetadataArgs { + var title: String? = null + var artist: String? = null + var album: String? = null + var artwork: String? = null +} + +@InvokeArg +class UpdateMediaSessionStateArgs { + var playing: Boolean? = null + var position: Double? = null + var duration: Double? = null +} + +@InvokeArg +class SetMediaSessionActiveArgs { + var active: Boolean? = null +} + +class MediaForegroundService : Service() { + companion object { + const val NOTIFICATION_ID = 1001 + const val CHANNEL_ID = "tts_media_playback_channel" + const val ACTION_START = "START_FOREGROUND_SERVICE" + const val ACTION_STOP = "STOP_FOREGROUND_SERVICE" + + var pluginInstance: NativeTTSPlugin? = null + } + + override fun onCreate() { + super.onCreate() + createNotificationChannel() + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + when (intent?.action) { + ACTION_START -> { + startForegroundService() + } + ACTION_STOP -> { + stopForegroundService() + } + } + return START_STICKY // Restart if killed + } + + override fun onBind(intent: Intent?): IBinder? = null + + private fun createNotificationChannel() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val channel = NotificationChannel( + CHANNEL_ID, + "TTS Media Playback", + NotificationManager.IMPORTANCE_MIN + ).apply { + description = "Controls for TTS media playback" + setShowBadge(false) + setSound(null, null) + } + val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + manager.createNotificationChannel(channel) + } + } + + private fun startForegroundService() { + val notification = NotificationCompat.Builder(this, CHANNEL_ID) + .setContentTitle("TTS Media Session") + .setContentText("Text-to-speech playback active") + .setSmallIcon(android.R.drawable.ic_media_play) // Replace with your icon + .setOngoing(true) + .setSilent(true) + .setPriority(NotificationCompat.PRIORITY_LOW) + .setCategory(NotificationCompat.CATEGORY_SERVICE) + .build() + + startForeground(NOTIFICATION_ID, notification) + Log.d("MediaForegroundService", "Foreground service started") + } + + private fun stopForegroundService() { + stopForeground(STOP_FOREGROUND_REMOVE) + stopSelf() + Log.d("MediaForegroundService", "Foreground service stopped") + } + + override fun onDestroy() { + super.onDestroy() + Log.d("MediaForegroundService", "Service destroyed") + } +} + +@TauriPlugin( + permissions = [ + Permission(strings = [Manifest.permission.POST_NOTIFICATIONS], alias = "postNotification") + ] +) class NativeTTSPlugin(private val activity: Activity) : Plugin(activity) { companion object { @@ -78,7 +195,15 @@ class NativeTTSPlugin(private val activity: Activity) : Plugin(activity) { private val eventChannels = ConcurrentHashMap>() private val speakingJobs = ConcurrentHashMap() + private var player: Player? = null + private var mediaSession: MediaSession? = null + private var isMediaSessionActive = false + private val coroutineScope = CoroutineScope(Dispatchers.Main + SupervisorJob()) + + init { + MediaForegroundService.pluginInstance = this + } @Command fun init(invoke: Invoke) { @@ -373,13 +498,236 @@ class NativeTTSPlugin(private val activity: Activity) : Plugin(activity) { invoke.reject("Exception getting voices: ${e.message}") } } + + @UnstableApi + private class WebViewPlayer(player: Player) : ForwardingPlayer(player) { + var eventTrigger: ((String, JSObject) -> Unit)? = null + + override fun play() { + eventTrigger?.invoke("media-session-play", JSObject()) + if (!this.playWhenReady) { + this.playWhenReady = true + } else { + this.playWhenReady = false + } + } + override fun pause() { + eventTrigger?.invoke("media-session-pause", JSObject()) + if (!this.playWhenReady) { + this.playWhenReady = true + } else { + this.playWhenReady = false + } + } + override fun seekToNext() { + eventTrigger?.invoke("media-session-next", JSObject()) + } + override fun seekToPrevious() { + eventTrigger?.invoke("media-session-previous", JSObject()) + } + } + + private fun initializeMediaSession() { + var basePlayer = ExoPlayer.Builder(activity).build() + val webViewPlayer = WebViewPlayer(basePlayer) + webViewPlayer.eventTrigger = { event, data -> trigger(event, data) } + player = webViewPlayer + + mediaSession = MediaSession.Builder(activity, player!!) + .setCallback(object : MediaSession.Callback { + override fun onConnect( + session: MediaSession, + controller: MediaSession.ControllerInfo + ): MediaSession.ConnectionResult { + val builder = MediaSession.ConnectionResult.AcceptedResultBuilder(session) + builder.setAvailablePlayerCommands( + Player.Commands.Builder() + .add(Player.COMMAND_PLAY_PAUSE) + .add(Player.COMMAND_SEEK_TO_NEXT) + .add(Player.COMMAND_SEEK_TO_PREVIOUS) + .build() + ) + return builder.build() + } + }) + .build() + } + + private fun deinitializeMediaSession() { + mediaSession?.release() + mediaSession = null + player?.release() + player = null + } + + private fun startForegroundService() { + try { + val intent = Intent(activity, MediaForegroundService::class.java).apply { + action = MediaForegroundService.ACTION_START + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + activity.startForegroundService(intent) + } else { + activity.startService(intent) + } + Log.d(TAG, "Foreground service started") + } catch (e: Exception) { + Log.e(TAG, "Failed to start foreground service", e) + } + } + + private fun stopForegroundService() { + try { + val intent = Intent(activity, MediaForegroundService::class.java).apply { + action = MediaForegroundService.ACTION_STOP + } + activity.stopService(intent) + Log.d(TAG, "Foreground service stopped") + } catch (e: Exception) { + Log.e(TAG, "Failed to stop foreground service", e) + } + } + + private suspend fun loadArtworkFromUrl(urlString: String): Bitmap? { + return withContext(Dispatchers.IO) { + try { + when { + urlString.startsWith("data:image/") -> { + val base64Data = urlString.substringAfter("base64,") + val decodedBytes = android.util.Base64.decode(base64Data, android.util.Base64.DEFAULT) + BitmapFactory.decodeByteArray(decodedBytes, 0, decodedBytes.size) + } + urlString.startsWith("http") -> { + val url = URL(urlString) + val input: java.io.InputStream = url.openStream() + BitmapFactory.decodeStream(input) + } + else -> { + val assetPath = urlString.removePrefix("/") + val inputStream = activity.assets.open(assetPath) + BitmapFactory.decodeStream(inputStream) + } + } + } catch (e: Exception) { + null + } + } + } + + private var currentArtworkByteArray: ByteArray? = null + + @Command + fun update_media_session_metadata(invoke: Invoke) { + val args = invoke.parseArgs(UpdateMediaSessionMetadataArgs::class.java) + val title = args.title ?: "" + val artist = args.artist ?: "" + val album = args.album ?: "" + val artworkUrl = args.artwork ?: "" + + coroutineScope.launch { + try { + val metadataBuilder = MediaMetadata.Builder() + .setTitle(title) + .setArtist(artist) + .setAlbumTitle(album) + .apply { + if (artworkUrl.isNotEmpty()) { + loadArtworkFromUrl(artworkUrl)?.let { bitmap -> + java.io.ByteArrayOutputStream().use { stream -> + bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream) + currentArtworkByteArray = stream.toByteArray() + } + } + } + currentArtworkByteArray?.let { + setArtworkData(it, MediaMetadata.PICTURE_TYPE_FRONT_COVER) + } + } + + val mediaItem: MediaItem = MediaItem.Builder() + .setMediaId("silent_media") + .setUri("data:audio/wav;base64,UklGRigAAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQQAAAAAAA==") + .setMediaMetadata(metadataBuilder.build()) + .build() + + player?.setMediaItems(listOf(mediaItem, mediaItem)) + player?.prepare() + player?.playWhenReady = true + player?.setRepeatMode(Player.REPEAT_MODE_ALL) + + invoke.resolve() + } catch (e: Exception) { + Log.e("NativeBridgePlugin", "Failed to update metadata: ${e.message}") + invoke.reject("Failed to update metadata: ${e.message}") + } + } + } + + @Command + fun update_media_session_state(invoke: Invoke) { + var args = invoke.parseArgs(UpdateMediaSessionStateArgs::class.java) + val isPlaying = args.playing ?: false + val position = args.position ?: 0.0 + val duration = args.duration ?: 0.0 + + try { + player?.let { player -> + if (position > 0) { + player.seekTo((position * 1000).toLong()) + } + player.playWhenReady = isPlaying + } + invoke.resolve() + } catch (e: Exception) { + invoke.reject("Failed to update playback state: ${e.message}") + } + } + + @Command + fun set_media_session_active(invoke: Invoke) { + var args = invoke.parseArgs(SetMediaSessionActiveArgs::class.java) + val active = args.active ?: true + + try { + if (active && !isMediaSessionActive) { + initializeMediaSession() + startForegroundService() + isMediaSessionActive = true + Log.d(TAG, "Media session activated with foreground service") + } else if (!active && isMediaSessionActive) { + deinitializeMediaSession() + stopForegroundService() + isMediaSessionActive = false + Log.d(TAG, "Media session deactivated and foreground service stopped") + } + invoke.resolve() + } catch (e: Exception) { + Log.e(TAG, "Failed to set media session active state: ${e.message}") + invoke.reject("Failed to set media session active state: ${e.message}") + } + } fun destroy() { - coroutineScope.cancel() - textToSpeech?.shutdown() - eventChannels.values.forEach { it.close() } - eventChannels.clear() - speakingJobs.values.forEach { it.cancel() } - speakingJobs.clear() + try { + if (isMediaSessionActive) { + stopForegroundService() + isMediaSessionActive = false + } + + deinitializeMediaSession() + + coroutineScope.cancel() + textToSpeech?.shutdown() + eventChannels.values.forEach { it.close() } + eventChannels.clear() + speakingJobs.values.forEach { it.cancel() } + speakingJobs.clear() + + MediaForegroundService.pluginInstance = null + + Log.d(TAG, "Plugin destroyed successfully") + } catch (e: Exception) { + Log.e(TAG, "Error during plugin destruction", e) + } } } diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/build.rs b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/build.rs index a7d2cd81..117a1bec 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/build.rs +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/build.rs @@ -8,6 +8,11 @@ const COMMANDS: &[&str] = &[ "set_pitch", "set_voice", "get_all_voices", + "set_media_session_active", + "update_media_session_state", + "update_media_session_metadata", + "checkPermissions", + "requestPermissions", "registerListener", "remove_listener", ]; diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/checkPermissions.toml b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/checkPermissions.toml new file mode 100644 index 00000000..dd209449 --- /dev/null +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/checkPermissions.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-checkPermissions" +description = "Enables the checkPermissions command without any pre-configured scope." +commands.allow = ["checkPermissions"] + +[[permission]] +identifier = "deny-checkPermissions" +description = "Denies the checkPermissions command without any pre-configured scope." +commands.deny = ["checkPermissions"] diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/requestPermissions.toml b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/requestPermissions.toml new file mode 100644 index 00000000..b3238d63 --- /dev/null +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/requestPermissions.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-requestPermissions" +description = "Enables the requestPermissions command without any pre-configured scope." +commands.allow = ["requestPermissions"] + +[[permission]] +identifier = "deny-requestPermissions" +description = "Denies the requestPermissions command without any pre-configured scope." +commands.deny = ["requestPermissions"] diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/set_media_session_active.toml b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/set_media_session_active.toml new file mode 100644 index 00000000..8a8df3ba --- /dev/null +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/set_media_session_active.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-set-media-session-active" +description = "Enables the set_media_session_active command without any pre-configured scope." +commands.allow = ["set_media_session_active"] + +[[permission]] +identifier = "deny-set-media-session-active" +description = "Denies the set_media_session_active command without any pre-configured scope." +commands.deny = ["set_media_session_active"] diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/update_media_session_metadata.toml b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/update_media_session_metadata.toml new file mode 100644 index 00000000..1f10016d --- /dev/null +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/update_media_session_metadata.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-update-media-session-metadata" +description = "Enables the update_media_session_metadata command without any pre-configured scope." +commands.allow = ["update_media_session_metadata"] + +[[permission]] +identifier = "deny-update-media-session-metadata" +description = "Denies the update_media_session_metadata command without any pre-configured scope." +commands.deny = ["update_media_session_metadata"] diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/update_media_session_state.toml b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/update_media_session_state.toml new file mode 100644 index 00000000..cecf1fae --- /dev/null +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/commands/update_media_session_state.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-update-media-session-state" +description = "Enables the update_media_session_state command without any pre-configured scope." +commands.allow = ["update_media_session_state"] + +[[permission]] +identifier = "deny-update-media-session-state" +description = "Denies the update_media_session_state command without any pre-configured scope." +commands.deny = ["update_media_session_state"] diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/reference.md b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/reference.md index 080bd489..384323df 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/reference.md +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/autogenerated/reference.md @@ -13,6 +13,11 @@ Default permissions for the plugin - `allow-set-pitch` - `allow-set-voice` - `allow-get-all-voices` +- `allow-set-media-session-active` +- `allow-update-media-session-state` +- `allow-update-media-session-metadata` +- `allow-checkPermissions` +- `allow-requestPermissions` - `allow-registerListener` - `allow-remove-listener` @@ -25,6 +30,32 @@ Default permissions for the plugin + + + +`native-tts:allow-checkPermissions` + + + + +Enables the checkPermissions command without any pre-configured scope. + + + + + + + +`native-tts:deny-checkPermissions` + + + + +Denies the checkPermissions command without any pre-configured scope. + + + + @@ -158,6 +189,32 @@ Denies the remove_listener command without any pre-configured scope. +`native-tts:allow-requestPermissions` + + + + +Enables the requestPermissions command without any pre-configured scope. + + + + + + + +`native-tts:deny-requestPermissions` + + + + +Denies the requestPermissions command without any pre-configured scope. + + + + + + + `native-tts:allow-resume` @@ -184,6 +241,32 @@ Denies the resume command without any pre-configured scope. +`native-tts:allow-set-media-session-active` + + + + +Enables the set_media_session_active command without any pre-configured scope. + + + + + + + +`native-tts:deny-set-media-session-active` + + + + +Denies the set_media_session_active command without any pre-configured scope. + + + + + + + `native-tts:allow-set-pitch` @@ -308,6 +391,58 @@ Enables the stop command without any pre-configured scope. Denies the stop command without any pre-configured scope. + + + + + + +`native-tts:allow-update-media-session-metadata` + + + + +Enables the update_media_session_metadata command without any pre-configured scope. + + + + + + + +`native-tts:deny-update-media-session-metadata` + + + + +Denies the update_media_session_metadata command without any pre-configured scope. + + + + + + + +`native-tts:allow-update-media-session-state` + + + + +Enables the update_media_session_state command without any pre-configured scope. + + + + + + + +`native-tts:deny-update-media-session-state` + + + + +Denies the update_media_session_state command without any pre-configured scope. + diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/default.toml b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/default.toml index 7c2de83c..210dbb9f 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/default.toml +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/default.toml @@ -10,6 +10,11 @@ permissions = [ "allow-set-pitch", "allow-set-voice", "allow-get-all-voices", + "allow-set-media-session-active", + "allow-update-media-session-state", + "allow-update-media-session-metadata", + "allow-checkPermissions", + "allow-requestPermissions", "allow-registerListener", "allow-remove-listener", ] diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/schemas/schema.json b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/schemas/schema.json index 4cf53bc9..d384d3ad 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/schemas/schema.json +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/permissions/schemas/schema.json @@ -294,6 +294,18 @@ "PermissionKind": { "type": "string", "oneOf": [ + { + "description": "Enables the checkPermissions command without any pre-configured scope.", + "type": "string", + "const": "allow-checkPermissions", + "markdownDescription": "Enables the checkPermissions command without any pre-configured scope." + }, + { + "description": "Denies the checkPermissions command without any pre-configured scope.", + "type": "string", + "const": "deny-checkPermissions", + "markdownDescription": "Denies the checkPermissions command without any pre-configured scope." + }, { "description": "Enables the get_all_voices command without any pre-configured scope.", "type": "string", @@ -354,6 +366,18 @@ "const": "deny-remove-listener", "markdownDescription": "Denies the remove_listener command without any pre-configured scope." }, + { + "description": "Enables the requestPermissions command without any pre-configured scope.", + "type": "string", + "const": "allow-requestPermissions", + "markdownDescription": "Enables the requestPermissions command without any pre-configured scope." + }, + { + "description": "Denies the requestPermissions command without any pre-configured scope.", + "type": "string", + "const": "deny-requestPermissions", + "markdownDescription": "Denies the requestPermissions command without any pre-configured scope." + }, { "description": "Enables the resume command without any pre-configured scope.", "type": "string", @@ -366,6 +390,18 @@ "const": "deny-resume", "markdownDescription": "Denies the resume command without any pre-configured scope." }, + { + "description": "Enables the set_media_session_active command without any pre-configured scope.", + "type": "string", + "const": "allow-set-media-session-active", + "markdownDescription": "Enables the set_media_session_active command without any pre-configured scope." + }, + { + "description": "Denies the set_media_session_active command without any pre-configured scope.", + "type": "string", + "const": "deny-set-media-session-active", + "markdownDescription": "Denies the set_media_session_active command without any pre-configured scope." + }, { "description": "Enables the set_pitch command without any pre-configured scope.", "type": "string", @@ -427,10 +463,34 @@ "markdownDescription": "Denies the stop command without any pre-configured scope." }, { - "description": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-init`\n- `allow-speak`\n- `allow-stop`\n- `allow-pause`\n- `allow-resume`\n- `allow-set-rate`\n- `allow-set-pitch`\n- `allow-set-voice`\n- `allow-get-all-voices`\n- `allow-registerListener`\n- `allow-remove-listener`", + "description": "Enables the update_media_session_metadata command without any pre-configured scope.", + "type": "string", + "const": "allow-update-media-session-metadata", + "markdownDescription": "Enables the update_media_session_metadata command without any pre-configured scope." + }, + { + "description": "Denies the update_media_session_metadata command without any pre-configured scope.", + "type": "string", + "const": "deny-update-media-session-metadata", + "markdownDescription": "Denies the update_media_session_metadata command without any pre-configured scope." + }, + { + "description": "Enables the update_media_session_state command without any pre-configured scope.", + "type": "string", + "const": "allow-update-media-session-state", + "markdownDescription": "Enables the update_media_session_state command without any pre-configured scope." + }, + { + "description": "Denies the update_media_session_state command without any pre-configured scope.", + "type": "string", + "const": "deny-update-media-session-state", + "markdownDescription": "Denies the update_media_session_state command without any pre-configured scope." + }, + { + "description": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-init`\n- `allow-speak`\n- `allow-stop`\n- `allow-pause`\n- `allow-resume`\n- `allow-set-rate`\n- `allow-set-pitch`\n- `allow-set-voice`\n- `allow-get-all-voices`\n- `allow-set-media-session-active`\n- `allow-update-media-session-state`\n- `allow-update-media-session-metadata`\n- `allow-checkPermissions`\n- `allow-requestPermissions`\n- `allow-registerListener`\n- `allow-remove-listener`", "type": "string", "const": "default", - "markdownDescription": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-init`\n- `allow-speak`\n- `allow-stop`\n- `allow-pause`\n- `allow-resume`\n- `allow-set-rate`\n- `allow-set-pitch`\n- `allow-set-voice`\n- `allow-get-all-voices`\n- `allow-registerListener`\n- `allow-remove-listener`" + "markdownDescription": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-init`\n- `allow-speak`\n- `allow-stop`\n- `allow-pause`\n- `allow-resume`\n- `allow-set-rate`\n- `allow-set-pitch`\n- `allow-set-voice`\n- `allow-get-all-voices`\n- `allow-set-media-session-active`\n- `allow-update-media-session-state`\n- `allow-update-media-session-metadata`\n- `allow-checkPermissions`\n- `allow-requestPermissions`\n- `allow-registerListener`\n- `allow-remove-listener`" } ] } diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/commands.rs b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/commands.rs index d7978b42..eee1376c 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/commands.rs +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/commands.rs @@ -51,3 +51,27 @@ pub(crate) async fn set_voice(app: AppHandle, payload: SetVoiceAr pub(crate) async fn get_all_voices(app: AppHandle) -> Result { app.native_tts().get_all_voices() } + +#[command] +pub(crate) async fn set_media_session_active( + app: AppHandle, + payload: SetMediaSessionActiveRequest, +) -> Result<()> { + app.native_tts().set_media_session_active(payload) +} + +#[command] +pub(crate) async fn update_media_session_state( + app: AppHandle, + payload: UpdateMediaSessionStateRequest, +) -> Result<()> { + app.native_tts().update_media_session_state(payload) +} + +#[command] +pub(crate) async fn update_media_session_metadata( + app: AppHandle, + payload: UpdateMediaSessionMetadataRequest, +) -> Result<()> { + app.native_tts().update_media_session_metadata(payload) +} diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/desktop.rs b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/desktop.rs index 42fe356c..3931e8d1 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/desktop.rs +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/desktop.rs @@ -41,4 +41,22 @@ impl NativeTts { pub fn get_all_voices(&self) -> crate::Result { Err(crate::Error::UnsupportedPlatformError) } + pub fn set_media_session_active( + &self, + _payload: SetMediaSessionActiveRequest, + ) -> crate::Result<()> { + Err(crate::Error::UnsupportedPlatformError) + } + pub fn update_media_session_state( + &self, + _payload: UpdateMediaSessionStateRequest, + ) -> crate::Result<()> { + Err(crate::Error::UnsupportedPlatformError) + } + pub fn update_media_session_metadata( + &self, + _payload: UpdateMediaSessionMetadataRequest, + ) -> crate::Result<()> { + Err(crate::Error::UnsupportedPlatformError) + } } diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/lib.rs b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/lib.rs index 20045008..ea01b1af 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/lib.rs +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/lib.rs @@ -45,6 +45,9 @@ pub fn init() -> TauriPlugin { commands::set_pitch, commands::set_voice, commands::get_all_voices, + commands::set_media_session_active, + commands::update_media_session_state, + commands::update_media_session_metadata, ]) .setup(|app, api| { #[cfg(mobile)] diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/mobile.rs b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/mobile.rs index fcc181c6..f6ee4644 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/mobile.rs +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/mobile.rs @@ -87,3 +87,36 @@ impl NativeTts { .map_err(Into::into) } } + +impl NativeTts { + pub fn set_media_session_active( + &self, + payload: SetMediaSessionActiveRequest, + ) -> crate::Result<()> { + self.0 + .run_mobile_plugin("set_media_session_active", payload) + .map_err(Into::into) + } +} + +impl NativeTts { + pub fn update_media_session_state( + &self, + payload: UpdateMediaSessionStateRequest, + ) -> crate::Result<()> { + self.0 + .run_mobile_plugin("update_media_session_state", payload) + .map_err(Into::into) + } +} + +impl NativeTts { + pub fn update_media_session_metadata( + &self, + payload: UpdateMediaSessionMetadataRequest, + ) -> crate::Result<()> { + self.0 + .run_mobile_plugin("update_media_session_metadata", payload) + .map_err(Into::into) + } +} diff --git a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/models.rs b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/models.rs index 8689673a..e0fa88f9 100644 --- a/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/models.rs +++ b/apps/readest-app/src-tauri/plugins/tauri-plugin-native-tts/src/models.rs @@ -61,3 +61,26 @@ pub struct SetVoiceArgs { pub struct GetVoicesResponse { pub voices: Vec, } + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SetMediaSessionActiveRequest { + pub active: bool, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UpdateMediaSessionStateRequest { + pub playing: bool, + pub position: Option, + pub duration: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UpdateMediaSessionMetadataRequest { + pub title: Option, + pub artist: Option, + pub album: Option, + pub artwork: Option, +} diff --git a/apps/readest-app/src/app/library/components/SettingsMenu.tsx b/apps/readest-app/src/app/library/components/SettingsMenu.tsx index 5f375fb9..7e78084e 100644 --- a/apps/readest-app/src/app/library/components/SettingsMenu.tsx +++ b/apps/readest-app/src/app/library/components/SettingsMenu.tsx @@ -6,6 +6,7 @@ import { PiUserCircleCheck } from 'react-icons/pi'; import { TbSunMoon } from 'react-icons/tb'; import { BiMoon, BiSun } from 'react-icons/bi'; +import { invoke, PermissionState } from '@tauri-apps/api/core'; import { isTauriAppPlatform, isWebAppPlatform } from '@/services/environment'; import { DOWNLOAD_READEST_URL } from '@/services/constants'; import { useAuth } from '@/context/AuthContext'; @@ -29,6 +30,10 @@ interface SettingsMenuProps { setIsDropdownOpen?: (isOpen: boolean) => void; } +interface Permissions { + postNotification: PermissionState; +} + const SettingsMenu: React.FC = ({ setIsDropdownOpen }) => { const _ = useTranslation(); const router = useRouter(); @@ -47,6 +52,7 @@ const SettingsMenu: React.FC = ({ setIsDropdownOpen }) => { settings.autoImportBooksOnOpen, ); const [isTelemetryEnabled, setIsTelemetryEnabled] = useState(settings.telemetryEnabled); + const [alwaysInForeground, setAlwaysInForeground] = useState(settings.alwaysInForeground); const iconSize = useResponsiveSize(16); const showAboutReadest = () => { @@ -168,6 +174,25 @@ const SettingsMenu: React.FC = ({ setIsDropdownOpen }) => { setIsDropdownOpen?.(false); }; + const toggleAlwaysInForeground = async () => { + const requestAlwaysInForeground = !settings.alwaysInForeground; + + if (requestAlwaysInForeground) { + let permission = await invoke('plugin:native-tts|checkPermissions'); + if (permission.postNotification !== 'granted') { + permission = await invoke('plugin:native-tts|requestPermissions', { + permissions: ['postNotification'], + }); + } + if (permission.postNotification !== 'granted') return; + } + + settings.alwaysInForeground = requestAlwaysInForeground; + setSettings(settings); + saveSettings(envConfig, settings); + setAlwaysInForeground(settings.alwaysInForeground); + }; + const avatarUrl = user?.user_metadata?.['picture'] || user?.user_metadata?.['avatar_url']; const userFullName = user?.user_metadata?.['full_name']; const userDisplayName = userFullName ? userFullName.split(' ')[0] : null; @@ -262,6 +287,13 @@ const SettingsMenu: React.FC = ({ setIsDropdownOpen }) => { toggled={isScreenWakeLock} onClick={toggleScreenWakeLock} /> + {appService?.isAndroidApp && ( + + )} = ({ bookKey, gridInsets }) => { const _ = useTranslation(); const { appService } = useEnv(); const { safeAreaInsets } = useThemeStore(); + const { settings } = useSettingsStore(); const { getBookData } = useBookDataStore(); const { hoveredBookKey, getView, getProgress, getViewSettings } = useReaderStore(); const { setViewSettings, setTTSEnabled } = useReaderStore(); @@ -58,6 +62,7 @@ const TTSControl: React.FC = ({ bookKey, gridInsets }) => { const iconRef = useRef(null); const unblockerAudioRef = useRef(null); const ttsControllerRef = useRef(null); + const mediaSessionRef = useRef(null); const [ttsController, setTtsController] = useState(null); const [ttsClientsInited, setTtsClientsInitialized] = useState(false); @@ -92,6 +97,43 @@ const TTSControl: React.FC = ({ bookKey, gridInsets }) => { } }; + const initMediaSession = async () => { + const mediaSession = getMediaSession(); + if (!mediaSession) return; + + mediaSessionRef.current = mediaSession; + + if (mediaSession instanceof TauriMediaSession) { + const bookData = getBookData(bookKey); + const progress = getProgress(bookKey); + if (!bookData || !bookData.book) return; + const { title, author, coverImageUrl } = bookData.book; + const { sectionLabel } = progress || {}; + + let artworkImage = '/icon.png'; + try { + artworkImage = await fetchImageAsBase64(coverImageUrl || '/icon.png'); + } catch { + artworkImage = await fetchImageAsBase64('/icon.png'); + } + + await mediaSession.setActive(true, settings.alwaysInForeground); + mediaSession.updateMetadata({ + title: title, + artist: sectionLabel || title, + album: author, + artwork: artworkImage, + }); + } + }; + + const deinitMediaSession = async () => { + if (mediaSessionRef.current && mediaSessionRef.current instanceof TauriMediaSession) { + await mediaSessionRef.current.setActive(false); + } + mediaSessionRef.current = null; + }; + useEffect(() => { return () => { if (ttsControllerRef.current) { @@ -121,13 +163,24 @@ const TTSControl: React.FC = ({ bookKey, gridInsets }) => { const progress = getProgress(bookKey); const { sectionLabel } = progress || {}; const mark = (e as CustomEvent).detail; - if ('mediaSession' in navigator) { - navigator.mediaSession.metadata = new MediaMetadata({ - title: mark?.text || '', - artist: sectionLabel || title, - album: author, - artwork: [{ src: coverImageUrl || '/icon.png', sizes: '512x512', type: 'image/png' }], - }); + + if (mediaSessionRef.current) { + const mediaSession = mediaSessionRef.current; + if (mediaSession instanceof TauriMediaSession) { + mediaSession.updateMetadata({ + title: mark?.text || '', + artist: sectionLabel || title, + album: author, + artwork: '', + }); + } else { + mediaSession.metadata = new MediaMetadata({ + title: mark?.text || '', + artist: sectionLabel || title, + album: author, + artwork: [{ src: coverImageUrl || '/icon.png', sizes: '512x512', type: 'image/png' }], + }); + } } }; @@ -205,7 +258,10 @@ const TTSControl: React.FC = ({ bookKey, gridInsets }) => { if (appService?.isMobile) { unblockAudio(); } + await initMediaSession(); setTtsClientsInitialized(false); + + setShowIndicator(true); const ttsController = new TTSController(appService, view); await ttsController.init(); await ttsController.initViewTTS(); @@ -223,7 +279,6 @@ const TTSControl: React.FC = ({ bookKey, gridInsets }) => { } setTtsClientsInitialized(true); setTTSEnabled(bookKey, true); - setShowIndicator(true); } catch (error) { eventDispatcher.dispatch('toast', { message: _('TTS not supported for this document'), @@ -302,6 +357,7 @@ const TTSControl: React.FC = ({ bookKey, gridInsets }) => { if (appService?.isMobile) { releaseUnblockAudio(); } + await deinitMediaSession(); setTTSEnabled(bookKey, false); }, // eslint-disable-next-line react-hooks/exhaustive-deps @@ -425,32 +481,33 @@ const TTSControl: React.FC = ({ bookKey, gridInsets }) => { }; useEffect(() => { - if ('mediaSession' in navigator) { - navigator.mediaSession.setActionHandler('play', () => { + const { current: mediaSession } = mediaSessionRef; + if (mediaSession) { + mediaSession.setActionHandler('play', () => { handleTogglePlay(); }); - navigator.mediaSession.setActionHandler('pause', () => { + mediaSession.setActionHandler('pause', () => { handleTogglePlay(); }); - navigator.mediaSession.setActionHandler('stop', () => { + mediaSession.setActionHandler('stop', () => { handlePause(); }); - navigator.mediaSession.setActionHandler('seekforward', () => { + mediaSession.setActionHandler('seekforward', () => { handleForward(true); }); - navigator.mediaSession.setActionHandler('seekbackward', () => { + mediaSession.setActionHandler('seekbackward', () => { handleBackward(true); }); - navigator.mediaSession.setActionHandler('nexttrack', () => { + mediaSession.setActionHandler('nexttrack', () => { handleForward(); }); - navigator.mediaSession.setActionHandler('previoustrack', () => { + mediaSession.setActionHandler('previoustrack', () => { handleBackward(); }); } diff --git a/apps/readest-app/src/app/reader/hooks/useTextTranslation.ts b/apps/readest-app/src/app/reader/hooks/useTextTranslation.ts index 40252bed..02a265b4 100644 --- a/apps/readest-app/src/app/reader/hooks/useTextTranslation.ts +++ b/apps/readest-app/src/app/reader/hooks/useTextTranslation.ts @@ -263,7 +263,7 @@ export function useTextTranslation( ); useEffect(() => { - if (viewState?.ttsEnabled && progress && document.hidden) { + if (enabled.current && viewState?.ttsEnabled && progress && document.hidden) { const { range } = progress; translateInRange(range); } diff --git a/apps/readest-app/src/libs/mediaSession.ts b/apps/readest-app/src/libs/mediaSession.ts new file mode 100644 index 00000000..283070e7 --- /dev/null +++ b/apps/readest-app/src/libs/mediaSession.ts @@ -0,0 +1,135 @@ +import { isTauriAppPlatform } from '@/services/environment'; +import { getOSPlatform } from '@/utils/misc'; +import { invoke } from '@tauri-apps/api/core'; +import { addPluginListener, PluginListener, PermissionState } from '@tauri-apps/api/core'; + +export interface MediaMetadata { + title?: string; + artist?: string; + album?: string; + artwork?: string; +} + +export interface PlaybackState { + playing: boolean; + position: number; + duration: number; +} + +interface Permissions { + postNotification: PermissionState; +} + +export class TauriMediaSession { + private handlers: { [key: string]: (() => void) | ((position: number) => void) } = {}; + private eventListenerInited: boolean = false; + private eventListeners: PluginListener[] = []; + + private async initializeListeners(keepAppInForeground: boolean) { + if (keepAppInForeground) { + const permission = await invoke('plugin:native-tts|checkPermissions'); + if (permission.postNotification.startsWith('prompt')) { + await invoke('plugin:native-tts|requestPermissions', { + permissions: ['postNotification'], + }); + } + } + + if (this.eventListenerInited) return; + this.eventListenerInited = true; + + const playListener = await addPluginListener('native-tts', 'media-session-play', () => { + if (this.handlers['play']) { + (this.handlers['play'] as () => void)(); + } + }); + this.eventListeners.push(playListener); + + const pauseListener = await addPluginListener('native-tts', 'media-session-pause', () => { + if (this.handlers['pause']) { + (this.handlers['pause'] as () => void)(); + } + }); + this.eventListeners.push(pauseListener); + + const nextListener = await addPluginListener('native-tts', 'media-session-next', () => { + if (this.handlers['nexttrack']) { + (this.handlers['nexttrack'] as () => void)(); + } + }); + this.eventListeners.push(nextListener); + + const previousListener = await addPluginListener('native-tts', 'media-session-previous', () => { + if (this.handlers['previoustrack']) { + (this.handlers['previoustrack'] as () => void)(); + } + }); + this.eventListeners.push(previousListener); + + const seekListener = await addPluginListener( + 'native-tts', + 'media-session-seek', + (event: { payload: { position: number } }) => { + const position = event.payload.position; + if (this.handlers['seekto']) { + (this.handlers['seekto'] as (position: number) => void)(position); + } + }, + ); + this.eventListeners.push(seekListener); + } + + private async cleanupListeners() { + for (const listener of this.eventListeners) { + await listener.unregister(); + } + this.eventListeners = []; + this.eventListenerInited = false; + } + + async updateMetadata(metadata: MediaMetadata) { + try { + await invoke('plugin:native-tts|update_media_session_metadata', { payload: metadata }); + } catch (error) { + console.error('Failed to update media metadata:', error); + } + } + + async updatePlaybackState(state: PlaybackState) { + try { + await invoke('plugin:native-tts|update_media_session_state', { payload: state }); + } catch (error) { + console.error('Failed to update playback state:', error); + } + } + + async setActive(active: boolean, keepAppInForeground: boolean = false) { + try { + if (active) { + await this.initializeListeners(keepAppInForeground); + } else { + await this.cleanupListeners(); + } + await invoke('plugin:native-tts|set_media_session_active', { payload: { active } }); + } catch (error) { + console.error('Failed to set media session active state:', error); + } + } + + setActionHandler(action: string, handler: (() => void) | ((position: number) => void) | null) { + if (handler) { + this.handlers[action] = handler; + } else { + delete this.handlers[action]; + } + } +} + +export function getMediaSession() { + if ('mediaSession' in navigator) { + return navigator.mediaSession; + } else if (getOSPlatform() === 'android' && isTauriAppPlatform()) { + return new TauriMediaSession(); + } + return null; +} diff --git a/apps/readest-app/src/services/constants.ts b/apps/readest-app/src/services/constants.ts index 67c22f62..1735a843 100644 --- a/apps/readest-app/src/services/constants.ts +++ b/apps/readest-app/src/services/constants.ts @@ -57,6 +57,7 @@ export const DEFAULT_SYSTEM_SETTINGS: Partial = { alwaysOnTop: false, openBookInNewWindow: true, alwaysShowStatusBar: false, + alwaysInForeground: false, autoCheckUpdates: true, screenWakeLock: false, openLastBooks: false, diff --git a/apps/readest-app/src/types/settings.ts b/apps/readest-app/src/types/settings.ts index 1fa6b5ea..71b4832a 100644 --- a/apps/readest-app/src/types/settings.ts +++ b/apps/readest-app/src/types/settings.ts @@ -47,6 +47,7 @@ export interface SystemSettings { autoCheckUpdates: boolean; screenWakeLock: boolean; alwaysShowStatusBar: boolean; + alwaysInForeground: boolean; openLastBooks: boolean; lastOpenBooks: string[]; autoImportBooksOnOpen: boolean; diff --git a/apps/readest-app/src/utils/image.ts b/apps/readest-app/src/utils/image.ts new file mode 100644 index 00000000..7cf60004 --- /dev/null +++ b/apps/readest-app/src/utils/image.ts @@ -0,0 +1,74 @@ +export async function fetchImageAsBase64( + url: string, + options: { + targetWidth?: number; + format?: 'image/jpeg' | 'image/png' | 'image/webp'; + quality?: number; + } = {}, +): Promise { + const { targetWidth = 256, format = 'image/jpeg', quality = 0.85 } = options; + + try { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to fetch image: ${response.status} ${response.statusText}`); + } + const blob = await response.blob(); + + const img = new Image(); + img.crossOrigin = 'anonymous'; + + return new Promise((resolve, reject) => { + img.onload = () => { + try { + const aspectRatio = img.height / img.width; + const newWidth = targetWidth; + const newHeight = Math.round(newWidth * aspectRatio); + + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + + if (!ctx) { + reject(new Error('Failed to get canvas context')); + return; + } + + canvas.width = newWidth; + canvas.height = newHeight; + + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = 'high'; + + ctx.drawImage(img, 0, 0, newWidth, newHeight); + + const base64 = canvas.toDataURL(format, quality); + resolve(base64); + } catch (error) { + reject(new Error(`Failed to scale image: ${error}`)); + } + }; + + img.onerror = () => reject(new Error('Failed to load image for scaling')); + + const objectUrl = URL.createObjectURL(blob); + img.src = objectUrl; + + const cleanup = () => URL.revokeObjectURL(objectUrl); + const originalOnload = img.onload; + const originalOnerror = img.onerror; + + img.onload = function (ev) { + cleanup(); + if (originalOnload) originalOnload.call(this, ev); + }; + + img.onerror = function (ev) { + cleanup(); + if (originalOnerror) originalOnerror.call(this, ev); + }; + }); + } catch (error) { + console.error('Error fetching and encoding image:', error); + throw error; + } +}