feat(tts): add native local iOS TTS (AVSpeechSynthesizer) (#4697)
Implement on-device iOS text-to-speech using AVSpeechSynthesizer, mirroring the Android native TextToSpeech plugin so the shared NativeTTSClient drives both platforms through the same command and tts_events contract. - Swift NativeTTSPlugin: speak/stop/pause/resume/rate/pitch/voice and voice enumeration, with region-disambiguated duplicate voice names and a small preUtteranceDelay to avoid first-word clipping. - Enable the native TTS client on iOS in TTSController. - Make TTS teardown resilient: reset UI state up front and tear down the controller, media session, and background audio in parallel so a slow native shutdown can never leave the TTS icon or lock-screen session stuck on. - Keep iOS on navigator.mediaSession for the lock screen (Android uses the native foreground service), which restores the Edge TTS cover and current-sentence metadata. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+475
-8
@@ -1,16 +1,483 @@
|
||||
import SwiftRs
|
||||
import AVFoundation
|
||||
import MediaPlayer
|
||||
import Tauri
|
||||
import UIKit
|
||||
import WebKit
|
||||
|
||||
class PingArgs: Decodable {
|
||||
let value: String?
|
||||
// MARK: - Command arguments (camelCase, decoded from the Rust models)
|
||||
|
||||
class SpeakArgs: Decodable {
|
||||
let text: String?
|
||||
let preload: Bool?
|
||||
}
|
||||
|
||||
class NativeTTSPlugin: Plugin {
|
||||
@objc public func ping(_ invoke: Invoke) throws {
|
||||
let args = try invoke.parseArgs(PingArgs.self)
|
||||
invoke.resolve(["value": args.value ?? ""])
|
||||
class SetRateArgs: Decodable {
|
||||
let rate: Float?
|
||||
}
|
||||
|
||||
class SetPitchArgs: Decodable {
|
||||
let pitch: Float?
|
||||
}
|
||||
|
||||
class SetVoiceArgs: Decodable {
|
||||
let voice: String?
|
||||
}
|
||||
|
||||
class UpdateMediaSessionMetadataArgs: Decodable {
|
||||
let title: String?
|
||||
let artist: String?
|
||||
let album: String?
|
||||
let artwork: String?
|
||||
}
|
||||
|
||||
class UpdateMediaSessionStateArgs: Decodable {
|
||||
let playing: Bool?
|
||||
let position: Double? // milliseconds
|
||||
let duration: Double? // milliseconds
|
||||
}
|
||||
|
||||
class SetMediaSessionActiveArgs: Decodable {
|
||||
let active: Bool?
|
||||
let keepAppInForeground: Bool?
|
||||
let notificationTitle: String?
|
||||
let notificationText: String?
|
||||
let foregroundServiceTitle: String?
|
||||
let foregroundServiceText: String?
|
||||
}
|
||||
|
||||
// MARK: - Command responses (camelCase, re-decoded by the Rust models)
|
||||
|
||||
struct InitResponse: Encodable {
|
||||
let success: Bool
|
||||
}
|
||||
|
||||
struct SpeakResponse: Encodable {
|
||||
let utteranceId: String
|
||||
}
|
||||
|
||||
struct VoiceData: Encodable {
|
||||
let id: String
|
||||
let name: String
|
||||
let lang: String
|
||||
let disabled: Bool
|
||||
}
|
||||
|
||||
struct GetVoicesResponse: Encodable {
|
||||
let voices: [VoiceData]
|
||||
}
|
||||
|
||||
/// Native iOS Text-to-Speech backed by `AVSpeechSynthesizer`, mirroring the
|
||||
/// Android `NativeTTSPlugin` (Android `TextToSpeech`). The shared TypeScript
|
||||
/// `NativeTTSClient` drives both platforms through the same plugin command and
|
||||
/// `tts_events` channel contract.
|
||||
class NativeTTSPlugin: Plugin, AVSpeechSynthesizerDelegate {
|
||||
private let synthesizer = AVSpeechSynthesizer()
|
||||
|
||||
// App-level controls. `rate` arrives pre-curved by the JS client (see
|
||||
// `avRate(from:)`); `pitch` is a direct multiplier (1.0 == normal).
|
||||
private var currentRate: Float = 1.0
|
||||
private var currentPitch: Float = 1.0
|
||||
private var currentVoiceId: String = ""
|
||||
|
||||
// Maps a live utterance to the UUID the JS client awaits, so delegate
|
||||
// callbacks can route `tts_events` to the right async iterator.
|
||||
private var utteranceIds = [ObjectIdentifier: String]()
|
||||
|
||||
// Remote command targets we registered. The lock-screen command center
|
||||
// (`MPRemoteCommandCenter.shared()`) is app-global and also used by
|
||||
// native-bridge's hardware media-key page-turn handler, so we keep the exact
|
||||
// tokens and remove only our own on teardown.
|
||||
private var remoteCommandTargets: [(MPRemoteCommand, Any)] = []
|
||||
private var mediaSessionActive = false
|
||||
|
||||
override init() {
|
||||
super.init()
|
||||
synthesizer.delegate = self
|
||||
}
|
||||
|
||||
// MARK: - Lifecycle
|
||||
|
||||
// The "init" command maps to the Objective-C selector `init:`; `init` is a
|
||||
// Swift reserved word, so the method is named `initialize` and exposed under
|
||||
// the expected selector.
|
||||
@objc(init:)
|
||||
public func initialize(_ invoke: Invoke) {
|
||||
// AVSpeechSynthesizer and its voice list are available synchronously; there
|
||||
// is no engine handshake to await as there is on Android.
|
||||
invoke.resolve(InitResponse(success: true))
|
||||
}
|
||||
|
||||
// MARK: - Speech
|
||||
|
||||
@objc public func speak(_ invoke: Invoke) {
|
||||
do {
|
||||
let args = try invoke.parseArgs(SpeakArgs.self)
|
||||
let text = args.text ?? ""
|
||||
if text.isEmpty {
|
||||
invoke.reject("Text cannot be empty")
|
||||
return
|
||||
}
|
||||
|
||||
let utteranceId = UUID().uuidString
|
||||
let rate = currentRate
|
||||
let pitch = currentPitch
|
||||
let voiceId = currentVoiceId
|
||||
|
||||
// Resolve immediately with the id; events stream over `tts_events`.
|
||||
invoke.resolve(SpeakResponse(utteranceId: utteranceId))
|
||||
|
||||
DispatchQueue.main.async {
|
||||
let utterance = AVSpeechUtterance(string: text)
|
||||
utterance.rate = self.avRate(from: rate)
|
||||
utterance.pitchMultiplier = self.avPitch(from: pitch)
|
||||
// Each sentence is a separate utterance spoken after a gap, so the audio
|
||||
// route goes cold between them and the first word can be clipped. A small
|
||||
// pre-utterance delay plays silence first to warm the route. See #4676.
|
||||
utterance.preUtteranceDelay = 0.1
|
||||
if !voiceId.isEmpty, let voice = AVSpeechSynthesisVoice(identifier: voiceId) {
|
||||
utterance.voice = voice
|
||||
}
|
||||
self.utteranceIds[ObjectIdentifier(utterance)] = utteranceId
|
||||
self.synthesizer.speak(utterance)
|
||||
}
|
||||
} catch {
|
||||
invoke.reject("Failed to start speaking: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
@objc public func pause(_ invoke: Invoke) {
|
||||
// Mirror Android: pause is implemented as stop. The JS client returns
|
||||
// `false` from pause(), so the controller stops and re-speaks the current
|
||||
// sentence on resume.
|
||||
DispatchQueue.main.async {
|
||||
self.synthesizer.stopSpeaking(at: .immediate)
|
||||
}
|
||||
invoke.resolve()
|
||||
}
|
||||
|
||||
@objc public func resume(_ invoke: Invoke) {
|
||||
// No-op, mirroring Android; the controller re-speaks on resume.
|
||||
invoke.resolve()
|
||||
}
|
||||
|
||||
@objc public func stop(_ invoke: Invoke) {
|
||||
DispatchQueue.main.async {
|
||||
self.synthesizer.stopSpeaking(at: .immediate)
|
||||
self.utteranceIds.removeAll()
|
||||
}
|
||||
invoke.resolve()
|
||||
}
|
||||
|
||||
@objc public func set_rate(_ invoke: Invoke) {
|
||||
do {
|
||||
let args = try invoke.parseArgs(SetRateArgs.self)
|
||||
currentRate = args.rate ?? 1.0
|
||||
invoke.resolve()
|
||||
} catch {
|
||||
invoke.reject("Exception setting rate: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
@objc public func set_pitch(_ invoke: Invoke) {
|
||||
do {
|
||||
let args = try invoke.parseArgs(SetPitchArgs.self)
|
||||
currentPitch = args.pitch ?? 1.0
|
||||
invoke.resolve()
|
||||
} catch {
|
||||
invoke.reject("Exception setting pitch: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
@objc public func set_voice(_ invoke: Invoke) {
|
||||
do {
|
||||
let args = try invoke.parseArgs(SetVoiceArgs.self)
|
||||
currentVoiceId = args.voice ?? ""
|
||||
invoke.resolve()
|
||||
} catch {
|
||||
invoke.reject("Exception setting voice: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
@objc public func get_all_voices(_ invoke: Invoke) {
|
||||
let systemVoices = AVSpeechSynthesisVoice.speechVoices()
|
||||
|
||||
// The JS layer groups voices by primary language (isSameLang), so the same
|
||||
// display name can appear twice in one "System TTS" list when a voice exists
|
||||
// in multiple regions of the same language (e.g. the Eloquence "Rocko" in
|
||||
// both en-US and en-GB). Count (primaryLanguage, displayName) pairs and, for
|
||||
// any that collide, append the region so users can tell them apart.
|
||||
var nameCounts: [String: Int] = [:]
|
||||
for voice in systemVoices {
|
||||
let key = "\(primaryLanguage(voice.language))|\(displayName(for: voice))"
|
||||
nameCounts[key, default: 0] += 1
|
||||
}
|
||||
|
||||
let voices = systemVoices.map { voice -> VoiceData in
|
||||
let baseName = displayName(for: voice)
|
||||
let key = "\(primaryLanguage(voice.language))|\(baseName)"
|
||||
let name =
|
||||
(nameCounts[key] ?? 0) > 1
|
||||
? "\(baseName) (\(regionDescription(for: voice.language)))"
|
||||
: baseName
|
||||
return VoiceData(
|
||||
id: voice.identifier,
|
||||
name: name,
|
||||
lang: voice.language,
|
||||
disabled: false
|
||||
)
|
||||
}
|
||||
invoke.resolve(GetVoicesResponse(voices: voices))
|
||||
}
|
||||
|
||||
// MARK: - AVSpeechSynthesizerDelegate
|
||||
|
||||
func speechSynthesizer(
|
||||
_ synthesizer: AVSpeechSynthesizer, didStart utterance: AVSpeechUtterance
|
||||
) {
|
||||
if let id = utteranceIds[ObjectIdentifier(utterance)] {
|
||||
sendEvent(utteranceId: id, code: "boundary", message: "start")
|
||||
}
|
||||
}
|
||||
|
||||
func speechSynthesizer(
|
||||
_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance
|
||||
) {
|
||||
if let id = utteranceIds.removeValue(forKey: ObjectIdentifier(utterance)) {
|
||||
sendEvent(utteranceId: id, code: "end")
|
||||
}
|
||||
}
|
||||
|
||||
func speechSynthesizer(
|
||||
_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance
|
||||
) {
|
||||
// Cancellation comes from stop()/pause(). Do NOT emit "end": the controller
|
||||
// treats a finished utterance as a cue to advance, which must not happen on
|
||||
// a manual stop or pause (mirrors Android, where stop() emits no onDone).
|
||||
utteranceIds.removeValue(forKey: ObjectIdentifier(utterance))
|
||||
}
|
||||
|
||||
// MARK: - Media session (lock screen / now playing)
|
||||
|
||||
@objc public func set_media_session_active(_ invoke: Invoke) {
|
||||
do {
|
||||
let args = try invoke.parseArgs(SetMediaSessionActiveArgs.self)
|
||||
let active = args.active ?? true
|
||||
DispatchQueue.main.async {
|
||||
if active {
|
||||
self.activateRemoteCommands()
|
||||
} else {
|
||||
self.deactivateRemoteCommands()
|
||||
}
|
||||
}
|
||||
invoke.resolve()
|
||||
} catch {
|
||||
invoke.reject("Failed to set media session active state: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
@objc public func update_media_session_state(_ invoke: Invoke) {
|
||||
do {
|
||||
let args = try invoke.parseArgs(UpdateMediaSessionStateArgs.self)
|
||||
let playing = args.playing ?? false
|
||||
let position = args.position
|
||||
let duration = args.duration
|
||||
DispatchQueue.main.async {
|
||||
var info = MPNowPlayingInfoCenter.default().nowPlayingInfo ?? [String: Any]()
|
||||
info[MPNowPlayingInfoPropertyPlaybackRate] = playing ? 1.0 : 0.0
|
||||
if let position = position {
|
||||
info[MPNowPlayingInfoPropertyElapsedPlaybackTime] = position / 1000.0
|
||||
}
|
||||
if let duration = duration, duration > 0 {
|
||||
info[MPMediaItemPropertyPlaybackDuration] = duration / 1000.0
|
||||
}
|
||||
MPNowPlayingInfoCenter.default().nowPlayingInfo = info
|
||||
}
|
||||
invoke.resolve()
|
||||
} catch {
|
||||
invoke.reject("Failed to update playback state: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
@objc public func update_media_session_metadata(_ invoke: Invoke) {
|
||||
do {
|
||||
let args = try invoke.parseArgs(UpdateMediaSessionMetadataArgs.self)
|
||||
let title = args.title ?? ""
|
||||
let artist = args.artist ?? ""
|
||||
let album = args.album ?? ""
|
||||
let artwork = args.artwork
|
||||
|
||||
DispatchQueue.main.async {
|
||||
var info = MPNowPlayingInfoCenter.default().nowPlayingInfo ?? [String: Any]()
|
||||
info[MPMediaItemPropertyTitle] = title
|
||||
info[MPMediaItemPropertyArtist] = artist
|
||||
info[MPMediaItemPropertyAlbumTitle] = album
|
||||
MPNowPlayingInfoCenter.default().nowPlayingInfo = info
|
||||
}
|
||||
|
||||
// Artwork usually arrives as a base64 data URI; decode off the main thread
|
||||
// and apply it once ready so it does not block command handling.
|
||||
if let artwork = artwork {
|
||||
DispatchQueue.global(qos: .userInitiated).async {
|
||||
guard let image = self.loadImage(from: artwork) else { return }
|
||||
let mpArtwork = MPMediaItemArtwork(boundsSize: image.size) { _ in image }
|
||||
DispatchQueue.main.async {
|
||||
var info = MPNowPlayingInfoCenter.default().nowPlayingInfo ?? [String: Any]()
|
||||
info[MPMediaItemPropertyArtwork] = mpArtwork
|
||||
MPNowPlayingInfoCenter.default().nowPlayingInfo = info
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
invoke.resolve()
|
||||
} catch {
|
||||
invoke.reject("Failed to update metadata: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Permissions
|
||||
|
||||
// iOS lock-screen / now-playing needs no runtime permission. Resolve the
|
||||
// Android-style postNotification permission as granted so the shared JS media
|
||||
// session code does not try to prompt.
|
||||
@objc public override func checkPermissions(_ invoke: Invoke) {
|
||||
invoke.resolve(["postNotification": "granted"])
|
||||
}
|
||||
|
||||
@objc public override func requestPermissions(_ invoke: Invoke) {
|
||||
invoke.resolve(["postNotification": "granted"])
|
||||
}
|
||||
|
||||
// MARK: - Helpers
|
||||
|
||||
private func sendEvent(
|
||||
utteranceId: String, code: String, message: String? = nil, mark: String? = nil
|
||||
) {
|
||||
var data: JSObject = ["utteranceId": utteranceId, "code": code]
|
||||
if let message = message {
|
||||
data["message"] = message
|
||||
}
|
||||
if let mark = mark {
|
||||
data["mark"] = mark
|
||||
}
|
||||
trigger("tts_events", data: data)
|
||||
}
|
||||
|
||||
private func activateRemoteCommands() {
|
||||
if mediaSessionActive {
|
||||
return
|
||||
}
|
||||
mediaSessionActive = true
|
||||
let center = MPRemoteCommandCenter.shared()
|
||||
|
||||
center.playCommand.isEnabled = true
|
||||
addRemoteTarget(center.playCommand) { [weak self] _ in
|
||||
self?.triggerMediaSession("media-session-play")
|
||||
return .success
|
||||
}
|
||||
center.pauseCommand.isEnabled = true
|
||||
addRemoteTarget(center.pauseCommand) { [weak self] _ in
|
||||
self?.triggerMediaSession("media-session-pause")
|
||||
return .success
|
||||
}
|
||||
// The lock screen shows a single play/pause button; the controller's "play"
|
||||
// handler toggles, so route the toggle command there too.
|
||||
center.togglePlayPauseCommand.isEnabled = true
|
||||
addRemoteTarget(center.togglePlayPauseCommand) { [weak self] _ in
|
||||
self?.triggerMediaSession("media-session-play")
|
||||
return .success
|
||||
}
|
||||
center.nextTrackCommand.isEnabled = true
|
||||
addRemoteTarget(center.nextTrackCommand) { [weak self] _ in
|
||||
self?.triggerMediaSession("media-session-next")
|
||||
return .success
|
||||
}
|
||||
center.previousTrackCommand.isEnabled = true
|
||||
addRemoteTarget(center.previousTrackCommand) { [weak self] _ in
|
||||
self?.triggerMediaSession("media-session-previous")
|
||||
return .success
|
||||
}
|
||||
}
|
||||
|
||||
private func deactivateRemoteCommands() {
|
||||
for (command, token) in remoteCommandTargets {
|
||||
command.removeTarget(token)
|
||||
}
|
||||
remoteCommandTargets.removeAll()
|
||||
MPNowPlayingInfoCenter.default().nowPlayingInfo = nil
|
||||
mediaSessionActive = false
|
||||
}
|
||||
|
||||
private func addRemoteTarget(
|
||||
_ command: MPRemoteCommand,
|
||||
handler: @escaping (MPRemoteCommandEvent) -> MPRemoteCommandHandlerStatus
|
||||
) {
|
||||
let token = command.addTarget(handler: handler)
|
||||
remoteCommandTargets.append((command, token))
|
||||
}
|
||||
|
||||
private func triggerMediaSession(_ event: String) {
|
||||
trigger(event, data: JSObject())
|
||||
}
|
||||
|
||||
private func displayName(for voice: AVSpeechSynthesisVoice) -> String {
|
||||
switch voice.quality {
|
||||
case .enhanced:
|
||||
return "\(voice.name) (Enhanced)"
|
||||
case .premium:
|
||||
return "\(voice.name) (Premium)"
|
||||
default:
|
||||
return voice.name
|
||||
}
|
||||
}
|
||||
|
||||
private func primaryLanguage(_ language: String) -> String {
|
||||
return String(language.split(separator: "-").first ?? "")
|
||||
}
|
||||
|
||||
/// A human-readable region for disambiguating duplicate voice names, e.g.
|
||||
/// "en-US" -> "United States". Falls back to the raw subtag, then the full tag.
|
||||
private func regionDescription(for language: String) -> String {
|
||||
let parts = language.split(separator: "-")
|
||||
if parts.count > 1, let regionPart = parts.last {
|
||||
let region = String(regionPart)
|
||||
return Locale.current.localizedString(forRegionCode: region) ?? region
|
||||
}
|
||||
return language
|
||||
}
|
||||
|
||||
/// Loads artwork from a base64 data URI, a remote URL, or a bundled asset.
|
||||
private func loadImage(from urlString: String) -> UIImage? {
|
||||
if urlString.hasPrefix("data:image") {
|
||||
guard let commaIndex = urlString.firstIndex(of: ",") else {
|
||||
return nil
|
||||
}
|
||||
let base64 = String(urlString[urlString.index(after: commaIndex)...])
|
||||
guard let data = Data(base64Encoded: base64) else {
|
||||
return nil
|
||||
}
|
||||
return UIImage(data: data)
|
||||
} else if urlString.hasPrefix("http") {
|
||||
guard let url = URL(string: urlString), let data = try? Data(contentsOf: url) else {
|
||||
return nil
|
||||
}
|
||||
return UIImage(data: data)
|
||||
} else {
|
||||
return UIImage(named: urlString)
|
||||
}
|
||||
}
|
||||
|
||||
/// AVSpeechUtterance rate runs 0...1 with ~0.5 as normal, while the JS client
|
||||
/// sends an Android-tuned `pow(userMultiplier, 2.5)` value where 1.0 is
|
||||
/// normal. Recover the multiplier and rescale onto the AVSpeechUtterance range.
|
||||
private func avRate(from jsRate: Float) -> Float {
|
||||
let userMultiplier = pow(max(jsRate, 0.0001), 1.0 / 2.5)
|
||||
let mapped = AVSpeechUtteranceDefaultSpeechRate * userMultiplier
|
||||
return min(
|
||||
max(mapped, AVSpeechUtteranceMinimumSpeechRate), AVSpeechUtteranceMaximumSpeechRate)
|
||||
}
|
||||
|
||||
/// AVSpeechUtterance pitchMultiplier is clamped to [0.5, 2.0] (1.0 normal).
|
||||
private func avPitch(from jsPitch: Float) -> Float {
|
||||
return min(max(jsPitch, 0.5), 2.0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -198,19 +198,31 @@ vi.mock('@/utils/ttsTime', () => ({
|
||||
}),
|
||||
}));
|
||||
|
||||
const { mockDeinitMediaSession } = vi.hoisted(() => ({
|
||||
mockDeinitMediaSession: vi.fn(() => Promise.resolve()),
|
||||
}));
|
||||
|
||||
vi.mock('@/app/reader/hooks/useTTSMediaSession', () => ({
|
||||
useTTSMediaSession: () => ({
|
||||
mediaSessionRef: { current: null },
|
||||
unblockAudio: vi.fn(),
|
||||
releaseUnblockAudio: vi.fn(),
|
||||
initMediaSession: vi.fn().mockResolvedValue(undefined),
|
||||
deinitMediaSession: vi.fn().mockResolvedValue(undefined),
|
||||
deinitMediaSession: mockDeinitMediaSession,
|
||||
}),
|
||||
}));
|
||||
|
||||
// Imports must come AFTER vi.mock calls so they pick up the mocked modules.
|
||||
import { useTTSControl } from '@/app/reader/hooks/useTTSControl';
|
||||
import { eventDispatcher } from '@/utils/event';
|
||||
import { useReaderStore } from '@/store/readerStore';
|
||||
|
||||
const getSetTTSEnabledMock = () =>
|
||||
(
|
||||
useReaderStore as unknown as {
|
||||
getState: () => { setTTSEnabled: ReturnType<typeof vi.fn> };
|
||||
}
|
||||
).getState().setTTSEnabled;
|
||||
|
||||
const Harness = () => {
|
||||
useTTSControl({ bookKey: 'book-1' });
|
||||
@@ -324,6 +336,82 @@ describe('useTTSControl tts-sync-request (mode-entry replay)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('useTTSControl handleStop resilience (#4676)', () => {
|
||||
beforeEach(() => {
|
||||
ttsControllerInstances.length = 0;
|
||||
pendingInitResolvers.length = 0;
|
||||
mockDeinitMediaSession.mockReset();
|
||||
mockDeinitMediaSession.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
const startSession = async () => {
|
||||
render(<Harness />);
|
||||
await act(async () => {
|
||||
const p = eventDispatcher.dispatch('tts-speak', { bookKey: 'book-1' });
|
||||
for (let i = 0; i < 10; i++) await Promise.resolve();
|
||||
while (pendingInitResolvers.length > 0) pendingInitResolvers.shift()!();
|
||||
await p;
|
||||
});
|
||||
await act(async () => {
|
||||
for (let i = 0; i < 5; i++) await Promise.resolve();
|
||||
});
|
||||
return ttsControllerInstances[0] as { shutdown: ReturnType<typeof vi.fn> };
|
||||
};
|
||||
|
||||
it('disables TTS even when controller.shutdown rejects', async () => {
|
||||
// Regression: a native teardown that throws (observed with iOS system TTS)
|
||||
// must not skip the state resets that turn the TTS icon off.
|
||||
const controller = await startSession();
|
||||
const setTTSEnabled = getSetTTSEnabledMock();
|
||||
setTTSEnabled.mockClear();
|
||||
controller.shutdown.mockRejectedValueOnce(new Error('native teardown failed'));
|
||||
|
||||
await act(async () => {
|
||||
await eventDispatcher.dispatch('tts-stop', { bookKey: 'book-1' });
|
||||
for (let i = 0; i < 5; i++) await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(setTTSEnabled).toHaveBeenCalledWith('book-1', false);
|
||||
});
|
||||
|
||||
it('disables TTS even when controller.shutdown never resolves', async () => {
|
||||
// The state resets must run before (not after) the teardown await, so a
|
||||
// hung native teardown can never leave the TTS icon stuck on.
|
||||
const controller = await startSession();
|
||||
const setTTSEnabled = getSetTTSEnabledMock();
|
||||
setTTSEnabled.mockClear();
|
||||
controller.shutdown.mockReturnValueOnce(new Promise<void>(() => {}));
|
||||
|
||||
await act(async () => {
|
||||
// Do not await the dispatch: handleStop intentionally never settles here.
|
||||
eventDispatcher.dispatch('tts-stop', { bookKey: 'book-1' });
|
||||
for (let i = 0; i < 5; i++) await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(setTTSEnabled).toHaveBeenCalledWith('book-1', false);
|
||||
});
|
||||
|
||||
it('tears down the media session even when controller.shutdown never resolves', async () => {
|
||||
// Regression for the lock-screen Now Playing lingering with iOS system TTS:
|
||||
// the media-session teardown must not be gated behind the controller's own
|
||||
// shutdown, which can stall.
|
||||
const controller = await startSession();
|
||||
mockDeinitMediaSession.mockClear();
|
||||
controller.shutdown.mockReturnValueOnce(new Promise<void>(() => {}));
|
||||
|
||||
await act(async () => {
|
||||
eventDispatcher.dispatch('tts-stop', { bookKey: 'book-1' });
|
||||
for (let i = 0; i < 5; i++) await Promise.resolve();
|
||||
});
|
||||
|
||||
expect(mockDeinitMediaSession).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('useTTSControl handleHighlightMark cross-section navigation', () => {
|
||||
beforeEach(() => {
|
||||
ttsControllerInstances.length = 0;
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import { describe, test, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
// getMediaSession() consults the platform helpers; mock them so each test can
|
||||
// pin a platform without touching the real user agent / env.
|
||||
vi.mock('@/utils/misc', () => ({
|
||||
getOSPlatform: vi.fn(),
|
||||
}));
|
||||
vi.mock('@/services/environment', () => ({
|
||||
isTauriAppPlatform: vi.fn(),
|
||||
}));
|
||||
|
||||
// mediaSession.ts imports these at module load; TauriMediaSession construction
|
||||
// does not call them, but the imports must resolve in jsdom.
|
||||
vi.mock('@tauri-apps/api/core', () => ({
|
||||
invoke: vi.fn(),
|
||||
addPluginListener: vi.fn(),
|
||||
}));
|
||||
|
||||
import { getMediaSession, TauriMediaSession } from '@/libs/mediaSession';
|
||||
import { getOSPlatform } from '@/utils/misc';
|
||||
import { isTauriAppPlatform } from '@/services/environment';
|
||||
|
||||
const setNavigatorMediaSession = (present: boolean) => {
|
||||
if (present) {
|
||||
Object.defineProperty(navigator, 'mediaSession', {
|
||||
value: { metadata: null, setActionHandler: vi.fn() },
|
||||
configurable: true,
|
||||
});
|
||||
} else if ('mediaSession' in navigator) {
|
||||
delete (navigator as unknown as { mediaSession?: unknown }).mediaSession;
|
||||
}
|
||||
};
|
||||
|
||||
describe('getMediaSession', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
setNavigatorMediaSession(false);
|
||||
});
|
||||
|
||||
test('uses navigator.mediaSession on iOS, NOT the native plugin', () => {
|
||||
// iOS audio plays through the WebView (Edge TTS media element, or the silent
|
||||
// keep-alive element during system TTS), so navigator.mediaSession is what
|
||||
// surfaces the lock-screen cover + sentence + controls. Routing iOS through
|
||||
// the native plugin hid the Edge cover/sentence and gave system TTS no
|
||||
// controls (AVSpeechSynthesizer can't be surfaced that way). See #4676.
|
||||
vi.mocked(getOSPlatform).mockReturnValue('ios');
|
||||
vi.mocked(isTauriAppPlatform).mockReturnValue(true);
|
||||
setNavigatorMediaSession(true);
|
||||
|
||||
const result = getMediaSession();
|
||||
expect(result).not.toBeInstanceOf(TauriMediaSession);
|
||||
expect(result).toBe(navigator.mediaSession);
|
||||
});
|
||||
|
||||
test('returns TauriMediaSession on Android Tauri (native foreground service)', () => {
|
||||
// Android is checked first, so it uses the native session even though the
|
||||
// WebView may also expose navigator.mediaSession.
|
||||
vi.mocked(getOSPlatform).mockReturnValue('android');
|
||||
vi.mocked(isTauriAppPlatform).mockReturnValue(true);
|
||||
setNavigatorMediaSession(true);
|
||||
|
||||
expect(getMediaSession()).toBeInstanceOf(TauriMediaSession);
|
||||
});
|
||||
|
||||
test('falls back to navigator.mediaSession on the web', () => {
|
||||
vi.mocked(getOSPlatform).mockReturnValue('macos');
|
||||
vi.mocked(isTauriAppPlatform).mockReturnValue(false);
|
||||
setNavigatorMediaSession(true);
|
||||
|
||||
const result = getMediaSession();
|
||||
expect(result).not.toBeInstanceOf(TauriMediaSession);
|
||||
expect(result).toBe(navigator.mediaSession);
|
||||
});
|
||||
|
||||
test('returns null when neither a native nor a web media session is available', () => {
|
||||
vi.mocked(getOSPlatform).mockReturnValue('linux');
|
||||
vi.mocked(isTauriAppPlatform).mockReturnValue(false);
|
||||
setNavigatorMediaSession(false);
|
||||
|
||||
expect(getMediaSession()).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, test, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
vi.mock('@tauri-apps/api/core', () => ({
|
||||
invoke: vi.fn(),
|
||||
addPluginListener: vi.fn().mockResolvedValue({ unregister: vi.fn() }),
|
||||
}));
|
||||
|
||||
// Avoid pulling in the heavy TTSController module graph (foliate-js, etc.) — the
|
||||
// native client only references it as a type.
|
||||
vi.mock('@/services/tts/TTSController', () => ({ TTSController: class {} }));
|
||||
|
||||
import { NativeTTSClient } from '@/services/tts/NativeTTSClient';
|
||||
import { invoke } from '@tauri-apps/api/core';
|
||||
|
||||
describe('NativeTTSClient.stop', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
test('resolves promptly when the native stop resolves', async () => {
|
||||
vi.mocked(invoke).mockResolvedValue(undefined);
|
||||
const client = new NativeTTSClient();
|
||||
|
||||
await client.stop();
|
||||
|
||||
expect(invoke).toHaveBeenCalledWith('plugin:native-tts|stop');
|
||||
});
|
||||
|
||||
test('still resolves (bounded) when the native stop never resolves', async () => {
|
||||
// Regression for #4676: a hung native stop must not hang teardown
|
||||
// (controller.stop / shutdown), which would leave the TTS icon stuck.
|
||||
vi.mocked(invoke).mockReturnValue(new Promise(() => {}));
|
||||
const client = new NativeTTSClient();
|
||||
|
||||
let settled = false;
|
||||
const p = client.stop().then(() => {
|
||||
settled = true;
|
||||
});
|
||||
|
||||
// Pending until the timeout fires.
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1500);
|
||||
await p;
|
||||
expect(settled).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -142,9 +142,10 @@ function createMockView(): FoliateView {
|
||||
|
||||
// --- Helper: create mock AppService ---
|
||||
|
||||
function createMockAppService(isAndroid = false): AppService {
|
||||
function createMockAppService(isAndroid = false, isIOS = false): AppService {
|
||||
return {
|
||||
isAndroidApp: isAndroid,
|
||||
isIOSApp: isIOS,
|
||||
} as unknown as AppService;
|
||||
}
|
||||
|
||||
@@ -215,7 +216,13 @@ describe('TTSController', () => {
|
||||
expect(c.ttsNativeClient).not.toBeNull();
|
||||
});
|
||||
|
||||
test('does not create native client when not Android', () => {
|
||||
test('creates native client when isIOSApp', () => {
|
||||
const iosService = createMockAppService(false, true);
|
||||
const c = new TTSController(iosService, mockView);
|
||||
expect(c.ttsNativeClient).not.toBeNull();
|
||||
});
|
||||
|
||||
test('does not create native client when neither Android nor iOS', () => {
|
||||
expect(controller.ttsNativeClient).toBeNull();
|
||||
});
|
||||
|
||||
|
||||
@@ -535,26 +535,42 @@ export const useTTSControl = ({ bookKey, onRequestHidePanel }: UseTTSControlProp
|
||||
const handleStop = useCallback(
|
||||
async (bookKey: string) => {
|
||||
const ttsController = ttsControllerRef.current;
|
||||
if (ttsController) {
|
||||
await ttsController.shutdown();
|
||||
ttsControllerRef.current = null;
|
||||
setTtsController(null);
|
||||
getView(bookKey)?.deselect();
|
||||
setIsPlaying(false);
|
||||
emitPlaybackState('stopped');
|
||||
onRequestHidePanel?.();
|
||||
setShowIndicator(false);
|
||||
setShowBackToCurrentTTSLocation(false);
|
||||
}
|
||||
// Reset all UI/session state up front — including the TTS toggle
|
||||
// (ttsEnabled) and indicator that color the TTS icon — so disabling TTS
|
||||
// always takes effect immediately. The teardown below is best-effort and
|
||||
// must never block or skip these resets if it hangs or throws, which was
|
||||
// observed with iOS system TTS (Edge TTS was unaffected). See #4676.
|
||||
ttsControllerRef.current = null;
|
||||
setTtsController(null);
|
||||
setIsPlaying(false);
|
||||
emitPlaybackState('stopped');
|
||||
onRequestHidePanel?.();
|
||||
setShowIndicator(false);
|
||||
setShowBackToCurrentTTSLocation(false);
|
||||
previousSectionLabelRef.current = undefined;
|
||||
if (appService?.isIOSApp) {
|
||||
await invokeUseBackgroundAudio({ enabled: false });
|
||||
}
|
||||
setTTSEnabled(bookKey, false);
|
||||
getView(bookKey)?.deselect();
|
||||
if (appService?.isMobile) {
|
||||
releaseUnblockAudio();
|
||||
}
|
||||
await deinitMediaSession();
|
||||
setTTSEnabled(bookKey, false);
|
||||
|
||||
// Tear down the controller, the lock-screen media session, and the
|
||||
// background-audio session best-effort and IN PARALLEL. The controller's
|
||||
// own shutdown can stall on iOS system TTS, and it must NOT gate the media
|
||||
// session / background-audio teardown — otherwise the lock-screen Now
|
||||
// Playing keeps running after TTS is disabled (Edge TTS was unaffected
|
||||
// because it never hits the stalling native path). See #4676.
|
||||
await Promise.all([
|
||||
ttsController
|
||||
? Promise.resolve()
|
||||
.then(() => ttsController.shutdown())
|
||||
.catch((error) => console.warn('TTS shutdown failed:', error))
|
||||
: Promise.resolve(),
|
||||
appService?.isIOSApp
|
||||
? invokeUseBackgroundAudio({ enabled: false }).catch(() => {})
|
||||
: Promise.resolve(),
|
||||
deinitMediaSession().catch(() => {}),
|
||||
]);
|
||||
},
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
[appService],
|
||||
|
||||
@@ -140,10 +140,23 @@ export class TauriMediaSession {
|
||||
}
|
||||
|
||||
export function getMediaSession() {
|
||||
const platform = getOSPlatform();
|
||||
// Android: the native foreground-service media session (TextToSpeech and
|
||||
// WebAudio both run with the app as the media owner; the Android WebView
|
||||
// doesn't expose a usable Media Session here).
|
||||
if (platform === 'android' && isTauriAppPlatform()) {
|
||||
return new TauriMediaSession();
|
||||
}
|
||||
// iOS (and web): the audio always plays through the WebView — Edge TTS's media
|
||||
// element, or the silent keep-alive element (`unblockAudio`) that runs during
|
||||
// system TTS — so navigator.mediaSession, driven by that element, is what
|
||||
// surfaces the lock-screen card with the cover + current sentence and the
|
||||
// transport controls. AVSpeechSynthesizer is NOT a WebView media element and
|
||||
// can't be surfaced via the native MPNowPlayingInfo path, so iOS must NOT be
|
||||
// routed through the native plugin (doing so both hid the Edge cover/sentence
|
||||
// and left system TTS with no controls). See #4676.
|
||||
if ('mediaSession' in navigator) {
|
||||
return navigator.mediaSession;
|
||||
} else if (getOSPlatform() === 'android' && isTauriAppPlatform()) {
|
||||
return new TauriMediaSession();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -214,7 +214,17 @@ export class NativeTTSClient implements TTSClient {
|
||||
}
|
||||
|
||||
async stop() {
|
||||
await invoke('plugin:native-tts|stop');
|
||||
// Bound the native stop so teardown (controller.stop / shutdown, and thus
|
||||
// the TTS icon turning off) can never hang if the native side is slow to
|
||||
// resolve — e.g. iOS AVSpeechSynthesizer / audio-session teardown. See #4676.
|
||||
try {
|
||||
await Promise.race([
|
||||
invoke('plugin:native-tts|stop'),
|
||||
new Promise<void>((resolve) => setTimeout(resolve, 1500)),
|
||||
]);
|
||||
} catch (error) {
|
||||
console.warn('Native TTS stop failed:', error);
|
||||
}
|
||||
this.#activeUtterances.clear();
|
||||
}
|
||||
|
||||
|
||||
@@ -87,8 +87,9 @@ export class TTSController extends EventTarget {
|
||||
super();
|
||||
this.ttsWebClient = new WebSpeechClient(this);
|
||||
this.ttsEdgeClient = new EdgeTTSClient(this, appService);
|
||||
// TODO: implement native TTS client for iOS and PC
|
||||
if (appService?.isAndroidApp) {
|
||||
// Native TTS is backed by Android TextToSpeech and iOS AVSpeechSynthesizer.
|
||||
// TODO: implement native TTS client for desktop platforms.
|
||||
if (appService?.isAndroidApp || appService?.isIOSApp) {
|
||||
this.ttsNativeClient = new NativeTTSClient(this);
|
||||
}
|
||||
this.ttsClient = this.ttsWebClient;
|
||||
|
||||
Reference in New Issue
Block a user