diff --git a/apps/desktop/src/lib/voice-playback.test.ts b/apps/desktop/src/lib/voice-playback.test.ts new file mode 100644 index 000000000000..b3efa3676d37 --- /dev/null +++ b/apps/desktop/src/lib/voice-playback.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { startSpeechStream } from './voice-playback' + +class FakeWebSocket { + static CONNECTING = 0 + static OPEN = 1 + static CLOSED = 3 + static instances: FakeWebSocket[] = [] + + binaryType: BinaryType = 'blob' + close = vi.fn(() => { + this.readyState = FakeWebSocket.CLOSED + }) + onclose: ((event: CloseEvent) => void) | null = null + onerror: ((event: Event) => void) | null = null + onmessage: ((event: MessageEvent) => void) | null = null + onopen: ((event: Event) => void) | null = null + readyState = FakeWebSocket.CONNECTING + send = vi.fn() + + constructor(readonly url: string) { + FakeWebSocket.instances.push(this) + } +} + +describe('voice playback streaming', () => { + afterEach(() => { + FakeWebSocket.instances = [] + vi.unstubAllGlobals() + delete (window as { hermesDesktop?: unknown }).hermesDesktop + }) + + it('falls back to plain audio playback when WebAudio cannot initialize', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + vi.stubGlobal('WebSocket', FakeWebSocket) + vi.stubGlobal( + 'AudioContext', + class { + constructor() { + throw new Error('audio device unavailable') + } + } + ) + ;(window as { hermesDesktop?: unknown }).hermesDesktop = { + getConnection: vi.fn().mockResolvedValue({ authMode: 'token', wsUrl: 'ws://local/api/ws?token=abc' }) + } + + const session = await startSpeechStream({ source: 'read-aloud' }) + + expect(session).not.toBeNull() + const socket = FakeWebSocket.instances[0] + + expect(() => { + socket.onmessage?.( + new MessageEvent('message', { + data: JSON.stringify({ channels: 1, sample_rate: 24_000, type: 'start' }) + }) + ) + }).not.toThrow() + await expect(session?.done).resolves.toBe('fallback') + expect(warn).toHaveBeenCalledWith( + 'Voice playback streaming disabled: AudioContext unavailable', + expect.any(Error) + ) + }) +}) diff --git a/apps/desktop/src/lib/voice-playback.ts b/apps/desktop/src/lib/voice-playback.ts index 48a51d370817..ddc7b958c170 100644 --- a/apps/desktop/src/lib/voice-playback.ts +++ b/apps/desktop/src/lib/voice-playback.ts @@ -41,12 +41,25 @@ async function unlockAutoplay(): Promise { return } - if (!unlockCtx) { - unlockCtx = new Ctor() + try { + if (!unlockCtx) { + unlockCtx = new Ctor() + } + + if (unlockCtx.state === 'suspended') { + await unlockCtx.resume() + } + } catch (error) { + console.warn('Voice playback autoplay unlock skipped: AudioContext unavailable', error) } +} - if (unlockCtx.state === 'suspended') { - await unlockCtx.resume() +function createAudioContextOrNull(): AudioContext | null { + try { + return new AudioContext() + } catch (error) { + console.warn('Voice playback streaming disabled: AudioContext unavailable', error) + return null } } @@ -272,7 +285,12 @@ function openSpeechStream(wsUrl: string, options: VoicePlaybackOptions): SpeechS if (frame.type === 'start') { streamRate = frame.sample_rate || 24_000 - context = new AudioContext() + context = createAudioContextOrNull() + + if (!context) { + settle(started ? 'done' : 'fallback') + return + } // Autoplay policy can hand back a suspended context when playback wasn't // started by a user gesture (e.g. a wake-word-started voice turn). Resume diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index a9a922aa5474..c16f02202fbd 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -4381,6 +4381,35 @@ def _fetch() -> Dict[str, Any]: return {"available": True, "voices": voices} +DESKTOP_TTS_TIMEOUT_SECONDS = 75.0 + + +def _unlink_desktop_tts_path(file_path: Any) -> None: + if not file_path: + return + try: + Path(file_path).unlink(missing_ok=True) + except OSError: + pass + + +def _cleanup_timed_out_desktop_tts(future: "asyncio.Future", output_path: Path) -> None: + """Remove audio files produced after the desktop speak request timed out.""" + _unlink_desktop_tts_path(output_path) + try: + result_json = future.result() + except Exception: + return + try: + result = json.loads(result_json) if isinstance(result_json, str) else result_json + except Exception: + return + if isinstance(result, dict): + returned_path = result.get("file_path") + if returned_path and Path(returned_path) != output_path: + _unlink_desktop_tts_path(returned_path) + + @app.post("/api/audio/speak") async def speak_text(payload: TTSSpeakRequest, profile: Optional[str] = None): """Synthesize speech and return audio as base64 data URL. @@ -4394,6 +4423,8 @@ async def speak_text(payload: TTSSpeakRequest, profile: Optional[str] = None): if not text: raise HTTPException(status_code=400, detail="Text is required") + output_path = Path(get_hermes_home()) / "audio_cache" / f"desktop_speak_{secrets.token_hex(8)}.mp3" + try: from tools.tts_tool import text_to_speech_tool @@ -4403,14 +4434,28 @@ def _speak_scoped(): # resolution, so the task-local override inside this worker # thread is sufficient (same reasoning as the MCP probe scope). with _config_profile_scope(profile): - return text_to_speech_tool(text) + return text_to_speech_tool(text, output_path=str(output_path)) loop = asyncio.get_running_loop() - result_json = await loop.run_in_executor(None, _speak_scoped) + speak_future = loop.run_in_executor(None, _speak_scoped) + result_json = await asyncio.wait_for( + asyncio.shield(speak_future), + timeout=DESKTOP_TTS_TIMEOUT_SECONDS, + ) except HTTPException: # _config_profile_scope raises 400/404 for a bad profile — pass it # through instead of masking it as a 500 synthesis failure. raise + except (asyncio.TimeoutError, TimeoutError) as exc: + _log.warning( + "Desktop voice TTS timed out after %.1fs (text_len=%d)", + DESKTOP_TTS_TIMEOUT_SECONDS, + len(text), + ) + speak_future.add_done_callback( + lambda future: _cleanup_timed_out_desktop_tts(future, output_path) + ) + raise HTTPException(status_code=504, detail="Speech synthesis timed out") from exc except Exception as exc: _log.exception("Desktop voice TTS failed") raise HTTPException(status_code=500, detail=f"Speech synthesis failed: {exc}") diff --git a/tests/hermes_cli/test_web_server_profile_unification.py b/tests/hermes_cli/test_web_server_profile_unification.py index a0534ac27339..0b0d87a56839 100644 --- a/tests/hermes_cli/test_web_server_profile_unification.py +++ b/tests/hermes_cli/test_web_server_profile_unification.py @@ -444,6 +444,47 @@ def _fake_transcribe(path): assert resp.json()["transcript"] == "hi" assert seen["home"] == str(isolated_profiles["worker_beta"]) + def test_speak_text_times_out_stalled_provider(self, client, monkeypatch, tmp_path): + import json + import threading + import time + from pathlib import Path + + import hermes_cli.web_server as web_server + import tools.tts_tool as tts_tool + + captured = {} + finished = threading.Event() + + def stalled_tts(text, output_path=None): + time.sleep(0.05) + audio_file = Path(output_path) + captured["audio_file"] = audio_file + audio_file.parent.mkdir(parents=True, exist_ok=True) + audio_file.write_bytes(b"ID3late-audio-bytes") + finished.set() + return json.dumps({ + "success": True, + "file_path": str(audio_file), + "provider": "test", + }) + + monkeypatch.setattr(tts_tool, "text_to_speech_tool", stalled_tts) + monkeypatch.setattr(web_server, "DESKTOP_TTS_TIMEOUT_SECONDS", 0.01) + + resp = client.post("/api/audio/speak", json={"text": "hello there"}) + + assert resp.status_code == 504 + assert "timed out" in resp.json()["detail"].lower() + assert finished.wait(timeout=1.0) + + audio_file = captured["audio_file"] + for _ in range(20): + if not audio_file.exists(): + break + time.sleep(0.01) + assert not audio_file.exists() + def test_audio_endpoints_unknown_profile_404(self, client, isolated_profiles): resp = client.get("/api/audio/elevenlabs/voices?profile=ghost") assert resp.status_code == 404