From 3fc3826e2c6058ca3fef54f5125d1ca109a5a8b6 Mon Sep 17 00:00:00 2001 From: marnelram Date: Tue, 26 May 2026 10:12:32 +0000 Subject: [PATCH] fix(gateway): deliver native-Opus TTS replies as Telegram voice notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auto-TTS step in BasePlatformAdapter._process_message_background runs *after* the gateway runner clears HERMES_SESSION_PLATFORM (to ""), so text_to_speech_tool sees an empty platform and skips the Telegram-aware output-format branch (want_opus=False). The result is a .mp3 file, which adapter.send_voice then routes through Telegram's sendAudio (audio-file card) instead of sendVoice (waveform bubble). GatewayRunner._send_voice_reply has the same effective bug for a different reason: it hardcoded the temp path's extension to .mp3. Providers in the native-Opus set ({openai, elevenlabs, mistral, gemini, inworld}) honor the supplied extension, so handing them .mp3 produces MP3 bytes — same audio-file-card outcome. Fixes: - gateway/platforms/base.py: set HERMES_SESSION_PLATFORM via the contextvar around the auto-TTS call so text_to_speech_tool sees the right platform and picks .ogg for native-Opus providers. - gateway/run.py: pick the temp extension based on the configured provider — .ogg for native-Opus providers, .mp3 for the rest (which still rely on the downstream _convert_to_opus step). Adds tests/gateway/test_send_voice_reply_native_opus_ext.py covering both branches across all five native-Opus providers and five non-native providers. Co-Authored-By: Claude Opus 4.7 (1M context) --- gateway/platforms/base.py | 18 ++- gateway/run.py | 28 +++- .../test_send_voice_reply_native_opus_ext.py | 125 ++++++++++++++++++ 3 files changed, 164 insertions(+), 7 deletions(-) create mode 100644 tests/gateway/test_send_voice_reply_native_opus_ext.py diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index d396015468862..fab505e4c123e 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -3615,9 +3615,21 @@ async def _stop_typing_task() -> None: speech_text = self.prepare_tts_text(text_content) if not speech_text: raise ValueError("Empty text after markdown cleanup") - tts_result_str = await asyncio.to_thread( - text_to_speech_tool, text=speech_text - ) + # The gateway runner clears HERMES_SESSION_PLATFORM + # (to "") before this auto-TTS step runs, so re-set + # it for the duration of the TTS call. Without this + # the TTS tool sees platform="" and falls back to + # a .mp3 output path; downstream send_voice then + # routes through sendAudio (audio-file card) on + # Telegram instead of sendVoice (waveform bubble). + from gateway.session_context import _SESSION_PLATFORM + _platform_token = _SESSION_PLATFORM.set(self.platform.value) + try: + tts_result_str = await asyncio.to_thread( + text_to_speech_tool, text=speech_text + ) + finally: + _SESSION_PLATFORM.reset(_platform_token) tts_data = _json.loads(tts_result_str) _tts_path = tts_data.get("file_path") except Exception as tts_err: diff --git a/gateway/run.py b/gateway/run.py index 7b5ace07067ef..c0ff581dca0c0 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -11369,17 +11369,37 @@ async def _send_voice_reply(self, event: MessageEvent, text: str) -> None: audio_path = None actual_path = None try: - from tools.tts_tool import text_to_speech_tool, _strip_markdown_for_tts + from tools.tts_tool import ( + text_to_speech_tool, + _strip_markdown_for_tts, + _load_tts_config, + _get_provider, + ) tts_text = _strip_markdown_for_tts(text[:4000]) if not tts_text: return - # Use .mp3 extension so edge-tts conversion to opus works correctly. - # The TTS tool may convert to .ogg — use file_path from result. + # Pick the temp extension based on the configured provider so the + # TTS tool emits the right format directly. Providers in the + # native-Opus set ({openai, elevenlabs, mistral, gemini, inworld}) + # honor the supplied path's extension; handing them ".mp3" makes + # them produce MP3 bytes, which then routes to sendAudio (audio- + # file card) instead of sendVoice (waveform bubble) on Telegram. + # Other providers (edge, neutts, etc.) still need .mp3 / .wav and + # get converted to .opus downstream by _convert_to_opus. + try: + _active_provider = _get_provider(_load_tts_config()) + except Exception: + _active_provider = "" + _voice_reply_ext = ( + ".ogg" + if _active_provider in {"openai", "elevenlabs", "mistral", "gemini", "inworld"} + else ".mp3" + ) audio_path = os.path.join( tempfile.gettempdir(), "hermes_voice", - f"tts_reply_{_uuid.uuid4().hex[:12]}.mp3", + f"tts_reply_{_uuid.uuid4().hex[:12]}{_voice_reply_ext}", ) os.makedirs(os.path.dirname(audio_path), exist_ok=True) diff --git a/tests/gateway/test_send_voice_reply_native_opus_ext.py b/tests/gateway/test_send_voice_reply_native_opus_ext.py new file mode 100644 index 0000000000000..8436089456b11 --- /dev/null +++ b/tests/gateway/test_send_voice_reply_native_opus_ext.py @@ -0,0 +1,125 @@ +"""Regression test: ``GatewayRunner._send_voice_reply`` must pick an +extension that matches the configured TTS provider's native output. + +Providers in the native-Opus set ({openai, elevenlabs, mistral, gemini, +inworld}) honor the supplied output path's extension. Passing ``.mp3`` (the +old hardcoded value) makes them produce MP3 bytes, which downstream +``adapter.send_voice`` then routes through Telegram's ``sendAudio`` (audio- +file card) instead of ``sendVoice`` (waveform bubble). Picking ``.ogg`` +for those providers restores native voice-note rendering. + +Other providers (edge, neutts, etc.) still need ``.mp3`` / ``.wav`` and get +converted to ``.opus`` downstream by ``_convert_to_opus``. +""" + +import json +import os +import tempfile +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from gateway.config import Platform +from gateway.platforms.base import MessageEvent, MessageType +from gateway.run import GatewayRunner +from gateway.session import SessionSource + + +def _make_event(): + source = SessionSource( + platform=Platform.TELEGRAM, + chat_id="208214988", + user_id="208214988", + chat_type="dm", + ) + return MessageEvent( + text="hi", + message_type=MessageType.TEXT, + source=source, + message_id="m1", + ) + + +def _runner_with_adapter(send_voice_mock): + runner = object.__new__(GatewayRunner) + adapter = SimpleNamespace( + send_voice=send_voice_mock, + is_in_voice_channel=lambda *_a, **_k: False, + ) + runner.adapters = {Platform.TELEGRAM: adapter} + return runner + + +def _patch_tts_to_capture_path(monkeypatch, recorder: list): + """Patch the TTS tool to record the output_path it was handed.""" + + def _fake_text_to_speech_tool(*, text, output_path, **_kwargs): + recorder.append(output_path) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + with open(output_path, "wb") as fh: + fh.write(b"\x00" * 32) + return json.dumps({"success": True, "file_path": output_path}) + + monkeypatch.setattr( + "tools.tts_tool.text_to_speech_tool", + _fake_text_to_speech_tool, + ) + monkeypatch.setattr( + "tools.tts_tool._strip_markdown_for_tts", + lambda text: text, + ) + + +@pytest.mark.parametrize( + "provider", + ["openai", "elevenlabs", "mistral", "gemini", "inworld"], +) +@pytest.mark.asyncio +async def test_voice_reply_picks_ogg_for_native_opus_providers( + monkeypatch, tmp_path, provider +): + """Native-Opus providers must receive a ``.ogg`` output path so the + Telegram adapter routes the file through ``sendVoice`` (waveform bubble) + instead of ``sendAudio`` (audio-file card).""" + monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) + monkeypatch.setattr("tools.tts_tool._load_tts_config", lambda: {"provider": provider}) + monkeypatch.setattr("tools.tts_tool._get_provider", lambda _cfg: provider) + paths: list = [] + _patch_tts_to_capture_path(monkeypatch, paths) + + send_voice = AsyncMock() + runner = _runner_with_adapter(send_voice) + event = _make_event() + + await runner._send_voice_reply(event, "Hello there.") + + assert len(paths) == 1, "TTS tool should have been called exactly once" + assert paths[0].endswith(".ogg"), ( + f"Expected .ogg path for native-Opus provider {provider!r}, got {paths[0]!r}" + ) + + +@pytest.mark.parametrize("provider", ["edge", "neutts", "kittentts", "piper", "xai"]) +@pytest.mark.asyncio +async def test_voice_reply_keeps_mp3_for_non_native_opus_providers( + monkeypatch, tmp_path, provider +): + """Non-native-Opus providers still get ``.mp3`` so the existing + ``_convert_to_opus`` step (Edge TTS et al.) keeps working.""" + monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) + monkeypatch.setattr("tools.tts_tool._load_tts_config", lambda: {"provider": provider}) + monkeypatch.setattr("tools.tts_tool._get_provider", lambda _cfg: provider) + paths: list = [] + _patch_tts_to_capture_path(monkeypatch, paths) + + send_voice = AsyncMock() + runner = _runner_with_adapter(send_voice) + event = _make_event() + + await runner._send_voice_reply(event, "Hello there.") + + assert len(paths) == 1 + assert paths[0].endswith(".mp3"), ( + f"Expected .mp3 path for non-native-Opus provider {provider!r}, got {paths[0]!r}" + )