From edb4dcd57f11748d04b0d3296800627b9c4ac930 Mon Sep 17 00:00:00 2001 From: adridot Date: Mon, 22 Jun 2026 16:41:54 +0000 Subject: [PATCH 1/3] fix(gateway): transcribe voice replies to a pending clarify prompt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clarify-reply interception in the gateway only reads `(event.text or "").strip()`. A voice reply to a pending `clarify` prompt has empty `text`, so it falls through: the clarify never resolves and the audio is processed as an unrelated new turn — the user's answer is silently dropped. Transcribe the voice reply via the existing `_enrich_message_with_transcription` pipeline and use the transcript as the clarify answer, echoed back as 🎙️ "...". If transcription yields nothing usable, the clarify stays pending and the user is asked to reply in text, instead of the answer being lost. Builds on existing primitives (MessageType.VOICE, _enrich_message_with_transcription); no new dependencies. Co-Authored-By: Claude Opus 4.8 (1M context) --- gateway/run.py | 73 +++++++++++++++ tests/gateway/test_telegram_audio_vs_voice.py | 89 ++++++++++++++++++- 2 files changed, 161 insertions(+), 1 deletion(-) diff --git a/gateway/run.py b/gateway/run.py index 4f3b12375d66..a4b81cee4572 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -7466,6 +7466,61 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: _pending_clarify = None if _pending_clarify is not None: _raw_clarify_reply = (event.text or "").strip() + _attempted_voice_clarify = False + if not _raw_clarify_reply: + _media_urls = getattr(event, "media_urls", None) or [] + _media_types = getattr(event, "media_types", None) or [] + _audio_paths = [] + for _i, _path in enumerate(_media_urls): + _mtype = _media_types[_i] if _i < len(_media_types) else "" + _is_voice = ( + getattr(event, "message_type", None) == MessageType.VOICE + or ( + _mtype.startswith("audio/") + and getattr(event, "message_type", None) + not in {MessageType.AUDIO, MessageType.DOCUMENT} + ) + ) + if _is_voice: + _audio_paths.append(_path) + if _audio_paths: + _attempted_voice_clarify = True + # Call _enrich_message_with_transcription directly rather than + # the canonical _prepare_inbound_message_text: the clarify + # answer must be the RAW transcript, not the agent-facing + # "voice message" wrapper that method emits, and we must avoid + # its native-image-buffer side effect for a reply we only read + # as text. + try: + _enriched, _transcripts = await self._enrich_message_with_transcription( + "", _audio_paths, + ) + # Clarify needs the answer itself, not the agent-facing + # wrapper text. Keep wrappers only as fallback if STT + # could not produce a clean transcript. + _raw_clarify_reply = "\n".join( + tx.strip() for tx in _transcripts if tx and tx.strip() + ).strip() or (_enriched or "").strip() + if _transcripts: + _echo_adapter = self.adapters.get(source.platform) + _echo_meta = {"thread_id": source.thread_id} if source.thread_id else None + if _echo_adapter: + for _tx in _transcripts: + try: + await _echo_adapter.send( + source.chat_id, + f'🎙️ "{_tx}"', + metadata=_echo_meta, + ) + except Exception as _echo_exc: + logger.debug( + "Clarify voice echo failed (non-fatal): %s", + _echo_exc, + ) + except Exception as _trans_exc: + logger.warning( + "Clarify voice transcription failed: %s", _trans_exc, + ) # Skip slash commands — the user clearly wanted to issue a # command, not answer the clarify. Leave the clarify pending # so the user can retry; if it times out, the agent unblocks @@ -7483,6 +7538,24 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: # the agent's response don't double-post. The agent # itself will produce the next user-facing message. return "" + if not _raw_clarify_reply and not (event.text or "").strip().startswith("/"): + logger.info( + "Ignoring non-text clarify response for session=%s id=%s " + "(voice_attempted=%s); keeping clarify pending", + _quick_key, + _pending_clarify.clarify_id, + _attempted_voice_clarify, + ) + if _attempted_voice_clarify: + return ( + "I couldn't transcribe that voice reply for the pending " + "question. Please answer in text, or send a shorter voice " + "message." + ) + return ( + "I'm still waiting for an answer to the question above. " + "Please reply in text to continue." + ) # Intercept messages that are responses to a pending /reload-mcp # (or future) slash-confirm prompt. Recognized confirm replies are diff --git a/tests/gateway/test_telegram_audio_vs_voice.py b/tests/gateway/test_telegram_audio_vs_voice.py index 1d1bf0cb78b3..9b4d22f0f8a9 100644 --- a/tests/gateway/test_telegram_audio_vs_voice.py +++ b/tests/gateway/test_telegram_audio_vs_voice.py @@ -12,7 +12,7 @@ 3. Mixed media lists (voice + audio) split correctly. """ -from unittest.mock import patch +from unittest.mock import AsyncMock, patch import pytest @@ -186,3 +186,90 @@ def test_telegram_media_type_detection_audio_vs_voice(): assert MessageType.VOICE.value == "voice" # Sanity: they are distinct assert MessageType.AUDIO != MessageType.VOICE + + +# --------------------------------------------------------------------------- +# 5. Voice reply to a PENDING CLARIFY resolves it with the raw transcript +# (#50925 — voice answers to clarify prompts were silently dropped) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_voice_reply_resolves_pending_clarify_with_transcript(): + """A voice answer to an open clarify resolves it with the RAW transcript.""" + from gateway.run import GatewayRunner + from gateway.session import build_session_key + from tools import clarify_gateway as cm + + with cm._lock: + cm._entries.clear() + cm._session_index.clear() + + runner = _make_runner(stt_enabled=True) + runner.session_store = None + + source = SessionSource( + platform=Platform.TELEGRAM, chat_id="1", chat_type="dm", user_id="user1", + ) + session_key = build_session_key(source) + cm.register("cid-voice", session_key, "Which option?", choices=None) + + event = MessageEvent( + text="", + message_type=MessageType.VOICE, + source=source, + media_urls=["/tmp/voice.ogg"], + media_types=["audio/ogg"], + internal=True, + ) + + with patch( + "tools.transcription_tools.transcribe_audio", + return_value={"success": True, "transcript": "the blue one", "provider": "whisper"}, + ) as mock_transcribe: + result = await GatewayRunner._handle_message(runner, event) + + mock_transcribe.assert_called_once_with("/tmp/voice.ogg") + # Acknowledged with an empty string so adapters don't double-post. + assert result == "" + # Resolves with the RAW transcript — not a wrapped "voice message ..." note. + assert cm.wait_for_response("cid-voice", timeout=0.01) == "the blue one" + + +@pytest.mark.asyncio +async def test_voice_reply_failed_transcription_keeps_clarify_pending(): + """If STT yields no usable text, the clarify stays pending and the user is nudged to text.""" + from gateway.run import GatewayRunner + from gateway.session import build_session_key + from tools import clarify_gateway as cm + + with cm._lock: + cm._entries.clear() + cm._session_index.clear() + + runner = _make_runner(stt_enabled=True) + runner.session_store = None + + source = SessionSource( + platform=Platform.TELEGRAM, chat_id="1", chat_type="dm", user_id="user1", + ) + session_key = build_session_key(source) + cm.register("cid-voice-fail", session_key, "Which option?", choices=None) + + event = MessageEvent( + text="", + message_type=MessageType.VOICE, + source=source, + media_urls=["/tmp/voice.ogg"], + media_types=["audio/ogg"], + internal=True, + ) + + # No usable transcript -> the clarify must NOT resolve with garbage. + with patch.object( + runner, "_enrich_message_with_transcription", + new=AsyncMock(return_value=("", [])), + ): + result = await GatewayRunner._handle_message(runner, event) + + assert "text" in result.lower() + assert cm.get_pending_for_session(session_key) is not None From bc8ad6a0e06ad6c36aae0d0e73568a8f6b41383a Mon Sep 17 00:00:00 2001 From: LauraGPT <18321252+LauraGPT@users.noreply.github.com> Date: Tue, 14 Jul 2026 03:30:24 +0000 Subject: [PATCH 2/3] fix(gateway): handle Matrix voice clarify filenames Transcribe voice media before resolving a pending clarify even when the event text contains a cached filename. Accept only non-empty raw transcripts so failed STT leaves the prompt pending. Cover both Telegram empty-text voice events and Matrix cached-filename events. Refs NousResearch/hermes-agent#52998. --- gateway/run.py | 20 ++++++----- tests/gateway/test_telegram_audio_vs_voice.py | 34 ++++++++++++++----- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index a4b81cee4572..0379ff329c25 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -7467,7 +7467,7 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: if _pending_clarify is not None: _raw_clarify_reply = (event.text or "").strip() _attempted_voice_clarify = False - if not _raw_clarify_reply: + if not _raw_clarify_reply.startswith("/"): _media_urls = getattr(event, "media_urls", None) or [] _media_types = getattr(event, "media_types", None) or [] _audio_paths = [] @@ -7485,6 +7485,10 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: _audio_paths.append(_path) if _audio_paths: _attempted_voice_clarify = True + # Voice events may expose a cached filename in ``text``. + # Never let that filename resolve the pending clarify if + # transcription fails or returns no usable transcript. + _raw_clarify_reply = "" # Call _enrich_message_with_transcription directly rather than # the canonical _prepare_inbound_message_text: the clarify # answer must be the RAW transcript, not the agent-facing @@ -7492,20 +7496,18 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: # its native-image-buffer side effect for a reply we only read # as text. try: - _enriched, _transcripts = await self._enrich_message_with_transcription( + _, _transcripts = await self._enrich_message_with_transcription( "", _audio_paths, ) - # Clarify needs the answer itself, not the agent-facing - # wrapper text. Keep wrappers only as fallback if STT - # could not produce a clean transcript. - _raw_clarify_reply = "\n".join( + _clean_transcripts = [ tx.strip() for tx in _transcripts if tx and tx.strip() - ).strip() or (_enriched or "").strip() - if _transcripts: + ] + _raw_clarify_reply = "\n".join(_clean_transcripts) + if _clean_transcripts: _echo_adapter = self.adapters.get(source.platform) _echo_meta = {"thread_id": source.thread_id} if source.thread_id else None if _echo_adapter: - for _tx in _transcripts: + for _tx in _clean_transcripts: try: await _echo_adapter.send( source.chat_id, diff --git a/tests/gateway/test_telegram_audio_vs_voice.py b/tests/gateway/test_telegram_audio_vs_voice.py index 9b4d22f0f8a9..53aa116f8e13 100644 --- a/tests/gateway/test_telegram_audio_vs_voice.py +++ b/tests/gateway/test_telegram_audio_vs_voice.py @@ -194,8 +194,17 @@ def test_telegram_media_type_detection_audio_vs_voice(): # --------------------------------------------------------------------------- @pytest.mark.asyncio -async def test_voice_reply_resolves_pending_clarify_with_transcript(): - """A voice answer to an open clarify resolves it with the RAW transcript.""" +@pytest.mark.parametrize( + ("platform", "event_text"), + [ + (Platform.TELEGRAM, ""), + (Platform.MATRIX, "voice_message_123.ogg"), + ], +) +async def test_voice_reply_resolves_pending_clarify_with_transcript( + platform, event_text +): + """A voice answer resolves clarify with the raw transcript, not its filename.""" from gateway.run import GatewayRunner from gateway.session import build_session_key from tools import clarify_gateway as cm @@ -208,13 +217,13 @@ async def test_voice_reply_resolves_pending_clarify_with_transcript(): runner.session_store = None source = SessionSource( - platform=Platform.TELEGRAM, chat_id="1", chat_type="dm", user_id="user1", + platform=platform, chat_id="1", chat_type="dm", user_id="user1", ) session_key = build_session_key(source) cm.register("cid-voice", session_key, "Which option?", choices=None) event = MessageEvent( - text="", + text=event_text, message_type=MessageType.VOICE, source=source, media_urls=["/tmp/voice.ogg"], @@ -236,8 +245,17 @@ async def test_voice_reply_resolves_pending_clarify_with_transcript(): @pytest.mark.asyncio -async def test_voice_reply_failed_transcription_keeps_clarify_pending(): - """If STT yields no usable text, the clarify stays pending and the user is nudged to text.""" +@pytest.mark.parametrize( + ("platform", "event_text"), + [ + (Platform.TELEGRAM, ""), + (Platform.MATRIX, "voice_message_456.ogg"), + ], +) +async def test_voice_reply_failed_transcription_keeps_clarify_pending( + platform, event_text +): + """If STT yields no text, clarify stays pending even when text is a filename.""" from gateway.run import GatewayRunner from gateway.session import build_session_key from tools import clarify_gateway as cm @@ -250,13 +268,13 @@ async def test_voice_reply_failed_transcription_keeps_clarify_pending(): runner.session_store = None source = SessionSource( - platform=Platform.TELEGRAM, chat_id="1", chat_type="dm", user_id="user1", + platform=platform, chat_id="1", chat_type="dm", user_id="user1", ) session_key = build_session_key(source) cm.register("cid-voice-fail", session_key, "Which option?", choices=None) event = MessageEvent( - text="", + text=event_text, message_type=MessageType.VOICE, source=source, media_urls=["/tmp/voice.ogg"], From b5ba9fdd043dc2a4c857d61113223cc350b414f0 Mon Sep 17 00:00:00 2001 From: adridot Date: Wed, 15 Jul 2026 09:10:14 +0000 Subject: [PATCH 3/3] fix(gateway): gate clarify transcript echo and use topic-aware metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback on #50925: - Gate the clarify 🎙️ transcript echo behind _should_echo_stt_transcripts() so stt.echo_transcripts: false is honored, matching every other echo path. - Build the echo metadata with _thread_metadata_for_source() and the event reply anchor so Telegram DM-topic echoes keep direct_messages_topic_id, the reply anchor, and the topic fallback flag instead of a bare thread_id. - Regressions: echo disabled still resolves the clarify with no send; DM-topic echo preserves the full routing metadata. Co-Authored-By: Claude Fable 5 --- gateway/run.py | 6 +- tests/gateway/test_telegram_audio_vs_voice.py | 109 +++++++++++++++++- 2 files changed, 111 insertions(+), 4 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index 03bb37380fb4..43fcf9a0ee98 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -9234,8 +9234,10 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]: _raw_clarify_reply = "\n".join(_clean_transcripts) if _clean_transcripts: _echo_adapter = self.adapters.get(source.platform) - _echo_meta = {"thread_id": source.thread_id} if source.thread_id else None - if _echo_adapter: + _echo_meta = self._thread_metadata_for_source( + source, self._reply_anchor_for_event(event), + ) + if _echo_adapter and self._should_echo_stt_transcripts(): for _tx in _clean_transcripts: try: await _echo_adapter.send( diff --git a/tests/gateway/test_telegram_audio_vs_voice.py b/tests/gateway/test_telegram_audio_vs_voice.py index ede69b1f9ea6..0a94bb4f25f5 100644 --- a/tests/gateway/test_telegram_audio_vs_voice.py +++ b/tests/gateway/test_telegram_audio_vs_voice.py @@ -21,11 +21,11 @@ from gateway.session import SessionSource -def _make_runner(stt_enabled: bool = True) -> "GatewayRunner": # type: ignore[name-defined] +def _make_runner(stt_enabled: bool = True, **config_kwargs) -> "GatewayRunner": # type: ignore[name-defined] from gateway.run import GatewayRunner runner = GatewayRunner.__new__(GatewayRunner) - runner.config = GatewayConfig(stt_enabled=stt_enabled) + runner.config = GatewayConfig(stt_enabled=stt_enabled, **config_kwargs) runner.adapters = {} runner._model = "test-model" runner._base_url = "" @@ -292,3 +292,108 @@ async def test_voice_reply_failed_transcription_keeps_clarify_pending( assert "text" in result.lower() assert cm.get_pending_for_session(session_key) is not None + + +# --------------------------------------------------------------------------- +# 6. Clarify transcript echo honors stt.echo_transcripts and thread metadata +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_voice_clarify_echo_disabled_still_resolves_without_send(): + """stt_echo_transcripts=False: clarify resolves, but no 🎙️ echo is sent.""" + from unittest.mock import MagicMock + + from gateway.run import GatewayRunner + from gateway.session import build_session_key + from tools import clarify_gateway as cm + + with cm._lock: + cm._entries.clear() + cm._session_index.clear() + + runner = _make_runner(stt_enabled=True, stt_echo_transcripts=False) + runner.session_store = None + adapter = MagicMock() + adapter.send = AsyncMock() + runner.adapters = {Platform.TELEGRAM: adapter} + + source = SessionSource( + platform=Platform.TELEGRAM, chat_id="1", chat_type="dm", user_id="user1", + ) + session_key = build_session_key(source) + cm.register("cid-no-echo", session_key, "Which option?", choices=None) + + event = MessageEvent( + text="", + message_type=MessageType.VOICE, + source=source, + media_urls=["/tmp/voice.ogg"], + media_types=["audio/ogg"], + internal=True, + ) + + with patch( + "tools.transcription_tools.transcribe_audio", + return_value={"success": True, "transcript": "use the local model", "provider": "whisper"}, + ): + result = await GatewayRunner._handle_message(runner, event) + + assert result == "" + assert cm.wait_for_response("cid-no-echo", timeout=0.01) == "use the local model" + adapter.send.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_voice_clarify_echo_preserves_telegram_dm_topic_metadata(): + """The 🎙️ echo in a Telegram DM topic keeps topic routing + reply anchor.""" + from unittest.mock import MagicMock + + from gateway.run import GatewayRunner + from gateway.session import build_session_key + from tools import clarify_gateway as cm + + with cm._lock: + cm._entries.clear() + cm._session_index.clear() + + runner = _make_runner(stt_enabled=True) + runner.session_store = None + adapter = MagicMock() + adapter.send = AsyncMock() + runner.adapters = {Platform.TELEGRAM: adapter} + + source = SessionSource( + platform=Platform.TELEGRAM, chat_id="1", chat_type="dm", + user_id="user1", thread_id="42", + ) + session_key = build_session_key(source) + cm.register("cid-topic", session_key, "Which option?", choices=None) + + event = MessageEvent( + text="", + message_type=MessageType.VOICE, + source=source, + message_id="777", + media_urls=["/tmp/voice.ogg"], + media_types=["audio/ogg"], + internal=True, + ) + + with patch( + "tools.transcription_tools.transcribe_audio", + return_value={"success": True, "transcript": "the blue one", "provider": "whisper"}, + ): + result = await GatewayRunner._handle_message(runner, event) + + assert result == "" + assert cm.wait_for_response("cid-topic", timeout=0.01) == "the blue one" + adapter.send.assert_awaited_once_with( + "1", + '🎙️ "the blue one"', + metadata={ + "thread_id": "42", + "telegram_dm_topic_reply_fallback": True, + "direct_messages_topic_id": "42", + "telegram_reply_to_message_id": "777", + }, + )