From 20371ddb721b7af5ba05bb78ddac71203376309b Mon Sep 17 00:00:00 2001 From: Joey Neleber Date: Mon, 8 Jun 2026 22:10:08 -0600 Subject: [PATCH] fix(discord): keep voice meetings responsive --- gateway/run.py | 47 ++++++++++-- plugins/platforms/discord/adapter.py | 103 ++++++++++++++++++++++++++- tests/gateway/test_voice_command.py | 82 +++++++++++++++++++-- 3 files changed, 219 insertions(+), 13 deletions(-) diff --git a/gateway/run.py b/gateway/run.py index dc8e0f14cc0be..126d01a5342c5 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -67,6 +67,13 @@ _ADAPTER_DISCONNECT_TIMEOUT_SECS_DEFAULT = 5.0 _TELEGRAM_COMMAND_MENTION_RE = re.compile(r"(? bool: + raw = os.getenv(name, "").strip().lower() + if not raw: + return default + return raw in {"1", "true", "yes", "on"} + + _TELEGRAM_NOISY_STATUS_RE = re.compile( r"(" # transient/auxiliary status that should stay in logs, not Telegram chat r"auxiliary\s+.+\s+failed" @@ -12113,12 +12120,25 @@ async def _handle_voice_channel_join(self, event: MessageEvent) -> str: adapter._voice_text_channels[guild_id] = int(event.source.chat_id) if hasattr(adapter, "_voice_sources"): adapter._voice_sources[guild_id] = event.source.to_dict() + # Meeting/listen mode should not make every transcript trigger an + # agent+TTS turn. Keep text-channel replies speakable via voice_mode + # but disable adapter auto-TTS unless transcript agent turns are + # explicitly enabled. self._voice_mode[self._voice_key(event.source.platform, event.source.chat_id)] = "all" self._save_voice_modes() - self._set_adapter_auto_tts_enabled(adapter, event.source.chat_id, enabled=True) + transcript_agent_turns = self._voice_transcripts_trigger_agent_turns() + if transcript_agent_turns: + self._set_adapter_auto_tts_enabled(adapter, event.source.chat_id, enabled=True) + else: + self._set_adapter_auto_tts_disabled(adapter, event.source.chat_id, disabled=True) + mode_note = ( + "I'll speak my replies and listen to you." + if transcript_agent_turns + else "I'll transcribe voice to this side chat without answering every utterance." + ) return ( f"Joined voice channel **{voice_channel.name}**.\n" - f"I'll speak my replies and listen to you. Use /voice leave to disconnect." + f"{mode_note} Use /voice leave to disconnect." ) # Join failed — clear callback adapter._voice_input_callback = None @@ -12198,13 +12218,24 @@ def _is_duplicate_voice_transcript(self, guild_id: int, user_id: int, transcript recent_store[key] = recent[-5:] return False + def _voice_transcripts_trigger_agent_turns(self) -> bool: + """Whether raw Discord VC transcripts should run the full agent loop. + + Meeting mode should be cheap and durable: publish transcripts to the + linked side chat without replaying a growing conversation into the LLM + for every utterance. Operators can opt back into the old behavior with + HERMES_DISCORD_VOICE_TRANSCRIPT_AGENT_TURNS=1. + """ + return _env_flag("HERMES_DISCORD_VOICE_TRANSCRIPT_AGENT_TURNS", False) + async def _handle_voice_channel_input( self, guild_id: int, user_id: int, transcript: str ): """Handle transcribed voice from a user in a voice channel. - Creates a synthetic MessageEvent and processes it through the - adapter's full message pipeline (session, typing, agent, TTS reply). + Always posts the transcript to the linked side chat. By default it does + not turn every utterance into a full agent request, because that makes + long meetings slower as Discord/session history grows. """ adapter = self.adapters.get(Platform.DISCORD) if not adapter: @@ -12253,6 +12284,14 @@ async def _handle_voice_channel_input( except Exception: pass + if not self._voice_transcripts_trigger_agent_turns(): + logger.debug( + "Posted Discord voice transcript without agent turn for guild=%s user=%s", + guild_id, + user_id, + ) + return + # Build a synthetic MessageEvent and feed through the normal pipeline # Use SimpleNamespace as raw_message so _get_guild_id() can extract # guild_id and _send_voice_reply() plays audio in the voice channel. diff --git a/plugins/platforms/discord/adapter.py b/plugins/platforms/discord/adapter.py index fa0f81c9b2e54..655ed73479dd4 100644 --- a/plugins/platforms/discord/adapter.py +++ b/plugins/platforms/discord/adapter.py @@ -135,6 +135,13 @@ def check_discord_requirements() -> bool: return True +def _env_flag(name: str, default: bool = False) -> bool: + raw = os.getenv(name, "").strip().lower() + if not raw: + return default + return raw in {"1", "true", "yes", "on"} + + def _build_allowed_mentions(): """Build Discord ``AllowedMentions`` with safe defaults, overridable via env. @@ -595,10 +602,12 @@ def __init__(self, config: PlatformConfig): self._voice_text_channels: Dict[int, int] = {} # guild_id -> text_channel_id self._voice_sources: Dict[int, Dict[str, Any]] = {} # guild_id -> linked text channel source metadata self._voice_timeout_tasks: Dict[int, asyncio.Task] = {} # guild_id -> timeout task + self._voice_activity_last_reset: Dict[int, float] = {} # guild_id -> last inbound activity timer reset # Phase 2: voice listening self._voice_receivers: Dict[int, VoiceReceiver] = {} # guild_id -> VoiceReceiver self._voice_listen_tasks: Dict[int, asyncio.Task] = {} # guild_id -> listen loop self._voice_input_callback: Optional[Callable] = None # set by run.py + self._recent_voice_transcripts: Dict[Tuple[int, int], List[Tuple[float, str]]] = {} self._on_voice_disconnect: Optional[Callable] = None # set by run.py # Phase 3: continuous voice mixer (ambient idle bed + ducked speech). # Installed once per guild on join; lets acks / TTS / the "thinking" @@ -2154,6 +2163,7 @@ async def leave_voice_channel(self, guild_id: int) -> None: task = self._voice_timeout_tasks.pop(guild_id, None) if task: task.cancel() + self._voice_activity_last_reset.pop(guild_id, None) self._voice_text_channels.pop(guild_id, None) self._voice_sources.pop(guild_id, None) @@ -2256,6 +2266,22 @@ def _reset_voice_timeout(self, guild_id: int) -> None: self._voice_timeout_handler(guild_id) ) + def _note_voice_activity(self, guild_id: int, *, min_interval: float = 30.0) -> None: + """Record inbound voice activity and keep the voice session alive. + + The inactivity timeout exists to leave empty/forgotten voice channels, + but inbound speech is activity too. Previously the timer was reset on + join/playback only, so a listen-only meeting could be disconnected + after VOICE_TIMEOUT even while users were actively talking. Throttle + resets to avoid churning timeout tasks on every 200ms listen-loop tick. + """ + now = time.monotonic() + last = self._voice_activity_last_reset.get(guild_id, 0.0) + if now - last < min_interval: + return + self._voice_activity_last_reset[guild_id] = now + self._reset_voice_timeout(guild_id) + async def _voice_timeout_handler(self, guild_id: int) -> None: """Auto-disconnect after VOICE_TIMEOUT seconds of inactivity.""" try: @@ -2263,6 +2289,7 @@ async def _voice_timeout_handler(self, guild_id: int) -> None: except asyncio.CancelledError: return text_ch_id = self._voice_text_channels.get(guild_id) + self._voice_activity_last_reset.pop(guild_id, None) await self.leave_voice_channel(guild_id) # Notify the runner so it can clean up voice_mode state if self._on_voice_disconnect and text_ch_id: @@ -2381,6 +2408,20 @@ async def _voice_listen_loop(self, guild_id: int): except Exception: pass + # Any recent inbound packet means the meeting is active, even + # before an utterance reaches silence/STT. This prevents + # continuous or listen-only meetings from timing out mid-call. + try: + with receiver._lock: + recent_audio = any( + now - last_t < self._KEEPALIVE_INTERVAL + for last_t in receiver._last_packet_time.values() + ) + if recent_audio: + self._note_voice_activity(guild_id) + except Exception: + pass + completed = receiver.check_silence() # Voice inputs always originate from a specific guild # (guild_id is in scope). Pass it so role checks are @@ -2393,14 +2434,61 @@ async def _voice_listen_loop(self, guild_id: int): is_dm=False, ): continue + self._note_voice_activity(guild_id) await self._process_voice_input(guild_id, user_id, pcm_data) except asyncio.CancelledError: pass except Exception as e: logger.error("Voice listen loop error: %s", e, exc_info=True) + def _voice_transcripts_trigger_agent_turns(self) -> bool: + """Whether completed VC utterances should enter the agent pipeline.""" + return _env_flag("HERMES_DISCORD_VOICE_TRANSCRIPT_AGENT_TURNS", False) + + def _is_duplicate_voice_transcript(self, guild_id: int, user_id: int, transcript: str) -> bool: + """Suppress repeated STT outputs before side-chat posting/callback.""" + from difflib import SequenceMatcher + + normalized = re.sub(r"\s+", " ", transcript).strip().lower() + normalized = re.sub(r"[^\w\s]", "", normalized) + if not normalized: + return False + + now = time.monotonic() + window_seconds = 12.0 + key = (guild_id, user_id) + recent = [ + (ts, txt) + for ts, txt in self._recent_voice_transcripts.get(key, []) + if now - ts <= window_seconds + ] + for _, prior in recent: + if prior == normalized: + self._recent_voice_transcripts[key] = recent + return True + if len(prior) >= 16 and len(normalized) >= 16: + if SequenceMatcher(None, prior, normalized).ratio() >= 0.95: + self._recent_voice_transcripts[key] = recent + return True + recent.append((now, normalized)) + self._recent_voice_transcripts[key] = recent[-5:] + return False + + async def _post_voice_transcript_to_side_chat(self, guild_id: int, user_id: int, transcript: str) -> None: + """Post a completed voice transcript directly from the Discord adapter.""" + text_ch_id = self._voice_text_channels.get(guild_id) + if not text_ch_id or not self._client: + return + try: + channel = self._client.get_channel(text_ch_id) + if channel: + safe_text = transcript[:2000].replace("@everyone", "@\u200beveryone").replace("@here", "@\u200bhere") + await channel.send(f"**[Voice]** <@{user_id}>: {safe_text}") + except Exception: + logger.debug("Failed to post Discord voice transcript", exc_info=True) + async def _process_voice_input(self, guild_id: int, user_id: int, pcm_data: bytes): - """Convert PCM -> WAV -> STT -> callback.""" + """Convert PCM -> WAV -> STT, post transcript, and optionally callback.""" from tools.voice_mode import is_whisper_hallucination tmp_f = tempfile.NamedTemporaryFile(suffix=".wav", prefix="vc_listen_", delete=False) @@ -2420,7 +2508,18 @@ async def _process_voice_input(self, guild_id: int, user_id: int, pcm_data: byte logger.info("Voice input from user %d: %s", user_id, transcript[:100]) - if self._voice_input_callback: + if self._is_duplicate_voice_transcript(guild_id, user_id, transcript): + logger.info( + "Suppressing duplicate voice transcript for guild=%s user=%s: %s", + guild_id, + user_id, + transcript[:100], + ) + return + + await self._post_voice_transcript_to_side_chat(guild_id, user_id, transcript) + + if self._voice_input_callback and self._voice_transcripts_trigger_agent_turns(): await self._voice_input_callback( guild_id=guild_id, user_id=user_id, diff --git a/tests/gateway/test_voice_command.py b/tests/gateway/test_voice_command.py index 5066f4952f658..56050d9d84a6e 100644 --- a/tests/gateway/test_voice_command.py +++ b/tests/gateway/test_voice_command.py @@ -523,6 +523,7 @@ def _make_discord_adapter(self): adapter._voice_text_channels = {} adapter._voice_sources = {} adapter._voice_timeout_tasks = {} + adapter._voice_activity_last_reset = {} adapter._voice_receivers = {} adapter._voice_listen_tasks = {} adapter._client = None @@ -919,8 +920,9 @@ async def test_input_no_text_channel(self, runner): await runner._handle_voice_channel_input(111, 42, "Hello") @pytest.mark.asyncio - async def test_input_creates_event_and_dispatches(self, runner): - """Voice input creates synthetic event and calls handle_message.""" + async def test_input_creates_event_and_dispatches_when_enabled(self, runner, monkeypatch): + """Voice input creates synthetic event when transcript agent turns are enabled.""" + monkeypatch.setenv("HERMES_DISCORD_VOICE_TRANSCRIPT_AGENT_TURNS", "1") from gateway.config import Platform mock_adapter = AsyncMock() mock_adapter._voice_text_channels = {111: 123} @@ -939,8 +941,9 @@ async def test_input_creates_event_and_dispatches(self, runner): assert event.source.chat_type == "channel" @pytest.mark.asyncio - async def test_input_reuses_bound_source_metadata(self, runner): + async def test_input_reuses_bound_source_metadata_when_enabled(self, runner, monkeypatch): """Voice input should share the linked text channel session metadata.""" + monkeypatch.setenv("HERMES_DISCORD_VOICE_TRANSCRIPT_AGENT_TURNS", "1") from gateway.config import Platform bound_source = SessionSource( @@ -988,9 +991,31 @@ async def test_input_posts_transcript_in_text_channel(self, runner): assert "Test transcript" in msg assert "42" in msg # user_id in mention + + @pytest.mark.asyncio + async def test_input_posts_transcript_without_agent_turn_by_default(self, runner, monkeypatch): + """Meeting mode posts side-chat transcripts without running the full agent loop.""" + monkeypatch.delenv("HERMES_DISCORD_VOICE_TRANSCRIPT_AGENT_TURNS", raising=False) + from gateway.config import Platform + + mock_adapter = AsyncMock() + mock_adapter._voice_text_channels = {111: 123} + mock_adapter._voice_sources = {} + mock_channel = AsyncMock() + mock_adapter._client = MagicMock() + mock_adapter._client.get_channel = MagicMock(return_value=mock_channel) + mock_adapter.handle_message = AsyncMock() + runner.adapters[Platform.DISCORD] = mock_adapter + + await runner._handle_voice_channel_input(111, 42, "Meeting transcript") + + mock_channel.send.assert_called_once() + mock_adapter.handle_message.assert_not_called() + @pytest.mark.asyncio - async def test_input_suppresses_duplicate_transcript(self, runner): + async def test_input_suppresses_duplicate_transcript(self, runner, monkeypatch): """Near-immediate duplicate STT output should not dispatch twice.""" + monkeypatch.setenv("HERMES_DISCORD_VOICE_TRANSCRIPT_AGENT_TURNS", "1") from gateway.config import Platform mock_adapter = AsyncMock() @@ -1009,8 +1034,9 @@ async def test_input_suppresses_duplicate_transcript(self, runner): mock_channel.send.assert_called_once() @pytest.mark.asyncio - async def test_input_suppresses_near_duplicate_transcript(self, runner): + async def test_input_suppresses_near_duplicate_transcript(self, runner, monkeypatch): """Small STT wording drift should still be treated as the same utterance.""" + monkeypatch.setenv("HERMES_DISCORD_VOICE_TRANSCRIPT_AGENT_TURNS", "1") from gateway.config import Platform mock_adapter = AsyncMock() @@ -1078,9 +1104,11 @@ def _make_adapter(self): adapter._voice_text_channels = {} adapter._voice_sources = {} adapter._voice_timeout_tasks = {} + adapter._voice_activity_last_reset = {} adapter._voice_receivers = {} adapter._voice_listen_tasks = {} adapter._voice_input_callback = None + adapter._recent_voice_transcripts = {} adapter._allowed_user_ids = set() adapter._running = True return adapter @@ -1112,6 +1140,7 @@ async def test_leave_voice_channel_cleans_up(self): adapter._voice_clients[111] = mock_vc adapter._voice_text_channels[111] = 123 adapter._voice_sources[111] = {"chat_id": "123", "chat_type": "group"} + adapter._voice_activity_last_reset[111] = 123.0 mock_receiver = MagicMock() adapter._voice_receivers[111] = mock_receiver @@ -1131,6 +1160,7 @@ async def test_leave_voice_channel_cleans_up(self): assert 111 not in adapter._voice_clients assert 111 not in adapter._voice_text_channels assert 111 not in adapter._voice_sources + assert 111 not in adapter._voice_activity_last_reset assert 111 not in adapter._voice_receivers @pytest.mark.asyncio @@ -1197,9 +1227,25 @@ def test_is_allowed_user_not_in_list(self): adapter._allowed_user_ids = {"99"} assert adapter._is_allowed_user("42") is False + + def test_note_voice_activity_resets_timeout_and_throttles(self, monkeypatch): + """Inbound voice activity keeps the Discord VC timeout alive without churn.""" + adapter = self._make_adapter() + calls = [] + monkeypatch.setattr(adapter, "_reset_voice_timeout", lambda guild_id: calls.append(guild_id)) + times = iter([100.0, 110.0, 131.0]) + monkeypatch.setattr("plugins.platforms.discord.adapter.time.monotonic", lambda: next(times)) + + adapter._note_voice_activity(111) + adapter._note_voice_activity(111) + adapter._note_voice_activity(111) + + assert calls == [111, 111] + @pytest.mark.asyncio - async def test_process_voice_input_success(self): - """Successful voice input: PCM->WAV->STT->callback.""" + async def test_process_voice_input_success(self, monkeypatch): + """Successful voice input: PCM->WAV->STT->callback when agent turns enabled.""" + monkeypatch.setenv("HERMES_DISCORD_VOICE_TRANSCRIPT_AGENT_TURNS", "1") adapter = self._make_adapter() callback = AsyncMock() adapter._voice_input_callback = callback @@ -1215,6 +1261,26 @@ async def test_process_voice_input_success(self): callback.assert_called_once_with(guild_id=111, user_id=42, transcript="Hello") + @pytest.mark.asyncio + async def test_process_voice_input_posts_side_chat_without_callback_by_default(self, monkeypatch): + """Default meeting mode keeps transcript posting bot-side and skips agent callback.""" + monkeypatch.delenv("HERMES_DISCORD_VOICE_TRANSCRIPT_AGENT_TURNS", raising=False) + adapter = self._make_adapter() + callback = AsyncMock() + adapter._voice_input_callback = callback + adapter._voice_text_channels = {111: 123} + mock_channel = AsyncMock() + adapter._client.get_channel = MagicMock(return_value=mock_channel) + + with patch("plugins.platforms.discord.adapter.VoiceReceiver.pcm_to_wav"), \ + patch("tools.transcription_tools.transcribe_audio", + return_value={"success": True, "transcript": "Hello bot side"}), \ + patch("tools.voice_mode.is_whisper_hallucination", return_value=False): + await adapter._process_voice_input(111, 42, b"\x00" * 96000) + + mock_channel.send.assert_called_once() + callback.assert_not_called() + @pytest.mark.asyncio async def test_process_voice_input_hallucination_filtered(self): """Whisper hallucination is filtered out.""" @@ -1860,6 +1926,7 @@ def _make_discord_adapter(): adapter._voice_text_channels = {} adapter._voice_sources = {} adapter._voice_timeout_tasks = {} + adapter._voice_activity_last_reset = {} adapter._voice_receivers = {} adapter._voice_listen_tasks = {} adapter._voice_input_callback = None @@ -1951,6 +2018,7 @@ def _make_discord_adapter(): adapter._voice_text_channels = {} adapter._voice_sources = {} adapter._voice_timeout_tasks = {} + adapter._voice_activity_last_reset = {} adapter._voice_receivers = {} adapter._voice_listen_tasks = {} adapter._voice_input_callback = None