diff --git a/gateway/run.py b/gateway/run.py index f697992d1256..112353b946f1 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -14482,20 +14482,27 @@ async def _send_voice_reply(self, event: MessageEvent, text: str) -> None: """Generate TTS audio and send as a voice message before the text reply.""" import uuid as _uuid audio_path = None + generated_path = None actual_path = None try: - from tools.tts_tool import text_to_speech_tool, _strip_markdown_for_tts + from tools.tts_tool import ( + _convert_to_opus, + _strip_markdown_for_tts, + text_to_speech_tool, + ) tts_text = _strip_markdown_for_tts(text[:4000]) if not tts_text: return - # Telegram's adapter only sends native voice bubbles for OGG/Opus. - # Other platforms keep the existing MP3 default. - audio_ext = "ogg" if event.source.platform == Platform.TELEGRAM else "mp3" + # Matrix and Telegram native voice renderers expect Ogg/Opus. + # Generate MP3 first for broad provider compatibility, then + # transcode below. Passing a .ogg path directly is unsafe for Edge + # TTS: it writes MP3 bytes to whatever path it is given. + needs_opus_voice = event.source.platform in {Platform.MATRIX, Platform.TELEGRAM} audio_path = os.path.join( tempfile.gettempdir(), "hermes_voice", - f"tts_reply_{_uuid.uuid4().hex[:12]}.{audio_ext}", + f"tts_reply_{_uuid.uuid4().hex[:12]}.mp3", ) os.makedirs(os.path.dirname(audio_path), exist_ok=True) @@ -14510,10 +14517,23 @@ async def _send_voice_reply(self, event: MessageEvent, text: str) -> None: # Use the actual file path from result (may differ after opus conversion) actual_path = result.get("file_path", audio_path) + generated_path = actual_path if not result.get("success") or not os.path.isfile(actual_path): logger.warning("Auto voice reply TTS failed: %s", result.get("error")) return + if needs_opus_voice and not str(actual_path).lower().endswith(".ogg"): + opus_path = await asyncio.to_thread(_convert_to_opus, actual_path) + if opus_path and os.path.isfile(opus_path): + actual_path = opus_path + else: + logger.warning( + "Auto voice reply could not convert %s to Ogg/Opus for %s; " + "sending original audio", + actual_path, + event.source.platform.value, + ) + adapter = self._adapter_for_source(event.source) # If connected to a voice channel, play there instead of sending a file @@ -14548,7 +14568,7 @@ async def _send_voice_reply(self, event: MessageEvent, text: str) -> None: except Exception as e: logger.warning("Auto voice reply failed: %s", e, exc_info=True) finally: - for p in {audio_path, actual_path} - {None}: + for p in {audio_path, generated_path, actual_path} - {None}: try: os.unlink(p) except OSError: diff --git a/plugins/platforms/matrix/adapter.py b/plugins/platforms/matrix/adapter.py index 3835d8e31efa..8405b3be0ea9 100644 --- a/plugins/platforms/matrix/adapter.py +++ b/plugins/platforms/matrix/adapter.py @@ -51,11 +51,15 @@ from __future__ import annotations import asyncio +import array import inspect import logging import mimetypes import os import re +import shutil +import subprocess +import sys import time from urllib.parse import urljoin, urlsplit, urlunsplit from dataclasses import dataclass, field @@ -134,6 +138,134 @@ class _TrustStateStub: # type: ignore[no-redef] logger = logging.getLogger(__name__) +_MATRIX_VOICE_WAVEFORM_BINS = 30 + + +def _matrix_voice_metadata_for_file(path: Path) -> Dict[str, Any]: + """Return best-effort Matrix voice metadata for an audio file. + + Matrix clients such as Element render ``m.audio`` events with + ``org.matrix.msc3245.voice`` as voice bubbles. They are more reliable when + the event also includes duration and MSC1767 waveform metadata. Metadata + extraction is deliberately best-effort: media delivery must still work on + systems without ffprobe/ffmpeg. + """ + metadata: Dict[str, Any] = {} + + ffprobe = shutil.which("ffprobe") + if ffprobe: + try: + result = subprocess.run( + [ + ffprobe, + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + str(path), + ], + capture_output=True, + text=True, + timeout=10, + stdin=subprocess.DEVNULL, + ) + if result.returncode == 0: + duration = float((result.stdout or "").strip() or 0) + if duration > 0: + metadata["duration"] = int(duration * 1000) + except Exception: + logger.debug("Matrix: failed to probe voice duration for %s", path, exc_info=True) + + ffmpeg = shutil.which("ffmpeg") + if ffmpeg: + try: + result = subprocess.run( + [ + ffmpeg, + "-v", + "error", + "-i", + str(path), + "-ac", + "1", + "-ar", + "8000", + "-f", + "s16le", + "-", + ], + capture_output=True, + timeout=15, + stdin=subprocess.DEVNULL, + ) + if result.returncode == 0 and result.stdout: + samples = array.array("h") + samples.frombytes(result.stdout) + if sys.byteorder != "little": + samples.byteswap() + if samples: + count = len(samples) + waveform = [] + for idx in range(_MATRIX_VOICE_WAVEFORM_BINS): + start = idx * count // _MATRIX_VOICE_WAVEFORM_BINS + end = max(start + 1, (idx + 1) * count // _MATRIX_VOICE_WAVEFORM_BINS) + peak = max(abs(value) for value in samples[start:end]) + waveform.append(min(1024, int(peak / 32767 * 1024))) + metadata["waveform"] = waveform + except Exception: + logger.debug("Matrix: failed to build voice waveform for %s", path, exc_info=True) + + return metadata + +def _matrix_transcode_voice_to_ogg(path: str) -> Optional[str]: + """Best-effort transcode of an audio file to Ogg/Opus for MSC3245 delivery. + + Returns the path of a NEW temporary ``.ogg`` file (caller owns cleanup), or + ``None`` when ffmpeg is unavailable or fails — callers then send the + original file, matching the adapter's previous behaviour. Runs blocking + subprocess work; call via ``asyncio.to_thread`` from async code. + """ + ffmpeg = shutil.which("ffmpeg") + if not ffmpeg: + return None + import tempfile + + fd, ogg_path = tempfile.mkstemp(prefix="matrix_voice_", suffix=".ogg") + os.close(fd) + try: + result = subprocess.run( + [ + ffmpeg, + "-v", + "error", + "-y", + "-i", + str(path), + "-acodec", + "libopus", + "-ac", + "1", + "-b:a", + "64k", + ogg_path, + ], + capture_output=True, + timeout=30, + stdin=subprocess.DEVNULL, + ) + if result.returncode == 0 and os.path.getsize(ogg_path) > 0: + return ogg_path + except Exception: + logger.debug("Matrix: voice transcode to Ogg/Opus failed for %s", path, exc_info=True) + try: + os.unlink(ogg_path) + except OSError: + pass + return None + + _MATRIX_BANG_COMMAND_RE = re.compile( r"^!([A-Za-z][A-Za-z0-9_-]*)(?=$|\s)(.*)$", re.DOTALL, @@ -1993,16 +2125,47 @@ async def send_voice( reply_to: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, ) -> SendResult: - """Upload an audio file as a voice message (MSC3245 native voice).""" - return await self._send_local_file( - chat_id, - audio_path, - "m.audio", - caption, - reply_to, - metadata=metadata, - is_voice=True, - ) + """Upload an audio file as a voice message (MSC3245 native voice). + + Matrix voice bubbles require Opus in an Ogg container (MSC3245), but + callers can reach this with any audio format — e.g. a model-invoked + ``text_to_speech`` result routed through gateway media delivery, not + just ``_send_voice_reply``. Enforce the codec at this boundary: + transcode non-Ogg input to Ogg/Opus (best-effort — if ffmpeg is + unavailable the original file is sent unchanged, preserving the + previous behaviour). + """ + converted_path: Optional[str] = None + send_path = audio_path + if not str(audio_path).lower().endswith((".ogg", ".oga", ".opus")): + converted_path = await asyncio.to_thread( + _matrix_transcode_voice_to_ogg, audio_path + ) + if converted_path: + send_path = converted_path + try: + return await self._send_local_file( + chat_id, + send_path, + "m.audio", + caption, + reply_to, + # keep the caller's basename (the temp transcode file has a + # generated name) so the event body stays meaningful + file_name=( + Path(audio_path).with_suffix(".ogg").name + if converted_path + else None + ), + metadata=metadata, + is_voice=True, + ) + finally: + if converted_path: + try: + os.unlink(converted_path) + except OSError: + pass async def send_video( self, @@ -2241,6 +2404,7 @@ async def _upload_and_send( reply_to: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, is_voice: bool = False, + voice_metadata: Optional[Dict[str, Any]] = None, ) -> SendResult: """Upload bytes to Matrix and send as a media message.""" if len(data) > self._max_media_bytes: @@ -2297,6 +2461,17 @@ async def _upload_and_send( # Add MSC3245 voice flag for native voice messages. if is_voice: msg_content["org.matrix.msc3245.voice"] = {} + duration = (voice_metadata or {}).get("duration") + waveform = (voice_metadata or {}).get("waveform") + if duration is not None: + msg_content["info"]["duration"] = duration + if duration is not None or waveform is not None: + audio_metadata: Dict[str, Any] = {} + if duration is not None: + audio_metadata["duration"] = duration + if waveform is not None: + audio_metadata["waveform"] = waveform + msg_content["org.matrix.msc1767.audio"] = audio_metadata self._apply_relation_metadata(msg_content, reply_to=reply_to, metadata=metadata) @@ -2345,9 +2520,25 @@ async def _send_local_file( fname = file_name or p.name ct = mimetypes.guess_type(fname)[0] or "application/octet-stream" data = p.read_bytes() + # ffprobe/ffmpeg probing is blocking (subprocess timeouts up to 15s) — + # run it off the event loop so voice uploads never stall the adapter. + voice_metadata = ( + await asyncio.to_thread(_matrix_voice_metadata_for_file, p) + if is_voice + else None + ) return await self._upload_and_send( - room_id, data, fname, ct, msgtype, caption, reply_to, metadata, is_voice + room_id, + data, + fname, + ct, + msgtype, + caption, + reply_to, + metadata, + is_voice, + voice_metadata, ) # ------------------------------------------------------------------ diff --git a/tests/gateway/test_matrix_voice.py b/tests/gateway/test_matrix_voice.py index b113ba275caf..f1907d390f5d 100644 --- a/tests/gateway/test_matrix_voice.py +++ b/tests/gateway/test_matrix_voice.py @@ -323,17 +323,26 @@ async def mock_send_message_event(room_id, event_type, content): self.adapter._client.send_message_event = mock_send_message_event - await self.adapter.send_voice( - chat_id="!room:example.org", - audio_path=temp_path, - caption="Test voice", - ) + with patch( + "plugins.platforms.matrix.adapter._matrix_voice_metadata_for_file", + return_value={"duration": 1234, "waveform": [0, 512, 1024]}, + ): + await self.adapter.send_voice( + chat_id="!room:example.org", + audio_path=temp_path, + caption="Test voice", + ) assert sent_content is not None, "No message was sent" assert "org.matrix.msc3245.voice" in sent_content, \ f"MSC3245 voice field missing from content: {sent_content.keys()}" assert sent_content["msgtype"] == "m.audio" assert sent_content["info"]["mimetype"] == "audio/ogg" + assert sent_content["info"]["duration"] == 1234 + assert sent_content["org.matrix.msc1767.audio"] == { + "duration": 1234, + "waveform": [0, 512, 1024], + } assert self.upload_call is not None, "Expected upload_media() to be called" assert isinstance(self.upload_call["data"], bytes) assert self.upload_call["mime_type"] == "audio/ogg" @@ -341,3 +350,82 @@ async def mock_send_message_event(room_id, event_type, content): finally: os.unlink(temp_path) + + @pytest.mark.asyncio + async def test_send_voice_transcodes_non_ogg_to_opus(self): + """Non-Ogg audio reaching send_voice (e.g. direct text_to_speech MP3) + is transcoded to Ogg/Opus at the adapter boundary (issue #14841).""" + with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as f: + f.write(b"fake mp3 data") + temp_path = f.name + with tempfile.NamedTemporaryFile(suffix=".ogg", delete=False) as f: + f.write(b"fake ogg opus data") + converted_path = f.name + + try: + sent_content = None + + async def mock_send_message_event(room_id, event_type, content): + nonlocal sent_content + sent_content = content + return "$sent_event" + + self.adapter._client.send_message_event = mock_send_message_event + + with patch( + "plugins.platforms.matrix.adapter._matrix_transcode_voice_to_ogg", + return_value=converted_path, + ) as mock_transcode, patch( + "plugins.platforms.matrix.adapter._matrix_voice_metadata_for_file", + return_value={"duration": 1234, "waveform": [0, 512, 1024]}, + ): + await self.adapter.send_voice( + chat_id="!room:example.org", + audio_path=temp_path, + caption="Test voice", + ) + + mock_transcode.assert_called_once_with(temp_path) + assert sent_content is not None, "No message was sent" + assert "org.matrix.msc3245.voice" in sent_content + assert sent_content["info"]["mimetype"] == "audio/ogg" + assert self.upload_call is not None + assert self.upload_call["data"] == b"fake ogg opus data" + assert self.upload_call["mime_type"] == "audio/ogg" + assert self.upload_call["filename"].endswith(".ogg") + # converted temp file is cleaned up by send_voice + assert not os.path.exists(converted_path) + + finally: + os.unlink(temp_path) + if os.path.exists(converted_path): + os.unlink(converted_path) + + @pytest.mark.asyncio + async def test_send_voice_ogg_input_skips_transcode(self): + """Already-Ogg input must not be re-transcoded.""" + with tempfile.NamedTemporaryFile(suffix=".ogg", delete=False) as f: + f.write(b"fake ogg data") + temp_path = f.name + + try: + async def mock_send_message_event(room_id, event_type, content): + return "$sent_event" + + self.adapter._client.send_message_event = mock_send_message_event + + with patch( + "plugins.platforms.matrix.adapter._matrix_transcode_voice_to_ogg", + ) as mock_transcode, patch( + "plugins.platforms.matrix.adapter._matrix_voice_metadata_for_file", + return_value={}, + ): + await self.adapter.send_voice( + chat_id="!room:example.org", + audio_path=temp_path, + ) + + mock_transcode.assert_not_called() + + finally: + os.unlink(temp_path) diff --git a/tests/gateway/test_voice_command.py b/tests/gateway/test_voice_command.py index 539648886daf..984d80be5eca 100644 --- a/tests/gateway/test_voice_command.py +++ b/tests/gateway/test_voice_command.py @@ -423,19 +423,22 @@ async def test_calls_tts_and_send_voice(self, runner): event.source.platform = Platform.TELEGRAM runner.adapters[event.source.platform] = mock_adapter - tts_result = json.dumps({"success": True, "file_path": "/tmp/test.ogg"}) + tts_result = json.dumps({"success": True, "file_path": "/tmp/test.mp3"}) with patch("tools.tts_tool.text_to_speech_tool", return_value=tts_result) as mock_tts, \ patch("tools.tts_tool._strip_markdown_for_tts", side_effect=lambda t: t), \ + patch("tools.tts_tool._convert_to_opus", return_value="/tmp/test.ogg") as mock_convert, \ patch("os.path.isfile", return_value=True), \ patch("os.unlink"), \ patch("os.makedirs"): await runner._send_voice_reply(event, "Hello world") mock_adapter.send_voice.assert_called_once() - assert mock_tts.call_args.kwargs["output_path"].endswith(".ogg") + assert mock_tts.call_args.kwargs["output_path"].endswith(".mp3") + mock_convert.assert_called_once_with("/tmp/test.mp3") call_args = mock_adapter.send_voice.call_args assert call_args.kwargs.get("chat_id") == "123" + assert call_args.kwargs.get("audio_path") == "/tmp/test.ogg" @pytest.mark.asyncio async def test_non_telegram_auto_voice_reply_uses_mp3(self, runner): @@ -451,6 +454,7 @@ async def test_non_telegram_auto_voice_reply_uses_mp3(self, runner): with patch("tools.tts_tool.text_to_speech_tool", return_value=tts_result) as mock_tts, \ patch("tools.tts_tool._strip_markdown_for_tts", side_effect=lambda t: t), \ + patch("tools.tts_tool._convert_to_opus") as mock_convert, \ patch("os.path.isfile", return_value=True), \ patch("os.unlink"), \ patch("os.makedirs"): @@ -458,6 +462,32 @@ async def test_non_telegram_auto_voice_reply_uses_mp3(self, runner): mock_adapter.send_voice.assert_called_once() assert mock_tts.call_args.kwargs["output_path"].endswith(".mp3") + mock_convert.assert_not_called() + + @pytest.mark.asyncio + async def test_matrix_auto_voice_reply_converts_to_ogg_opus(self, runner): + from gateway.config import Platform + + mock_adapter = AsyncMock() + mock_adapter.send_voice = AsyncMock() + event = _make_event() + event.source.platform = Platform.MATRIX + runner.adapters[event.source.platform] = mock_adapter + + tts_result = json.dumps({"success": True, "file_path": "/tmp/test.mp3"}) + + with patch("tools.tts_tool.text_to_speech_tool", return_value=tts_result) as mock_tts, \ + patch("tools.tts_tool._strip_markdown_for_tts", side_effect=lambda t: t), \ + patch("tools.tts_tool._convert_to_opus", return_value="/tmp/test.ogg") as mock_convert, \ + patch("os.path.isfile", return_value=True), \ + patch("os.unlink"), \ + patch("os.makedirs"): + await runner._send_voice_reply(event, "Hello Matrix") + + mock_adapter.send_voice.assert_called_once() + assert mock_tts.call_args.kwargs["output_path"].endswith(".mp3") + mock_convert.assert_called_once_with("/tmp/test.mp3") + assert mock_adapter.send_voice.call_args.kwargs["audio_path"] == "/tmp/test.ogg" @pytest.mark.asyncio async def test_auto_voice_reply_uses_thread_metadata_helper(self, runner): diff --git a/tests/tools/test_tts_opus_routing.py b/tests/tools/test_tts_opus_routing.py index 0073146c3045..fee2fc93a739 100644 --- a/tests/tools/test_tts_opus_routing.py +++ b/tests/tools/test_tts_opus_routing.py @@ -68,3 +68,30 @@ def fake_convert(path: str) -> str: assert result["voice_compatible"] is True assert result["media_tag"] == f"[[audio_as_voice]]\nMEDIA:{opus}" convert.assert_called_once_with(str(out)) + + +def test_edge_matrix_converts_to_opus_voice(tmp_path, monkeypatch): + """Matrix voice bubbles need Ogg/Opus too (MSC3245, issue #14841).""" + out = tmp_path / "speech.mp3" + opus = tmp_path / "speech.ogg" + + def fake_convert(path: str) -> str: + assert path == str(out) + opus.write_bytes(b"ogg") + return str(opus) + + convert = Mock(side_effect=fake_convert) + + monkeypatch.setenv("HERMES_SESSION_PLATFORM", "matrix") + monkeypatch.setattr(tts_tool, "_load_tts_config", lambda: {"provider": "edge"}) + monkeypatch.setattr(tts_tool, "_import_edge_tts", lambda: object()) + monkeypatch.setattr(tts_tool, "_generate_edge_tts", _write_edge_output) + monkeypatch.setattr(tts_tool, "_convert_to_opus", convert) + + result = json.loads(tts_tool.text_to_speech_tool("hello", output_path=str(out))) + + assert result["success"] is True + assert result["file_path"] == str(opus) + assert result["voice_compatible"] is True + assert result["media_tag"] == f"[[audio_as_voice]]\nMEDIA:{opus}" + convert.assert_called_once_with(str(out)) diff --git a/tools/tts_tool.py b/tools/tts_tool.py index 545d72bb6907..b996c5bf18c0 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -2325,12 +2325,14 @@ def text_to_speech_tool( text = text[:max_len] # Detect platform from gateway env var to choose the best output format. - # Telegram voice bubbles require Opus (.ogg); OpenAI and ElevenLabs can - # produce Opus natively (no ffmpeg needed). Edge TTS always outputs MP3 - # and needs ffmpeg for conversion. + # Telegram and Matrix voice bubbles require Opus (.ogg) — Matrix per + # MSC3245 (issue #14841); OpenAI and ElevenLabs can produce Opus natively + # (no ffmpeg needed). Edge TTS always outputs MP3 and needs ffmpeg for + # conversion (handled below at the provider-specific conversion step, so + # extending want_opus here is safe for MP3-native providers too). from gateway.session_context import get_session_env platform = get_session_env("HERMES_SESSION_PLATFORM", "").lower() - want_opus = (platform == "telegram") + want_opus = platform in ("telegram", "matrix") # Determine output path if output_path: