From 1fdd8baf1ff630dd7957cb8905d418a40a9a9178 Mon Sep 17 00:00:00 2001 From: Tranquil-Flow Date: Tue, 14 Apr 2026 03:33:37 +1000 Subject: [PATCH 01/23] fix(stt): respect device and compute_type from config.yaml The local STT transcription function hardcoded device="auto" and compute_type="auto" when instantiating WhisperModel, ignoring the user's stt.local.device and stt.local.compute_type config values. Closes #8319 --- tests/tools/test_transcription_tools.py | 61 +++++++++++++++++++++++++ tools/transcription_tools.py | 24 ++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index 1674e0383aa2..231f0ba12732 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -611,6 +611,67 @@ def test_exception_returns_failure(self, tmp_path): assert result["success"] is False assert "CUDA out of memory" in result["error"] + def test_config_device_and_compute_type_passed_to_whisper(self, tmp_path): + """User-configured device and compute_type should be forwarded to WhisperModel. + + Regression test for #8319: these values were hardcoded to "auto". + """ + audio = tmp_path / "test.ogg" + audio.write_bytes(b"fake") + + mock_segment = MagicMock() + mock_segment.text = "hi" + mock_info = MagicMock() + mock_info.language = "en" + mock_info.duration = 1.0 + + mock_model = MagicMock() + mock_model.transcribe.return_value = ([mock_segment], mock_info) + mock_whisper_cls = MagicMock(return_value=mock_model) + + fake_config = { + "local": { + "device": "cpu", + "compute_type": "float32", + } + } + + with patch("tools.transcription_tools._HAS_FASTER_WHISPER", True), \ + patch("faster_whisper.WhisperModel", mock_whisper_cls), \ + patch("tools.transcription_tools._local_model", None), \ + patch("tools.transcription_tools._local_model_name", None), \ + patch("tools.transcription_tools._load_stt_config", return_value=fake_config): + from tools.transcription_tools import _transcribe_local + result = _transcribe_local(str(audio), "base") + + assert result["success"] is True + mock_whisper_cls.assert_called_once_with("base", device="cpu", compute_type="float32") + + def test_config_defaults_to_auto_when_not_set(self, tmp_path): + """Without config, device and compute_type should default to "auto".""" + audio = tmp_path / "test.ogg" + audio.write_bytes(b"fake") + + mock_segment = MagicMock() + mock_segment.text = "hi" + mock_info = MagicMock() + mock_info.language = "en" + mock_info.duration = 1.0 + + mock_model = MagicMock() + mock_model.transcribe.return_value = ([mock_segment], mock_info) + mock_whisper_cls = MagicMock(return_value=mock_model) + + with patch("tools.transcription_tools._HAS_FASTER_WHISPER", True), \ + patch("faster_whisper.WhisperModel", mock_whisper_cls), \ + patch("tools.transcription_tools._local_model", None), \ + patch("tools.transcription_tools._local_model_name", None), \ + patch("tools.transcription_tools._load_stt_config", return_value={}): + from tools.transcription_tools import _transcribe_local + _transcribe_local(str(audio), "base") + + mock_whisper_cls.assert_called_once_with("base", device="auto", compute_type="auto") + def test_multiple_segments_joined(self, tmp_path): audio = tmp_path / "test.ogg" audio.write_bytes(b"fake") diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index d8418dae878b..e16765c02d4a 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -1144,7 +1144,7 @@ def _looks_like_cuda_lib_error(exc: BaseException) -> bool: return any(marker in msg for marker in _CUDA_LIB_ERROR_MARKERS) -def _load_local_whisper_model(model_name: str): +def _load_local_whisper_model(model_name: str, device: str = "auto", compute_type: str = "auto"): """Load faster-whisper with graceful CUDA → CPU fallback. faster-whisper's ``device="auto"`` picks CUDA when the ctranslate2 wheel @@ -1154,12 +1154,16 @@ def _load_local_whisper_model(model_name: str): On those hosts the load itself sometimes succeeds and the dlopen failure only surfaces at first ``transcribe()`` call. - We try ``auto`` first (fast CUDA path when it works), and on any CUDA - library load failure fall back to CPU + int8. + ``device`` / ``compute_type`` default to ``"auto"`` so the historical + behaviour is unchanged; pass explicit values from ``stt.local.device`` / + ``stt.local.compute_type`` to pin a configuration (#9088). + + We try the requested config first (fast CUDA path when it works), and on + any CUDA library load failure fall back to CPU + int8. """ from faster_whisper import WhisperModel try: - return WhisperModel(model_name, device="auto", compute_type="auto") + return WhisperModel(model_name, device=device, compute_type=compute_type) except Exception as exc: if not _looks_like_cuda_lib_error(exc): raise @@ -1180,10 +1184,20 @@ def _transcribe_local(file_path: str, model_name: str) -> Dict[str, Any]: return {"success": False, "transcript": "", "error": "faster-whisper not installed"} try: + local_cfg = _load_stt_config().get("local", {}) # Lazy-load the model (downloads on first use, ~150 MB for 'base') if _local_model is None or _local_model_name != model_name: logger.info("Loading faster-whisper model '%s' (first load downloads the model)...", model_name) - _local_model = _load_local_whisper_model(model_name) + # Honour stt.local.device / stt.local.compute_type from config so + # users on hosts where ``auto`` mis-detects (NVIDIA libs present but + # not usable, etc.) can pin a working configuration (#9088). + # _load_local_whisper_model retains the CUDA→CPU fallback for the + # auto/CUDA paths. + _local_model = _load_local_whisper_model( + model_name, + device=local_cfg.get("device", "auto"), + compute_type=local_cfg.get("compute_type", "auto"), + ) _local_model_name = model_name # Language: stt.local.language > stt.language > env var > auto-detect. From 5d86ba6179b8b7e141617b2a2fb26321563e010a Mon Sep 17 00:00:00 2001 From: liuhao1024 Date: Thu, 30 Apr 2026 00:10:23 +0800 Subject: [PATCH 02/23] fix(stt): treat CUBLAS_STATUS_NOT_SUPPORTED as CUDA lib error - Add Blackwell-specific cuBLAS error marker to _CUDA_LIB_ERROR_MARKERS - Allows CPU fallback on RTX 5090 (sm_120) when faster-whisper reports CUBLAS_STATUS_NOT_SUPPORTED instead of loading successfully - Add regression test for CUBLAS_STATUS_NOT_SUPPORTED path Closes #17526 --- tests/tools/test_transcription_tools.py | 36 +++++++++++++++++++++++++ tools/transcription_tools.py | 1 + 2 files changed, 37 insertions(+) diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index 231f0ba12732..446304dcb1ac 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -772,6 +772,42 @@ def fake_whisper(model_name, device, compute_type): assert gpu_model.transcribe.call_count == 1 assert cpu_model.transcribe.call_count == 1 + def test_cublas_status_not_supported_retries_on_cpu(self, tmp_path): + """Blackwell cuBLAS unsupported errors should use the CPU fallback path.""" + audio = tmp_path / "test.ogg" + audio.write_bytes(b"fake") + + seg = MagicMock() + seg.text = "blackwell fallback" + info = MagicMock() + info.language = "en" + info.duration = 1.0 + + gpu_model = MagicMock() + gpu_model.transcribe.side_effect = RuntimeError( + "cuBLAS failed with status CUBLAS_STATUS_NOT_SUPPORTED" + ) + cpu_model = MagicMock() + cpu_model.transcribe.return_value = ([seg], info) + + models = [gpu_model, cpu_model] + call_args = [] + + def fake_whisper(model_name, device, compute_type): + call_args.append((device, compute_type)) + return models.pop(0) + + with patch("tools.transcription_tools._HAS_FASTER_WHISPER", True), \ + patch("faster_whisper.WhisperModel", side_effect=fake_whisper), \ + patch("tools.transcription_tools._local_model", None), \ + patch("tools.transcription_tools._local_model_name", None): + from tools.transcription_tools import _transcribe_local + result = _transcribe_local(str(audio), "base") + + assert result["success"] is True + assert result["transcript"] == "blackwell fallback" + assert call_args == [("auto", "auto"), ("cpu", "int8")] + def test_cuda_out_of_memory_does_not_trigger_cpu_fallback(self, tmp_path): """'CUDA out of memory' is a real error, not a missing lib — surface it.""" audio = tmp_path / "test.ogg" diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index e16765c02d4a..3611089da650 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -1127,6 +1127,7 @@ def _validate_audio_file(file_path: str) -> Optional[Dict[str, Any]]: "cannot be loaded", "cannot open shared object", "no kernel image is available", + "CUBLAS_STATUS_NOT_SUPPORTED", "no CUDA-capable device", "CUDA driver version is insufficient", ) From abd5bdf994c736c14aeeadcefc981916250c1a43 Mon Sep 17 00:00:00 2001 From: AnthonyAssistantAi Date: Tue, 28 Jul 2026 09:25:50 -0700 Subject: [PATCH 03/23] fix: avoid local STT crash on Apple Silicon Force CPU (int8) for faster-whisper on Apple Silicon / Rosetta, where ctranslate2's device=auto path can hard-abort in native code. Salvaged from PR #28624 without the numpy pin change (main already moved on). (cherry picked from commit 7edf2d5196, pyproject.toml hunk dropped) --- tests/tools/test_transcription_tools.py | 47 ++++++++++++++++++++++ tools/transcription_tools.py | 52 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index 446304dcb1ac..22ba05495459 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -696,6 +696,51 @@ def test_multiple_segments_joined(self, tmp_path): assert result["success"] is True assert result["transcript"] == "Hello world" + def test_apple_silicon_forces_cpu_without_auto_probe(self, tmp_path): + """Apple Silicon/Rosetta should skip device='auto' to avoid SIGABRT.""" + audio = tmp_path / "test.ogg" + audio.write_bytes(b"fake") + + seg = MagicMock() + seg.text = "safe" + info = MagicMock() + info.language = "en" + info.duration = 1.0 + cpu_model = MagicMock() + cpu_model.transcribe.return_value = ([seg], info) + mock_whisper_cls = MagicMock(return_value=cpu_model) + + with patch("tools.transcription_tools._HAS_FASTER_WHISPER", True), \ + patch("tools.transcription_tools._should_force_faster_whisper_cpu", return_value=True), \ + patch("faster_whisper.WhisperModel", mock_whisper_cls), \ + patch("tools.transcription_tools._local_model", None), \ + patch("tools.transcription_tools._local_model_name", None): + from tools.transcription_tools import _transcribe_local + result = _transcribe_local(str(audio), "base") + + assert result["success"] is True + assert result["transcript"] == "safe" + mock_whisper_cls.assert_called_once_with("base", device="cpu", compute_type="int8") + + def test_force_cpu_detects_rosetta_on_apple_silicon(self): + from tools.transcription_tools import _should_force_faster_whisper_cpu + + with patch("tools.transcription_tools.platform.system", return_value="Darwin"), \ + patch("tools.transcription_tools.platform.machine", return_value="x86_64"), \ + patch("tools.transcription_tools._sysctl_value", side_effect=lambda key: { + "sysctl.proc_translated": "1", + "hw.optional.arm64": "1", + }.get(key, "")): + assert _should_force_faster_whisper_cpu() is True + + def test_force_cpu_false_on_intel_macos(self): + from tools.transcription_tools import _should_force_faster_whisper_cpu + + with patch("tools.transcription_tools.platform.system", return_value="Darwin"), \ + patch("tools.transcription_tools.platform.machine", return_value="x86_64"), \ + patch("tools.transcription_tools._sysctl_value", return_value="0"): + assert _should_force_faster_whisper_cpu() is False + def test_load_time_cuda_lib_failure_falls_back_to_cpu(self, tmp_path): """Missing libcublas at load time → reload on CPU, succeed.""" audio = tmp_path / "test.ogg" @@ -719,6 +764,7 @@ def fake_whisper(model_name, device, compute_type): return cpu_model with patch("tools.transcription_tools._HAS_FASTER_WHISPER", True), \ + patch("tools.transcription_tools._should_force_faster_whisper_cpu", return_value=False), \ patch("faster_whisper.WhisperModel", side_effect=fake_whisper), \ patch("tools.transcription_tools._local_model", None), \ patch("tools.transcription_tools._local_model_name", None): @@ -757,6 +803,7 @@ def fake_whisper(model_name, device, compute_type): return models.pop(0) with patch("tools.transcription_tools._HAS_FASTER_WHISPER", True), \ + patch("tools.transcription_tools._should_force_faster_whisper_cpu", return_value=False), \ patch("faster_whisper.WhisperModel", side_effect=fake_whisper), \ patch("tools.transcription_tools._local_model", None), \ patch("tools.transcription_tools._local_model_name", None): diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index 3611089da650..ece0b7df0281 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -29,6 +29,7 @@ import logging import os +import platform import shlex import shutil import subprocess @@ -1145,6 +1146,42 @@ def _looks_like_cuda_lib_error(exc: BaseException) -> bool: return any(marker in msg for marker in _CUDA_LIB_ERROR_MARKERS) +def _sysctl_value(name: str) -> str: + """Return a sysctl value, or an empty string when unavailable.""" + try: + return subprocess.check_output( + ["/usr/sbin/sysctl", "-n", name], + stderr=subprocess.DEVNULL, + text=True, + timeout=2, + ).strip() + except Exception: + return "" + + +def _should_force_faster_whisper_cpu() -> bool: + """Avoid faster-whisper device autodetection paths known to hard-abort. + + On Apple Silicon, especially when Python is running as x86_64 under + Rosetta, ctranslate2's ``device=\"auto\"`` path can abort inside native + code before Python can catch an exception. Force CPU so local STT remains + reliable for gateway voice messages. + """ + if platform.system() != "Darwin": + return False + + machine = platform.machine().lower() + if machine in {"arm64", "aarch64"}: + return True + + # Under Rosetta, platform.machine() reports x86_64. sysctl.proc_translated + # tells us this process is translated, while hw.optional.arm64 distinguishes + # Apple Silicon hosts from Intel Macs. + if _sysctl_value("sysctl.proc_translated") == "1": + return True + return _sysctl_value("hw.optional.arm64") == "1" + + def _load_local_whisper_model(model_name: str, device: str = "auto", compute_type: str = "auto"): """Load faster-whisper with graceful CUDA → CPU fallback. @@ -1162,7 +1199,22 @@ def _load_local_whisper_model(model_name: str, device: str = "auto", compute_typ We try the requested config first (fast CUDA path when it works), and on any CUDA library load failure fall back to CPU + int8. """ + force_cpu = _should_force_faster_whisper_cpu() + if force_cpu: + # Importing ctranslate2/faster-whisper itself can abort on some + # Apple Silicon/Rosetta installs because multiple Intel OpenMP runtimes + # are already loaded. Set this before importing faster_whisper so the + # gateway survives, then keep inference on CPU to avoid device probing. + os.environ.setdefault("KMP_DUPLICATE_LIB_OK", "TRUE") + from faster_whisper import WhisperModel + if force_cpu: + logger.info( + "Apple Silicon/Rosetta detected — loading faster-whisper on CPU " + "(int8) to avoid native device autodetection crashes" + ) + return WhisperModel(model_name, device="cpu", compute_type="int8") + try: return WhisperModel(model_name, device=device, compute_type=compute_type) except Exception as exc: From 4d89d79c1464e3ab8ca47c783b066e914d897c44 Mon Sep 17 00:00:00 2001 From: Charles Cha <92324143+ypwcharles@users.noreply.github.com> Date: Fri, 17 Jul 2026 00:46:40 +0800 Subject: [PATCH 04/23] fix(stt): scope upload size limits to remote providers --- .../test_transcription_command_providers.py | 28 ++++- .../test_transcription_plugin_dispatch.py | 95 +++++++++------ tests/tools/test_transcription_tools.py | 45 +++++++ tests/tools/test_voice_mode.py | 112 ++++++++++++++++++ tools/transcription_tools.py | 49 ++++++-- tools/voice_mode.py | 9 +- 6 files changed, 280 insertions(+), 58 deletions(-) diff --git a/tests/tools/test_transcription_command_providers.py b/tests/tools/test_transcription_command_providers.py index 749ab5e839c3..10d6e9b332a8 100644 --- a/tests/tools/test_transcription_command_providers.py +++ b/tests/tools/test_transcription_command_providers.py @@ -396,7 +396,7 @@ def test_model_override_passed_to_template(self, tmp_path): audio = _make_silent_wav(tmp_path / "input.wav") # Write the model into the transcript so we can assert it propagated. interpreter = sys.executable - payload = "import sys; open(sys.argv[2], 'w').write(sys.argv[1])" + payload = "import sys; open(sys.argv[2], 'w', encoding='utf-8').write(sys.argv[1])" cfg = { "command": f'"{interpreter}" -c "{payload}" {{model}} {{output_path}}', "model": "config-model", @@ -410,7 +410,7 @@ def test_model_override_passed_to_template(self, tmp_path): def test_config_model_used_when_no_override(self, tmp_path): audio = _make_silent_wav(tmp_path / "input.wav") interpreter = sys.executable - payload = "import sys; open(sys.argv[2], 'w').write(sys.argv[1])" + payload = "import sys; open(sys.argv[2], 'w', encoding='utf-8').write(sys.argv[1])" cfg = { "command": f'"{interpreter}" -c "{payload}" {{model}} {{output_path}}', "model": "config-model", @@ -421,7 +421,7 @@ def test_config_model_used_when_no_override(self, tmp_path): def test_language_from_provider_config_wins(self, tmp_path): audio = _make_silent_wav(tmp_path / "input.wav") interpreter = sys.executable - payload = "import sys; open(sys.argv[2], 'w').write(sys.argv[1])" + payload = "import sys; open(sys.argv[2], 'w', encoding='utf-8').write(sys.argv[1])" cfg = { "command": f'"{interpreter}" -c "{payload}" {{language}} {{output_path}}', "language": "fr", @@ -435,7 +435,7 @@ def test_language_from_provider_config_wins(self, tmp_path): def test_language_falls_back_to_stt_section(self, tmp_path): audio = _make_silent_wav(tmp_path / "input.wav") interpreter = sys.executable - payload = "import sys; open(sys.argv[2], 'w').write(sys.argv[1])" + payload = "import sys; open(sys.argv[2], 'w', encoding='utf-8').write(sys.argv[1])" cfg = { "command": f'"{interpreter}" -c "{payload}" {{language}} {{output_path}}', } @@ -447,7 +447,7 @@ def test_language_falls_back_to_stt_section(self, tmp_path): def test_language_defaults_to_en(self, tmp_path): audio = _make_silent_wav(tmp_path / "input.wav") interpreter = sys.executable - payload = "import sys; open(sys.argv[2], 'w').write(sys.argv[1])" + payload = "import sys; open(sys.argv[2], 'w', encoding='utf-8').write(sys.argv[1])" cfg = { "command": f'"{interpreter}" -c "{payload}" {{language}} {{output_path}}', } @@ -486,6 +486,24 @@ def test_command_provider_dispatches_via_transcribe_audio(self, tmp_path): assert result["transcript"] == "dispatched via command" assert result["provider"] == "fake-cli" + def test_oversized_command_provider_file_is_rejected(self, tmp_path): + from tools.transcription_tools import MAX_FILE_SIZE + + audio = tmp_path / "oversized.wav" + with audio.open("wb") as audio_file: + audio_file.seek(MAX_FILE_SIZE) + audio_file.write(b"\0") + cfg = self._config_with_command_provider("fake-cli", "unused {input_path}") + + with patch("tools.transcription_tools._load_stt_config", return_value=cfg), \ + patch("tools.transcription_tools._transcribe_command_stt", + return_value={"success": True, "transcript": "hi"}) as mock_command: + result = transcribe_audio(str(audio)) + + assert result["success"] is False + assert "File too large" in result["error"] + mock_command.assert_not_called() + def test_builtin_name_shadow_does_not_route_to_command(self, tmp_path): # User mis-configures stt.providers.openai as a command — must NOT # hijack the real OpenAI built-in. The built-in elif chain owns diff --git a/tests/tools/test_transcription_plugin_dispatch.py b/tests/tools/test_transcription_plugin_dispatch.py index 834246769527..d7ea8dac4ee3 100644 --- a/tests/tools/test_transcription_plugin_dispatch.py +++ b/tests/tools/test_transcription_plugin_dispatch.py @@ -66,6 +66,14 @@ def _reset_registry(): transcription_registry._reset_for_tests() +@pytest.fixture +def sample_audio_file(tmp_path): + """Return a real, small input so E2E tests exercise validation too.""" + audio_path = tmp_path / "audio.mp3" + audio_path.write_bytes(b"fake audio data") + return str(audio_path) + + # --------------------------------------------------------------------------- # Built-in always wins # --------------------------------------------------------------------------- @@ -209,42 +217,39 @@ class TestTranscribeAudioE2E: """transcribe_audio() routes plugin dispatch correctly when the configured name is unknown to the built-in branches. - Note: we mock _validate_audio_file and _get_provider so the real - file-validation and provider-resolution don't fire — we're testing - the plugin-dispatch wiring, not those helpers. + Provider resolution is mocked to isolate routing, while a real small + input keeps the validation chain active. """ - def test_unknown_name_with_plugin_dispatches(self): + def test_unknown_name_with_plugin_dispatches(self, sample_audio_file): from unittest.mock import patch provider = _FakeProvider(name="openrouter") transcription_registry.register_provider(provider) - with patch("tools.transcription_tools._validate_audio_file", return_value=None), \ - patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openrouter"}), \ + with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openrouter"}), \ patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ patch("tools.transcription_tools._get_provider", return_value="openrouter"): - result = transcription_tools.transcribe_audio("/tmp/audio.mp3") + result = transcription_tools.transcribe_audio(sample_audio_file) assert result["success"] is True assert result["transcript"] == "fake transcript" assert result["provider"] == "openrouter" - def test_unknown_name_without_plugin_falls_to_legacy_error(self): + def test_unknown_name_without_plugin_falls_to_legacy_error(self, sample_audio_file): """When no plugin is registered for the unknown name, the dispatcher returns None and transcribe_audio falls through to the legacy 'No STT provider available' error message.""" from unittest.mock import patch - with patch("tools.transcription_tools._validate_audio_file", return_value=None), \ - patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openrouter"}), \ + with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openrouter"}), \ patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ patch("tools.transcription_tools._get_provider", return_value="openrouter"): - result = transcription_tools.transcribe_audio("/tmp/audio.mp3") + result = transcription_tools.transcribe_audio(sample_audio_file) assert result["success"] is False assert "No STT provider" in result["error"] - def test_builtin_name_does_not_consult_plugin_registry(self): + def test_builtin_name_does_not_consult_plugin_registry(self, sample_audio_file): """Even if a plugin's name collides with a built-in (which the registry blocks, but defense in depth matters), transcribe_audio with provider='groq' goes through the legacy elif chain, never @@ -255,12 +260,11 @@ def test_builtin_name_does_not_consult_plugin_registry(self): provider = _FakeProvider(name="openrouter") transcription_registry.register_provider(provider) - with patch("tools.transcription_tools._validate_audio_file", return_value=None), \ - patch("tools.transcription_tools._load_stt_config", return_value={"provider": "groq"}), \ + with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "groq"}), \ patch("tools.transcription_tools._get_provider", return_value="groq"), \ patch("tools.transcription_tools._transcribe_groq", return_value={"success": True, "transcript": "from groq", "provider": "groq"}) as mock_groq: - result = transcription_tools.transcribe_audio("/tmp/audio.mp3") + result = transcription_tools.transcribe_audio(sample_audio_file) assert result["provider"] == "groq" assert result["transcript"] == "from groq" @@ -268,6 +272,25 @@ def test_builtin_name_does_not_consult_plugin_registry(self): # Plugin was never called assert provider.last_call is None + def test_oversized_plugin_file_is_rejected_before_dispatch(self, tmp_path): + from unittest.mock import patch + + provider = _FakeProvider(name="openrouter") + transcription_registry.register_provider(provider) + audio_path = tmp_path / "oversized.mp3" + with audio_path.open("wb") as audio_file: + audio_file.seek(transcription_tools.MAX_FILE_SIZE) + audio_file.write(b"\0") + + with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openrouter"}), \ + patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ + patch("tools.transcription_tools._get_provider", return_value="openrouter"): + result = transcription_tools.transcribe_audio(str(audio_path)) + + assert result["success"] is False + assert "File too large" in result["error"] + assert provider.last_call is None + # --------------------------------------------------------------------------- # Availability gating (codex review feedback on PR #30493) @@ -330,7 +353,7 @@ def test_is_available_raising_treated_as_unavailable(self): assert "not available" in result["error"] assert provider.last_call is None - def test_unavailable_plugin_at_transcribe_audio_level(self): + def test_unavailable_plugin_at_transcribe_audio_level(self, sample_audio_file): """End-to-end: ``stt.provider: openrouter`` + plugin reports unavailable → ``transcribe_audio`` returns the unavailability envelope, NOT the generic "No STT provider available" message. @@ -339,11 +362,10 @@ def test_unavailable_plugin_at_transcribe_audio_level(self): provider = _FakeProvider(name="openrouter", available=False) transcription_registry.register_provider(provider) - with patch("tools.transcription_tools._validate_audio_file", return_value=None), \ - patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openrouter"}), \ + with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openrouter"}), \ patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ patch("tools.transcription_tools._get_provider", return_value="openrouter"): - result = transcription_tools.transcribe_audio("/tmp/audio.mp3") + result = transcription_tools.transcribe_audio(sample_audio_file) assert result["success"] is False # Must surface the plugin's unavailability — NOT the generic @@ -364,7 +386,7 @@ class TestLanguageForwardingFromConfig: ``stt.local.language``). """ - def test_language_read_from_provider_namespaced_config(self): + def test_language_read_from_provider_namespaced_config(self, sample_audio_file): """``stt.openrouter.language: ja`` reaches the plugin's transcribe() call as language='ja'.""" from unittest.mock import patch @@ -375,16 +397,15 @@ def test_language_read_from_provider_namespaced_config(self): "provider": "openrouter", "openrouter": {"language": "ja"}, } - with patch("tools.transcription_tools._validate_audio_file", return_value=None), \ - patch("tools.transcription_tools._load_stt_config", return_value=stt_config), \ + with patch("tools.transcription_tools._load_stt_config", return_value=stt_config), \ patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ patch("tools.transcription_tools._get_provider", return_value="openrouter"): - transcription_tools.transcribe_audio("/tmp/audio.mp3") + transcription_tools.transcribe_audio(sample_audio_file) assert provider.last_call is not None assert provider.last_call["kwargs"]["language"] == "ja" - def test_model_from_provider_namespaced_config(self): + def test_model_from_provider_namespaced_config(self, sample_audio_file): """``stt.openrouter.model: whisper-large-v3`` reaches the plugin as model='whisper-large-v3' when caller doesn't override.""" @@ -396,15 +417,14 @@ def test_model_from_provider_namespaced_config(self): "provider": "openrouter", "openrouter": {"model": "whisper-large-v3"}, } - with patch("tools.transcription_tools._validate_audio_file", return_value=None), \ - patch("tools.transcription_tools._load_stt_config", return_value=stt_config), \ + with patch("tools.transcription_tools._load_stt_config", return_value=stt_config), \ patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ patch("tools.transcription_tools._get_provider", return_value="openrouter"): - transcription_tools.transcribe_audio("/tmp/audio.mp3") + transcription_tools.transcribe_audio(sample_audio_file) assert provider.last_call["kwargs"]["model"] == "whisper-large-v3" - def test_caller_model_overrides_config_model(self): + def test_caller_model_overrides_config_model(self, sample_audio_file): """An explicit ``model`` arg to transcribe_audio wins over ``stt..model`` in config.""" from unittest.mock import patch @@ -415,33 +435,31 @@ def test_caller_model_overrides_config_model(self): "provider": "openrouter", "openrouter": {"model": "config-model"}, } - with patch("tools.transcription_tools._validate_audio_file", return_value=None), \ - patch("tools.transcription_tools._load_stt_config", return_value=stt_config), \ + with patch("tools.transcription_tools._load_stt_config", return_value=stt_config), \ patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ patch("tools.transcription_tools._get_provider", return_value="openrouter"): transcription_tools.transcribe_audio( - "/tmp/audio.mp3", model="explicit-arg-model", + sample_audio_file, model="explicit-arg-model", ) assert provider.last_call["kwargs"]["model"] == "explicit-arg-model" - def test_missing_provider_namespace_passes_none(self): + def test_missing_provider_namespace_passes_none(self, sample_audio_file): """No ``stt.`` subsection → language is None, model falls back to caller arg or None. No crash.""" from unittest.mock import patch provider = _FakeProvider(name="openrouter") transcription_registry.register_provider(provider) - with patch("tools.transcription_tools._validate_audio_file", return_value=None), \ - patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openrouter"}), \ + with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openrouter"}), \ patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ patch("tools.transcription_tools._get_provider", return_value="openrouter"): - transcription_tools.transcribe_audio("/tmp/audio.mp3") + transcription_tools.transcribe_audio(sample_audio_file) assert provider.last_call["kwargs"]["language"] is None assert provider.last_call["kwargs"]["model"] is None - def test_non_dict_provider_namespace_does_not_crash(self): + def test_non_dict_provider_namespace_does_not_crash(self, sample_audio_file): """If someone accidentally writes ``stt.openrouter: "foo"`` (a string instead of a dict), we should not crash — treat as empty config.""" @@ -450,11 +468,10 @@ def test_non_dict_provider_namespace_does_not_crash(self): transcription_registry.register_provider(provider) stt_config = {"provider": "openrouter", "openrouter": "garbage"} - with patch("tools.transcription_tools._validate_audio_file", return_value=None), \ - patch("tools.transcription_tools._load_stt_config", return_value=stt_config), \ + with patch("tools.transcription_tools._load_stt_config", return_value=stt_config), \ patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ patch("tools.transcription_tools._get_provider", return_value="openrouter"): - result = transcription_tools.transcribe_audio("/tmp/audio.mp3") + result = transcription_tools.transcribe_audio(sample_audio_file) # Should still dispatch successfully (config is just ignored) assert result["success"] is True diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index 22ba05495459..1584d9cbaba2 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -54,6 +54,18 @@ def sample_ogg(tmp_path): return str(ogg_path) +@pytest.fixture +def oversized_wav(tmp_path): + """Create a sparse WAV-shaped file just above the remote upload cap.""" + from tools.transcription_tools import MAX_FILE_SIZE + + wav_path = tmp_path / "oversized.wav" + with wav_path.open("wb") as audio_file: + audio_file.seek(MAX_FILE_SIZE) + audio_file.write(b"\0") + return str(wav_path) + + pytestmark = pytest.mark.usefixtures("disable_lazy_stt_install") @@ -1100,6 +1112,39 @@ def test_dispatches_to_local(self, sample_ogg): assert result["success"] is True mock_local.assert_called_once() + def test_oversized_local_file_reaches_dispatcher(self, oversized_wav): + with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "local"}), \ + patch("tools.transcription_tools._get_provider", return_value="local"), \ + patch("tools.transcription_tools._transcribe_local", + return_value={"success": True, "transcript": "hi"}) as mock_local: + from tools.transcription_tools import transcribe_audio + result = transcribe_audio(oversized_wav) + + assert result["success"] is True + mock_local.assert_called_once() + + def test_oversized_local_command_file_reaches_dispatcher(self, oversized_wav): + with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "local_command"}), \ + patch("tools.transcription_tools._get_provider", return_value="local_command"), \ + patch("tools.transcription_tools._transcribe_local_command", + return_value={"success": True, "transcript": "hi"}) as mock_command: + from tools.transcription_tools import transcribe_audio + result = transcribe_audio(oversized_wav) + + assert result["success"] is True + mock_command.assert_called_once() + + def test_oversized_remote_file_is_rejected_before_dispatch(self, oversized_wav): + with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openai"}), \ + patch("tools.transcription_tools._get_provider", return_value="openai"), \ + patch("tools.transcription_tools._transcribe_openai") as mock_openai: + from tools.transcription_tools import transcribe_audio + result = transcribe_audio(oversized_wav) + + assert result["success"] is False + assert "File too large" in result["error"] + mock_openai.assert_not_called() + def test_dispatches_to_openai(self, sample_ogg): with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openai"}), \ patch("tools.transcription_tools._get_provider", return_value="openai"), \ diff --git a/tests/tools/test_voice_mode.py b/tests/tools/test_voice_mode.py index 4611be901531..87ccdb297057 100644 --- a/tests/tools/test_voice_mode.py +++ b/tests/tools/test_voice_mode.py @@ -837,9 +837,20 @@ def test_oversized_wav_is_chunked_and_stitched(self, tmp_path, monkeypatch): monkeypatch.setattr("tools.voice_mode._TEMP_DIR", str(temp_dir)) monkeypatch.setattr("tools.transcription_tools.MAX_FILE_SIZE", 70 * 1024) + call_count = 0 seen_paths = [] def fake_transcribe(path, model=None): + nonlocal call_count + call_count += 1 + # First call is on the original file — simulate remote provider + # rejecting it as too large so chunking kicks in. + if call_count == 1: + return { + "success": False, + "transcript": "", + "error": "File too large: 0.1MB (max 0.1MB)", + } seen_paths.append(path) assert model == "base" assert path != str(wav_path) @@ -877,7 +888,17 @@ def test_oversized_wav_reports_failing_chunk(self, tmp_path, monkeypatch): monkeypatch.setattr("tools.voice_mode._TEMP_DIR", str(temp_dir)) monkeypatch.setattr("tools.transcription_tools.MAX_FILE_SIZE", 70 * 1024) + call_count = 0 + def fake_transcribe(path, model=None): + nonlocal call_count + call_count += 1 + if call_count == 1: + return { + "success": False, + "transcript": "", + "error": "File too large: 0.1MB (max 0.1MB)", + } return {"success": False, "transcript": "", "error": "provider rejected audio"} with patch("tools.transcription_tools.transcribe_audio", side_effect=fake_transcribe): @@ -889,6 +910,97 @@ def fake_transcribe(path, model=None): assert "provider rejected audio" in result["error"] assert list(temp_dir.iterdir()) == [] + def test_trusts_transcribe_audio_skip_chunk_for_local(self, tmp_path, monkeypatch): + """Local providers never return 'File too large' — no chunking.""" + wav_path = tmp_path / "record.wav" + n_frames = 50000 + audio = struct.pack(f"<{n_frames}h", *([1000] * n_frames)) + with wave.open(str(wav_path), "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(16000) + wf.writeframes(audio) + + mock_transcribe = MagicMock(return_value={ + "success": True, + "transcript": "local whisper result", + "provider": "local", + }) + + with patch("tools.transcription_tools.transcribe_audio", mock_transcribe): + from tools.voice_mode import transcribe_recording + result = transcribe_recording(str(wav_path), model="base") + + assert result["success"] is True + assert result["transcript"] == "local whisper result" + assert "chunks" not in result + mock_transcribe.assert_called_once_with(str(wav_path), model="base") + + def test_chunks_when_transcribe_audio_returns_file_too_large(self, tmp_path, monkeypatch): + """Remote provider rejects large file → chunking fallback.""" + wav_path = tmp_path / "record.wav" + n_frames = 50000 + audio = struct.pack(f"<{n_frames}h", *([1000] * n_frames)) + with wave.open(str(wav_path), "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(16000) + wf.writeframes(audio) + + temp_dir = tmp_path / "chunks" + temp_dir.mkdir() + monkeypatch.setattr("tools.voice_mode._TEMP_DIR", str(temp_dir)) + monkeypatch.setattr("tools.transcription_tools.MAX_FILE_SIZE", 70 * 1024) + + call_count = 0 + + def fake_transcribe(path, model=None): + nonlocal call_count + call_count += 1 + if call_count == 1: + return { + "success": False, + "transcript": "", + "error": "File too large: 30.0MB (max 25MB)", + } + return { + "success": True, + "transcript": f"chunk {call_count - 1}", + "provider": "openai", + } + + with patch("tools.transcription_tools.transcribe_audio", side_effect=fake_transcribe): + from tools.voice_mode import transcribe_recording + result = transcribe_recording(str(wav_path), model="whisper-1") + + assert result["success"] is True + assert result.get("chunks", 0) > 1 + + def test_other_error_does_not_trigger_chunk(self, tmp_path, monkeypatch): + """Non-size errors from transcribe_audio are returned as-is.""" + wav_path = tmp_path / "record.wav" + n_frames = 50000 + audio = struct.pack(f"<{n_frames}h", *([1000] * n_frames)) + with wave.open(str(wav_path), "wb") as wf: + wf.setnchannels(1) + wf.setsampwidth(2) + wf.setframerate(16000) + wf.writeframes(audio) + + mock_transcribe = MagicMock(return_value={ + "success": False, + "transcript": "", + "error": "STT is disabled in config.yaml", + }) + + with patch("tools.transcription_tools.transcribe_audio", mock_transcribe): + from tools.voice_mode import transcribe_recording + result = transcribe_recording(str(wav_path), model="base") + + assert result["success"] is False + assert "STT is disabled" in result["error"] + mock_transcribe.assert_called_once() + class TestWhisperHallucinationFilter: def test_known_hallucinations(self): diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index ece0b7df0281..fdb6d72cfedc 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -377,6 +377,14 @@ def _resolve_command_stt_provider_config( return None +def _is_local_stt_provider(provider: str, stt_config: Dict[str, Any]) -> bool: + """Return whether *provider* is exempt from Hermes's remote upload cap.""" + key = (provider or "").lower().strip() + if key in {"local", "local_command"}: + return True + return False + + def _iter_command_stt_providers(stt_config: Dict[str, Any]): """Yield (name, config) pairs for every declared command-type STT provider.""" if not isinstance(stt_config, dict): @@ -1078,7 +1086,26 @@ def _dispatch_to_plugin_provider( # --------------------------------------------------------------------------- -def _validate_audio_file(file_path: str) -> Optional[Dict[str, Any]]: +def _validate_audio_file_size(audio_path: Path) -> Optional[Dict[str, Any]]: + """Return an error when *audio_path* exceeds the remote upload cap.""" + try: + file_size = audio_path.stat().st_size + except OSError as e: + return {"success": False, "transcript": "", "error": f"Failed to access file: {e}"} + if file_size > MAX_FILE_SIZE: + return { + "success": False, + "transcript": "", + "error": f"File too large: {file_size / (1024*1024):.1f}MB (max {MAX_FILE_SIZE / (1024*1024):.0f}MB)", + } + return None + + +def _validate_audio_file( + file_path: str, + *, + enforce_size_limit: bool = True, +) -> Optional[Dict[str, Any]]: """Validate the audio file. Returns an error dict or None if OK.""" audio_path = Path(file_path) @@ -1094,17 +1121,12 @@ def _validate_audio_file(file_path: str) -> Optional[Dict[str, Any]]: "transcript": "", "error": f"Unsupported format: {audio_path.suffix}. Supported: {', '.join(sorted(SUPPORTED_FORMATS))}", } + if enforce_size_limit: + return _validate_audio_file_size(audio_path) try: - file_size = audio_path.stat().st_size - if file_size > MAX_FILE_SIZE: - return { - "success": False, - "transcript": "", - "error": f"File too large: {file_size / (1024*1024):.1f}MB (max {MAX_FILE_SIZE / (1024*1024):.0f}MB)", - } + audio_path.stat() except OSError as e: return {"success": False, "transcript": "", "error": f"Failed to access file: {e}"} - return None # --------------------------------------------------------------------------- @@ -1858,8 +1880,9 @@ def transcribe_audio(file_path: str, model: Optional[str] = None) -> Dict[str, A - "error" (str, optional): Error message if success is False - "provider" (str, optional): Which provider was used """ - # Validate input - error = _validate_audio_file(file_path) + # Apply common path validation before provider resolution so invalid files + # cannot trigger provider setup or lazy installation. + error = _validate_audio_file(file_path, enforce_size_limit=False) if error: return error @@ -1873,6 +1896,10 @@ def transcribe_audio(file_path: str, model: Optional[str] = None) -> Dict[str, A } provider = _get_provider(stt_config) + if not _is_local_stt_provider(provider, stt_config): + error = _validate_audio_file_size(Path(file_path)) + if error: + return error if provider == "local": local_cfg = stt_config.get("local") or {} diff --git a/tools/voice_mode.py b/tools/voice_mode.py index 093534c3be52..6f8b22bd520f 100644 --- a/tools/voice_mode.py +++ b/tools/voice_mode.py @@ -941,10 +941,13 @@ def transcribe_recording(wav_path: str, model: Optional[str] = None) -> Dict[str """ from tools.transcription_tools import MAX_FILE_SIZE, transcribe_audio - if _should_chunk_for_transcription(wav_path, MAX_FILE_SIZE): + result = transcribe_audio(wav_path, model=model) + + # Only chunk when the provider itself reports "File too large" — + # local providers (faster-whisper, whisper.cpp, etc.) have no upload + # cap so ``transcribe_audio`` will never return this error for them. + if not result.get("success") and "File too large" in result.get("error", ""): result = _transcribe_wav_in_chunks(wav_path, model=model, max_file_size=MAX_FILE_SIZE) - else: - result = transcribe_audio(wav_path, model=model) # Filter out Whisper hallucinations (common on silent/near-silent audio) if result.get("success") and is_whisper_hallucination(result.get("transcript", "")): From 027a2ddcb108c1f23e443030d0422b61bdc9535d Mon Sep 17 00:00:00 2001 From: Tushar Date: Sun, 19 Jul 2026 09:55:15 +0530 Subject: [PATCH 05/23] fix(tts): fall through to raw import when lazy_deps fails (#53259) Replace aise ImportError(str(e)) with pass in the except Exception handler of _import_edge_tts(), _import_elevenlabs(), and _import_mistral_client() so packages installed via PYTHONPATH or Docker layered filesystems still work when lazy_deps.ensure() raises. Also fix the Mistral STT path in transcription_tools.py which only caught ImportError, not FeatureUnavailable. Adds 6 regression tests using sys.modules fixtures (no builtins.__import__ patching). --- tests/tools/test_tts_pythonpath_fallback.py | 103 ++++++++++++++++++++ tools/transcription_tools.py | 2 +- tools/tts_tool.py | 12 +-- 3 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 tests/tools/test_tts_pythonpath_fallback.py diff --git a/tests/tools/test_tts_pythonpath_fallback.py b/tests/tools/test_tts_pythonpath_fallback.py new file mode 100644 index 000000000000..23a45fbb5fa0 --- /dev/null +++ b/tests/tools/test_tts_pythonpath_fallback.py @@ -0,0 +1,103 @@ +"""Regression tests for #53259. + +When TTS packages (edge-tts, elevenlabs, mistralai) installed outside the +venv but importable on sys.path (e.g. via PYTHONPATH, Docker layered +filesystems), the lazy-import helpers must fall through to the raw import +instead of re-raising lazy_deps.ensure() failures as ImportError. + +Uses sys.modules fixtures so builtins.__import__ stays intact — patching +__import__ replaces the helper's own ``from tools.lazy_deps import ...`` +and defeats the purpose of the test. +""" + +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from tools.lazy_deps import FeatureUnavailable + + +@pytest.fixture(autouse=True) +def _clean_tts_modules(): + """Remove TTS packages from sys.modules so each test starts fresh.""" + removed = {} + for name in ("edge_tts", "elevenlabs", "elevenlabs.client", + "mistralai", "mistralai.client"): + if name in sys.modules: + removed[name] = sys.modules.pop(name) + yield + for name in ("edge_tts", "elevenlabs", "elevenlabs.client", + "mistralai", "mistralai.client"): + sys.modules.pop(name, None) + sys.modules.update(removed) + + +class TestEdgeTtsPythonpathFallback: + def test_falls_through_on_lazy_deps_failure(self): + """FeatureUnavailable from ensure() must not prevent raw import.""" + mock_edge_tts = MagicMock() + with patch.dict(sys.modules, {"edge_tts": mock_edge_tts}), \ + patch("tools.lazy_deps.ensure", + side_effect=FeatureUnavailable("tts.edge", (), "test")): + from tools.tts_tool import _import_edge_tts + result = _import_edge_tts() + assert result is mock_edge_tts + + def test_raises_when_package_truly_missing(self): + """When the package is truly absent, ImportError must propagate.""" + with patch("tools.lazy_deps.ensure"), \ + patch.dict(sys.modules, {"edge_tts": None}): + from tools.tts_tool import _import_edge_tts + with pytest.raises(ImportError): + _import_edge_tts() + + +class TestElevenLabsPythonpathFallback: + def test_falls_through_on_lazy_deps_failure(self): + """FeatureUnavailable from ensure() must not prevent raw import.""" + mock_cls = MagicMock() + mock_client_pkg = MagicMock() + mock_client_pkg.ElevenLabs = mock_cls + with patch.dict(sys.modules, { + "elevenlabs": mock_client_pkg, + "elevenlabs.client": mock_client_pkg, + }), patch("tools.lazy_deps.ensure", + side_effect=FeatureUnavailable("tts.elevenlabs", (), "test")): + from tools.tts_tool import _import_elevenlabs + result = _import_elevenlabs() + assert result is mock_cls + + def test_raises_when_package_truly_missing(self): + """When the package is truly absent, ImportError must propagate.""" + with patch("tools.lazy_deps.ensure"), \ + patch.dict(sys.modules, {"elevenlabs": None, + "elevenlabs.client": None}): + from tools.tts_tool import _import_elevenlabs + with pytest.raises(ImportError): + _import_elevenlabs() + + +class TestMistralPythonpathFallback: + def test_falls_through_on_lazy_deps_failure(self): + """FeatureUnavailable from ensure() must not prevent raw import.""" + mock_cls = MagicMock() + mock_mistralai = MagicMock() + mock_mistralai.Mistral = mock_cls + with patch.dict(sys.modules, { + "mistralai": mock_mistralai, + "mistralai.client": mock_mistralai, + }), patch("tools.lazy_deps.ensure", + side_effect=FeatureUnavailable("tts.mistral", (), "test")): + from tools.tts_tool import _import_mistral_client + result = _import_mistral_client() + assert result is mock_cls + + def test_raises_when_package_truly_missing(self): + """When the package is truly absent, ImportError must propagate.""" + with patch("tools.lazy_deps.ensure"), \ + patch.dict(sys.modules, {"mistralai": None, + "mistralai.client": None}): + from tools.tts_tool import _import_mistral_client + with pytest.raises(ImportError): + _import_mistral_client() diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index fdb6d72cfedc..4ecd0b41f341 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -1580,7 +1580,7 @@ def _transcribe_mistral(file_path: str, model_name: str) -> Dict[str, Any]: try: from tools.lazy_deps import ensure as _lazy_ensure _lazy_ensure("stt.mistral", prompt=False) - except ImportError: + except Exception: pass from mistralai.client import Mistral diff --git a/tools/tts_tool.py b/tools/tts_tool.py index 374994367869..a40cfd0a5a19 100644 --- a/tools/tts_tool.py +++ b/tools/tts_tool.py @@ -90,8 +90,8 @@ def _import_edge_tts(): _lazy_ensure("tts.edge", prompt=False) except ImportError: pass - except Exception as e: - raise ImportError(str(e)) + except Exception: + pass import edge_tts return edge_tts @@ -111,8 +111,8 @@ def _import_elevenlabs(): # lazy_deps module itself missing — fall through to the raw import # so older code paths still get a clean ImportError. pass - except Exception as e: # FeatureUnavailable or any unexpected error - raise ImportError(str(e)) + except Exception: + pass from elevenlabs.client import ElevenLabs return ElevenLabs @@ -134,8 +134,8 @@ def _import_mistral_client(): ensure("tts.mistral", prompt=False) except ImportError: pass - except Exception as e: # FeatureUnavailable or any unexpected error - raise ImportError(str(e)) + except Exception: + pass from mistralai.client import Mistral return Mistral From 99d66d699b5f727976c4bc5860bc3c52486fa1ae Mon Sep 17 00:00:00 2001 From: Tushar Date: Sun, 19 Jul 2026 10:21:10 +0530 Subject: [PATCH 06/23] test(tts): add STT fallback regression for _transcribe_mistral (#53259) Add isolated test where ensure('stt.mistral') raises FeatureUnavailable but the raw mistralai.client.Mistral import succeeds, verifying the transcription_tools.py fallthrough path introduced in the same PR. --- tests/tools/test_tts_pythonpath_fallback.py | 48 ++++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/tests/tools/test_tts_pythonpath_fallback.py b/tests/tools/test_tts_pythonpath_fallback.py index 23a45fbb5fa0..39f873cf07bb 100644 --- a/tests/tools/test_tts_pythonpath_fallback.py +++ b/tests/tools/test_tts_pythonpath_fallback.py @@ -1,7 +1,7 @@ """Regression tests for #53259. -When TTS packages (edge-tts, elevenlabs, mistralai) installed outside the -venv but importable on sys.path (e.g. via PYTHONPATH, Docker layered +When TTS/STT packages (edge-tts, elevenlabs, mistralai) installed outside +the venv but importable on sys.path (e.g. via PYTHONPATH, Docker layered filesystems), the lazy-import helpers must fall through to the raw import instead of re-raising lazy_deps.ensure() failures as ImportError. @@ -101,3 +101,47 @@ def test_raises_when_package_truly_missing(self): from tools.tts_tool import _import_mistral_client with pytest.raises(ImportError): _import_mistral_client() + + +# ── STT: _transcribe_mistral fallthrough ─────────────────────────────────── + + +class TestMistralSttPythonpathFallback: + def test_transcribe_mistral_falls_through_on_lazy_deps_failure( + self, tmp_path, + ): + """FeatureUnavailable from ensure('stt.mistral') must not block + transcription when mistralai is importable via PYTHONPATH.""" + from tools.transcription_tools import _transcribe_mistral + + audio_file = tmp_path / "audio.wav" + audio_file.write_bytes(b"fake-audio") + + mock_client_cls = MagicMock() + mock_result = MagicMock() + mock_result.text = "hello world" + mock_client_cls.return_value.__enter__ = MagicMock( + return_value=MagicMock( + audio=MagicMock( + transcriptions=MagicMock( + complete=MagicMock(return_value=mock_result), + ), + ), + ), + ) + mock_client_cls.return_value.__exit__ = MagicMock(return_value=False) + + mock_mistralai = MagicMock() + mock_mistralai.Mistral = mock_client_cls + + with patch.dict(sys.modules, { + "mistralai": mock_mistralai, + "mistralai.client": mock_mistralai, + }), patch("tools.lazy_deps.ensure", + side_effect=FeatureUnavailable("stt.mistral", (), "test")), \ + patch("tools.transcription_tools.get_env_value", + return_value="test-key"): + result = _transcribe_mistral(str(audio_file), "mistral-large-latest") + + assert result["success"] is True + assert result["transcript"] == "hello world" From 4ebad50cbe43f59977ab788c27f62acfeb93dd33 Mon Sep 17 00:00:00 2001 From: Richard Jang Date: Mon, 13 Apr 2026 23:27:17 +0900 Subject: [PATCH 07/23] fix: handle missing transcription module gracefully --- gateway/run.py | 12 +++++++++- tests/gateway/test_stt_config.py | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/gateway/run.py b/gateway/run.py index eb209efbf99b..71473f34d329 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -17971,7 +17971,17 @@ async def _enrich_message_with_transcription( return f"{prefix}\n\n{user_text}", [] return prefix, [] - from tools.transcription_tools import transcribe_audio + try: + from tools.transcription_tools import transcribe_audio + except ModuleNotFoundError as e: + logger.error("Transcription module unavailable: %s", e) + unavailable_note = "[voice message could not be transcribed]" + _placeholder = "(The user sent a message with no text content)" + if user_text and user_text.strip() == _placeholder: + return unavailable_note, [] + if user_text: + return f"{unavailable_note}\n\n{user_text}", [] + return unavailable_note, [] enriched_parts = [] successful_transcripts: List[str] = [] diff --git a/tests/gateway/test_stt_config.py b/tests/gateway/test_stt_config.py index 5006eafee319..3a0b54ea69a6 100644 --- a/tests/gateway/test_stt_config.py +++ b/tests/gateway/test_stt_config.py @@ -143,6 +143,44 @@ async def test_enrich_message_with_transcription_returns_tuple_for_empty_content assert transcripts == ["hello from a captionless voice note"] +@pytest.mark.parametrize( + ("user_text", "expected_text"), + [ + ("caption", "[voice message could not be transcribed]\n\ncaption"), + ("", "[voice message could not be transcribed]"), + ( + "(The user sent a message with no text content)", + "[voice message could not be transcribed]", + ), + ], +) +@pytest.mark.asyncio +async def test_enrich_message_with_transcription_handles_missing_transcription_module_gracefully( + user_text, + expected_text, +): + from gateway.run import GatewayRunner + + runner = GatewayRunner.__new__(GatewayRunner) + runner.config = GatewayConfig(stt_enabled=True) + + real_import = __import__ + + def fake_import(name, globals=None, locals=None, fromlist=(), level=0): + if name == "tools.transcription_tools": + raise ModuleNotFoundError("No module named 'tools.transcription_tools'") + return real_import(name, globals, locals, fromlist, level) + + with patch("builtins.__import__", side_effect=fake_import): + result, transcripts = await runner._enrich_message_with_transcription( + user_text, + ["/tmp/voice.ogg"], + ) + + assert result == expected_text + assert transcripts == [] + + @pytest.mark.asyncio async def test_prepare_inbound_message_text_transcribes_queued_voice_event(): from gateway.run import GatewayRunner From 3162cb597f85ca0473eb7cefe76ed4eade2f7658 Mon Sep 17 00:00:00 2001 From: luyifan Date: Tue, 30 Jun 2026 04:39:32 +0800 Subject: [PATCH 08/23] fix(stt): report unregistered configured providers --- .../test_transcription_plugin_dispatch.py | 28 +++++++++++--- tools/transcription_tools.py | 37 ++++++++++++++++--- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/tests/tools/test_transcription_plugin_dispatch.py b/tests/tools/test_transcription_plugin_dispatch.py index d7ea8dac4ee3..7bfd311de1a8 100644 --- a/tests/tools/test_transcription_plugin_dispatch.py +++ b/tests/tools/test_transcription_plugin_dispatch.py @@ -8,7 +8,7 @@ built-in name (which the registry blocks), the dispatcher re-checks defensively. 2. Unknown name with no plugin → returns None (caller surfaces the - legacy "No STT provider available" error). + "provider_not_registered" error). 3. Unknown name with plugin registered → dispatches, returns result. 4. Plugin exceptions are caught and converted to the standard error envelope. @@ -235,10 +235,8 @@ def test_unknown_name_with_plugin_dispatches(self, sample_audio_file): assert result["transcript"] == "fake transcript" assert result["provider"] == "openrouter" - def test_unknown_name_without_plugin_falls_to_legacy_error(self, sample_audio_file): - """When no plugin is registered for the unknown name, the - dispatcher returns None and transcribe_audio falls through to - the legacy 'No STT provider available' error message.""" + def test_unknown_name_without_plugin_returns_provider_specific_error(self, sample_audio_file): + """Explicit unknown providers should get a named registration error.""" from unittest.mock import patch with patch("tools.transcription_tools._load_stt_config", return_value={"provider": "openrouter"}), \ @@ -247,7 +245,25 @@ def test_unknown_name_without_plugin_falls_to_legacy_error(self, sample_audio_fi result = transcription_tools.transcribe_audio(sample_audio_file) assert result["success"] is False - assert "No STT provider" in result["error"] + assert result["provider"] == "openrouter" + assert result["error_type"] == "provider_not_registered" + assert "stt.provider='openrouter'" in result["error"] + assert "hermes plugins list" in result["error"] + assert "No STT provider available" not in result["error"] + + def test_auto_detect_failure_keeps_legacy_no_provider_message(self): + """No explicit stt.provider remains the generic setup guidance path.""" + from unittest.mock import patch + + with patch("tools.transcription_tools._validate_audio_file", return_value=None), \ + patch("tools.transcription_tools._load_stt_config", return_value={}), \ + patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ + patch("tools.transcription_tools._get_provider", return_value="none"): + result = transcription_tools.transcribe_audio("/tmp/audio.mp3") + + assert result["success"] is False + assert result.get("error_type") is None + assert "No STT provider available" in result["error"] def test_builtin_name_does_not_consult_plugin_registry(self, sample_audio_file): """Even if a plugin's name collides with a built-in (which the diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index 4ecd0b41f341..9d02fa5c5e11 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -933,6 +933,22 @@ def _get_provider(stt_config: dict) -> str: return "none" +def _unregistered_stt_provider_error(provider: str) -> Dict[str, Any]: + key = str(provider or "").strip() + return { + "success": False, + "transcript": "", + "provider": key, + "error_type": "provider_not_registered", + "error": ( + f"stt.provider='{key}' is set but no built-in, command, or plugin " + "provider registered that name. Run `hermes plugins list` to see " + "installed STT plugins, or configure a command provider under " + f"`stt.providers.{key}.command`." + ), + } + + # --------------------------------------------------------------------------- # Plugin provider dispatch (issue follow-up to #30398 — STT pluggability) # --------------------------------------------------------------------------- @@ -949,8 +965,8 @@ def _dispatch_to_plugin_provider( """Route the call to a plugin-registered transcription provider, or return None. - Returns the transcribe-response dict on dispatch, or ``None`` to - fall through to the legacy "No STT provider available" error path. + Returns the transcribe-response dict on dispatch, or ``None`` when no + plugin claimed the provider name. Resolution invariants enforced here: @@ -968,8 +984,8 @@ def _dispatch_to_plugin_provider( 3. Plugin dispatch fires only when ``provider`` matches a registered :class:`TranscriptionProvider` whose ``name`` equals the configured value. Unknown names with no plugin registered - return None (caller surfaces the legacy "No STT provider" - message). + return None (caller surfaces the configured-provider error when + the name came from ``stt.provider``). 4. Availability gating: when the matched plugin reports ``is_available() == False`` (missing API key, missing optional SDK, etc.) this returns an error envelope identifying the @@ -1967,8 +1983,8 @@ def transcribe_audio(file_path: str, model: Optional[str] = None) -> Dict[str, A # nor ``"none"`` AND there is no same-name command provider. The # dispatcher enforces built-ins-always-win + command-wins-over-plugin # defensively. Returns None when no plugin is registered for the - # configured name, falling through to the legacy "No STT provider" - # error message below. + # configured name; explicit configured names get a provider-specific + # error before the generic auto-detect fallback below. # # Plugin-scoped config namespace mirrors the built-in pattern # (``stt.openai.model``, ``stt.mistral.model``): plugins read their @@ -1988,6 +2004,15 @@ def transcribe_audio(file_path: str, model: Optional[str] = None) -> Dict[str, A if plugin_result is not None: return plugin_result + provider_key = str(provider or "").strip().lower() + if ( + "provider" in stt_config + and provider_key + and provider_key not in BUILTIN_STT_PROVIDERS + and provider_key != "none" + ): + return _unregistered_stt_provider_error(provider_key) + # No provider available return { "success": False, From bc036cba0d8414a2767012f1ae9184481589cb2e Mon Sep 17 00:00:00 2001 From: LauraGPT <18321252+LauraGPT@users.noreply.github.com> Date: Thu, 16 Jul 2026 09:13:01 +0000 Subject: [PATCH 09/23] fix(stt): strip Qwen3-ASR response prefix Normalize the structured marker after extracting text from string, SDK object, and dictionary transcription responses. Preserve the current provider-aware STT configuration architecture. Refreshes #8773 on current main. Co-authored-by: angelos Assisted-by: Codex:gpt-5.6 --- tests/tools/test_managed_media_gateways.py | 27 ++++++++++++++++++++++ tools/transcription_tools.py | 21 ++++++++++++----- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/tests/tools/test_managed_media_gateways.py b/tests/tools/test_managed_media_gateways.py index 1b248ce09bf9..d7d35e4b7b69 100644 --- a/tests/tools/test_managed_media_gateways.py +++ b/tests/tools/test_managed_media_gateways.py @@ -347,6 +347,33 @@ def test_transcription_uses_model_specific_response_formats(monkeypatch, tmp_pat assert json_capture["close_calls"] == 1 +@pytest.mark.parametrize( + ("transcription", "expected"), + [ + ("language EnglishHello from Qwen.", "Hello from Qwen."), + ( + types.SimpleNamespace(text="language ChineseObject response."), + "Object response.", + ), + ( + {"text": "language EnglishDictionary response."}, + "Dictionary response.", + ), + ], +) +def test_extract_transcript_text_strips_qwen3_asr_prefix( + transcription, + expected, +): + _install_fake_tools_package() + transcription_tools = _load_tool_module( + "tools.transcription_tools", + "transcription_tools.py", + ) + + assert transcription_tools._extract_transcript_text(transcription) == expected + + PLUGINS_DIR = Path(__file__).resolve().parents[2] / "plugins" diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index 9d02fa5c5e11..2f336a43f313 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -2060,17 +2060,26 @@ def _resolve_openai_audio_client_config() -> tuple[str, str]: def _extract_transcript_text(transcription: Any) -> str: """Normalize text and JSON transcription responses to a plain string.""" + text: Optional[str] = None + if isinstance(transcription, str): - return transcription.strip() + text = transcription.strip() - if hasattr(transcription, "text"): + if text is None and hasattr(transcription, "text"): value = getattr(transcription, "text") if isinstance(value, str): - return value.strip() + text = value.strip() - if isinstance(transcription, dict): + if text is None and isinstance(transcription, dict): value = transcription.get("text") if isinstance(value, str): - return value.strip() + text = value.strip() + + if text is None: + text = str(transcription).strip() + + marker = "" + if marker in text: + text = text.split(marker, 1)[1].strip() - return str(transcription).strip() + return text From 113b67f9c3f7dff77bc551f5ba032edf4c61a0ac Mon Sep 17 00:00:00 2001 From: LauraGPT <18321252+LauraGPT@users.noreply.github.com> Date: Sat, 18 Jul 2026 21:33:07 +0000 Subject: [PATCH 10/23] fix(stt): anchor Qwen3-ASR envelope stripping --- tests/tools/test_transcription_tools.py | 24 ++++++++++++++++++++++++ tools/transcription_tools.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index 1584d9cbaba2..8338a4838168 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -1840,6 +1840,30 @@ def test_model_override_passed_to_elevenlabs(self, sample_ogg): assert mock_elevenlabs.call_args[0][1] == "scribe_v2" +# ============================================================================ +# _extract_transcript_text +# ============================================================================ + +class TestExtractTranscriptText: + def test_strips_qwen3_asr_language_envelope(self): + from tools.transcription_tools import _extract_transcript_text + + result = _extract_transcript_text( + "language zh\nzh\n你好，世界", + ) + + assert result == "你好，世界" + + def test_keeps_non_envelope_marker_literal(self): + from tools.transcription_tools import _extract_transcript_text + + result = _extract_transcript_text( + "The user literally said while reading markup.", + ) + + assert result == "The user literally said while reading markup." + + # Shell safety — shlex.split on auto-detected templates # ============================================================================ class TestShellSafety: diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index 2f336a43f313..e60d4e5ac476 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -2079,7 +2079,7 @@ def _extract_transcript_text(transcription: Any) -> str: text = str(transcription).strip() marker = "" - if marker in text: + if text.lstrip().lower().startswith("language") and marker in text: text = text.split(marker, 1)[1].strip() return text From fbfe93d6d7e9c6a473c55cf8e5640a6157337e23 Mon Sep 17 00:00:00 2001 From: LauraGPT <18321252+LauraGPT@users.noreply.github.com> Date: Sun, 19 Jul 2026 05:48:41 +0000 Subject: [PATCH 11/23] fix(stt): anchor qwen asr envelope stripping --- tests/tools/test_transcription_tools.py | 9 +++++++++ tools/transcription_tools.py | 11 ++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index 8338a4838168..9543fe7765f4 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -1863,6 +1863,15 @@ def test_keeps_non_envelope_marker_literal(self): assert result == "The user literally said while reading markup." + def test_keeps_language_sentence_with_marker_literal(self): + from tools.transcription_tools import _extract_transcript_text + + result = _extract_transcript_text( + "Language teachers may say when discussing markup.", + ) + + assert result == "Language teachers may say when discussing markup." + # Shell safety — shlex.split on auto-detected templates # ============================================================================ diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index e60d4e5ac476..8483a82d298a 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -30,6 +30,7 @@ import logging import os import platform +import re import shlex import shutil import subprocess @@ -2078,8 +2079,12 @@ def _extract_transcript_text(transcription: Any) -> str: if text is None: text = str(transcription).strip() - marker = "" - if text.lstrip().lower().startswith("language") and marker in text: - text = text.split(marker, 1)[1].strip() + match = re.match( + r"\s*language\s+[\w.-]+(?:\s*[^<]*)?\s*\s*(?P.*)", + text, + flags=re.IGNORECASE | re.DOTALL, + ) + if match: + text = match.group("text").strip() return text From 1a1445a1384cba2abf6289c106f854ef33bfe59f Mon Sep 17 00:00:00 2001 From: Zehua Wang Date: Sun, 7 Jun 2026 21:15:50 -0400 Subject: [PATCH 12/23] fix(stt): check_voice_requirements() should recognize all STT providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /voice status command only checked for 'local', 'groq', and 'openai' providers. Any other valid provider (local_command, mistral, xai, elevenlabs, or custom command providers) fell through to the generic MISSING message — even when transcription worked perfectly. - Import _has_any_command_stt_provider (already defined, never imported) - Add elif branches for local_command, mistral, xai, elevenlabs - Add generic catch-all via _has_any_command_stt_provider() for arbitrary custom command providers --- tools/voice_mode.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tools/voice_mode.py b/tools/voice_mode.py index 6f8b22bd520f..822eb6f35cda 100644 --- a/tools/voice_mode.py +++ b/tools/voice_mode.py @@ -1277,7 +1277,7 @@ def check_voice_requirements() -> Dict[str, Any]: ``missing_packages``, and ``details``. """ # Determine STT provider availability - from tools.transcription_tools import _get_provider, _load_stt_config, is_stt_enabled + from tools.transcription_tools import _get_provider, _has_any_command_stt_provider, _load_stt_config, is_stt_enabled stt_config = _load_stt_config() stt_enabled = is_stt_enabled(stt_config) stt_provider = _get_provider(stt_config) @@ -1307,10 +1307,20 @@ def check_voice_requirements() -> Dict[str, Any]: details_parts.append("STT provider: DISABLED in config (stt.enabled: false)") elif stt_provider == "local": details_parts.append("STT provider: OK (local faster-whisper)") + elif stt_provider == "local_command": + details_parts.append("STT provider: OK (local command)") elif stt_provider == "groq": details_parts.append("STT provider: OK (Groq)") elif stt_provider == "openai": details_parts.append("STT provider: OK (OpenAI)") + elif stt_provider == "mistral": + details_parts.append("STT provider: OK (Mistral Voxtral)") + elif stt_provider == "xai": + details_parts.append("STT provider: OK (xAI Grok STT)") + elif stt_provider == "elevenlabs": + details_parts.append("STT provider: OK (ElevenLabs Scribe)") + elif _has_any_command_stt_provider(stt_config): + details_parts.append(f"STT provider: OK ({stt_provider})") else: details_parts.append( "STT provider: MISSING (uv pip install faster-whisper — " From ad383960dcd0d0e2675decd59be16d549d0d3548 Mon Sep 17 00:00:00 2001 From: Zehua Wang Date: Tue, 14 Jul 2026 12:28:53 -0400 Subject: [PATCH 13/23] fix(stt): check selected provider (not any) + plugin support PR review feedback: - Replace _has_any_command_stt_provider() with selected-provider check via _resolve_command_stt_provider_config() - Add _check_plugin_stt_provider() for plugin-registered backends - Add tests: selected command, unrelated command (should NOT pass), and plugin provider path --- tests/tools/test_voice_mode.py | 50 ++++++++++++++++++++++++++++++++++ tools/voice_mode.py | 19 +++++++++++-- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/tests/tools/test_voice_mode.py b/tests/tools/test_voice_mode.py index 87ccdb297057..4b86e2d74064 100644 --- a/tests/tools/test_voice_mode.py +++ b/tests/tools/test_voice_mode.py @@ -500,6 +500,56 @@ def test_missing_stt_provider(self, monkeypatch): assert result["stt_available"] is False assert "STT provider: MISSING" in result["details"] + def test_command_stt_provider_selected(self, monkeypatch): + """Catch-all branch fires for a selected command provider (not any provider).""" + monkeypatch.setattr("tools.voice_mode._audio_available", lambda: True) + monkeypatch.setattr("tools.voice_mode.detect_audio_environment", + lambda: {"available": True, "warnings": []}) + monkeypatch.setattr("tools.transcription_tools._get_provider", lambda cfg: "my-custom-stt") + monkeypatch.setattr("tools.transcription_tools._resolve_command_stt_provider_config", + lambda p, c: {"command": "whisper_cpp"} if p == "my-custom-stt" else None) + + from tools.voice_mode import check_voice_requirements + + result = check_voice_requirements() + assert result["available"] is True + assert result["stt_available"] is True + assert "STT provider: OK (command: my-custom-stt)" in result["details"] + + def test_unrelated_command_provider_not_confused(self, monkeypatch): + """Unrelated command provider does NOT make a different selected provider appear OK.""" + monkeypatch.setattr("tools.voice_mode._audio_available", lambda: True) + monkeypatch.setattr("tools.voice_mode.detect_audio_environment", + lambda: {"available": True, "warnings": []}) + monkeypatch.setattr("tools.transcription_tools._get_provider", lambda cfg: "none") + monkeypatch.setattr("tools.transcription_tools._resolve_command_stt_provider_config", + lambda p, c: {"command": "whisper_cpp"} if p == "my-custom-stt" else None) + + from tools.voice_mode import check_voice_requirements + + result = check_voice_requirements() + assert result["available"] is False + assert result["stt_available"] is False + assert "STT provider: MISSING" in result["details"] + + def test_plugin_stt_provider(self, monkeypatch): + """Plugin STT provider is recognized.""" + monkeypatch.setattr("tools.voice_mode._audio_available", lambda: True) + monkeypatch.setattr("tools.voice_mode.detect_audio_environment", + lambda: {"available": True, "warnings": []}) + monkeypatch.setattr("tools.transcription_tools._get_provider", lambda cfg: "my-plugin-stt") + monkeypatch.setattr("tools.transcription_tools._resolve_command_stt_provider_config", + lambda p, c: None) + monkeypatch.setattr("tools.voice_mode._check_plugin_stt_provider", + lambda p: p == "my-plugin-stt") + + from tools.voice_mode import check_voice_requirements + + result = check_voice_requirements() + assert result["available"] is True + assert result["stt_available"] is True + assert "STT provider: OK (plugin: my-plugin-stt)" in result["details"] + # ============================================================================ # AudioRecorder diff --git a/tools/voice_mode.py b/tools/voice_mode.py index 822eb6f35cda..724aaa329a18 100644 --- a/tools/voice_mode.py +++ b/tools/voice_mode.py @@ -1269,6 +1269,17 @@ def listen_for_speech( # ============================================================================ # Requirements check # ============================================================================ +def _check_plugin_stt_provider(provider: str) -> bool: + """Return True when *provider* is backed by a registered TranscriptionProvider plugin.""" + if not provider: + return False + try: + from agent.transcription_registry import get_provider + return get_provider(provider.lower().strip()) is not None + except ImportError: + return False + + def check_voice_requirements() -> Dict[str, Any]: """Check if all voice mode requirements are met. @@ -1277,7 +1288,7 @@ def check_voice_requirements() -> Dict[str, Any]: ``missing_packages``, and ``details``. """ # Determine STT provider availability - from tools.transcription_tools import _get_provider, _has_any_command_stt_provider, _load_stt_config, is_stt_enabled + from tools.transcription_tools import _get_provider, _load_stt_config, _resolve_command_stt_provider_config, is_stt_enabled stt_config = _load_stt_config() stt_enabled = is_stt_enabled(stt_config) stt_provider = _get_provider(stt_config) @@ -1319,8 +1330,10 @@ def check_voice_requirements() -> Dict[str, Any]: details_parts.append("STT provider: OK (xAI Grok STT)") elif stt_provider == "elevenlabs": details_parts.append("STT provider: OK (ElevenLabs Scribe)") - elif _has_any_command_stt_provider(stt_config): - details_parts.append(f"STT provider: OK ({stt_provider})") + elif _resolve_command_stt_provider_config(stt_provider, stt_config): + details_parts.append(f"STT provider: OK (command: {stt_provider})") + elif _check_plugin_stt_provider(stt_provider): + details_parts.append(f"STT provider: OK (plugin: {stt_provider})") else: details_parts.append( "STT provider: MISSING (uv pip install faster-whisper — " From ba8141966c45cc1d821230802ae4dc4f7f90bfc2 Mon Sep 17 00:00:00 2001 From: Zehua Wang Date: Tue, 14 Jul 2026 20:31:42 -0400 Subject: [PATCH 14/23] fix(stt): validate selected voice provider availability --- tests/tools/test_voice_mode.py | 86 +++++++++++++++++++++++++++++----- tools/voice_mode.py | 69 ++++++++++++++++++++++++--- 2 files changed, 136 insertions(+), 19 deletions(-) diff --git a/tests/tools/test_voice_mode.py b/tests/tools/test_voice_mode.py index 4b86e2d74064..2a4024fb9a7d 100644 --- a/tests/tools/test_voice_mode.py +++ b/tests/tools/test_voice_mode.py @@ -505,10 +505,19 @@ def test_command_stt_provider_selected(self, monkeypatch): monkeypatch.setattr("tools.voice_mode._audio_available", lambda: True) monkeypatch.setattr("tools.voice_mode.detect_audio_environment", lambda: {"available": True, "warnings": []}) - monkeypatch.setattr("tools.transcription_tools._get_provider", lambda cfg: "my-custom-stt") - monkeypatch.setattr("tools.transcription_tools._resolve_command_stt_provider_config", - lambda p, c: {"command": "whisper_cpp"} if p == "my-custom-stt" else None) - + monkeypatch.setattr( + "tools.transcription_tools._load_stt_config", + lambda: { + "enabled": True, + "provider": "my-custom-stt", + "providers": { + "my-custom-stt": { + "type": "command", + "command": "whisper_cpp {input}", + }, + }, + }, + ) from tools.voice_mode import check_voice_requirements result = check_voice_requirements() @@ -521,9 +530,26 @@ def test_unrelated_command_provider_not_confused(self, monkeypatch): monkeypatch.setattr("tools.voice_mode._audio_available", lambda: True) monkeypatch.setattr("tools.voice_mode.detect_audio_environment", lambda: {"available": True, "warnings": []}) - monkeypatch.setattr("tools.transcription_tools._get_provider", lambda cfg: "none") - monkeypatch.setattr("tools.transcription_tools._resolve_command_stt_provider_config", - lambda p, c: {"command": "whisper_cpp"} if p == "my-custom-stt" else None) + monkeypatch.setattr( + "tools.transcription_tools._load_stt_config", + lambda: { + "enabled": True, + "provider": "unknown-selected", + "providers": { + "unrelated-command": { + "type": "command", + "command": "whisper_cpp {input}", + }, + }, + }, + ) + monkeypatch.setattr( + "agent.transcription_registry.get_provider", lambda p: None, + ) + monkeypatch.setattr( + "hermes_cli.plugins._ensure_plugins_discovered", + lambda force=False: None, + ) from tools.voice_mode import check_voice_requirements @@ -537,11 +563,20 @@ def test_plugin_stt_provider(self, monkeypatch): monkeypatch.setattr("tools.voice_mode._audio_available", lambda: True) monkeypatch.setattr("tools.voice_mode.detect_audio_environment", lambda: {"available": True, "warnings": []}) - monkeypatch.setattr("tools.transcription_tools._get_provider", lambda cfg: "my-plugin-stt") - monkeypatch.setattr("tools.transcription_tools._resolve_command_stt_provider_config", - lambda p, c: None) - monkeypatch.setattr("tools.voice_mode._check_plugin_stt_provider", - lambda p: p == "my-plugin-stt") + monkeypatch.setattr( + "tools.transcription_tools._load_stt_config", + lambda: {"enabled": True, "provider": "my-plugin-stt"}, + ) + plugin_provider = MagicMock() + plugin_provider.is_available.return_value = True + monkeypatch.setattr( + "agent.transcription_registry.get_provider", + lambda p: plugin_provider if p == "my-plugin-stt" else None, + ) + monkeypatch.setattr( + "hermes_cli.plugins._ensure_plugins_discovered", + lambda force=False: None, + ) from tools.voice_mode import check_voice_requirements @@ -550,6 +585,33 @@ def test_plugin_stt_provider(self, monkeypatch): assert result["stt_available"] is True assert "STT provider: OK (plugin: my-plugin-stt)" in result["details"] + def test_unavailable_plugin_stt_provider(self, monkeypatch): + """A registered but unavailable plugin does not satisfy requirements.""" + monkeypatch.setattr("tools.voice_mode._audio_available", lambda: True) + monkeypatch.setattr("tools.voice_mode.detect_audio_environment", + lambda: {"available": True, "warnings": []}) + monkeypatch.setattr( + "tools.transcription_tools._load_stt_config", + lambda: {"enabled": True, "provider": "my-plugin-stt"}, + ) + plugin_provider = MagicMock() + plugin_provider.is_available.return_value = False + monkeypatch.setattr( + "agent.transcription_registry.get_provider", + lambda p: plugin_provider if p == "my-plugin-stt" else None, + ) + monkeypatch.setattr( + "hermes_cli.plugins._ensure_plugins_discovered", + lambda force=False: None, + ) + + from tools.voice_mode import check_voice_requirements + + result = check_voice_requirements() + assert result["available"] is False + assert result["stt_available"] is False + assert "STT provider: MISSING" in result["details"] + # ============================================================================ # AudioRecorder diff --git a/tools/voice_mode.py b/tools/voice_mode.py index 724aaa329a18..74f688875a5a 100644 --- a/tools/voice_mode.py +++ b/tools/voice_mode.py @@ -1270,13 +1270,42 @@ def listen_for_speech( # Requirements check # ============================================================================ def _check_plugin_stt_provider(provider: str) -> bool: - """Return True when *provider* is backed by a registered TranscriptionProvider plugin.""" + """Return True when *provider* resolves to an available STT plugin.""" if not provider: return False + key = provider.lower().strip() + if key == "none": + return False try: from agent.transcription_registry import get_provider - return get_provider(provider.lower().strip()) is not None - except ImportError: + from hermes_cli.plugins import _ensure_plugins_discovered + + _ensure_plugins_discovered() + plugin_provider = get_provider(key) + if plugin_provider is None: + # Match the transcription dispatcher: long-lived processes may + # need one refresh after plugins or configuration change. + _ensure_plugins_discovered(force=True) + plugin_provider = get_provider(key) + except Exception as exc: # noqa: BLE001 - discovery failure is non-fatal + logger.debug( + "STT plugin requirements check skipped for '%s': %s", key, exc, + ) + return False + + if plugin_provider is None: + return False + + try: + return bool(plugin_provider.is_available()) + except Exception as exc: # noqa: BLE001 - plugins must not break status + logger.warning( + "STT plugin provider '%s' is_available() raised during requirements " + "check: %s - treating as unavailable", + key, + exc, + exc_info=True, + ) return False @@ -1288,11 +1317,37 @@ def check_voice_requirements() -> Dict[str, Any]: ``missing_packages``, and ``details``. """ # Determine STT provider availability - from tools.transcription_tools import _get_provider, _load_stt_config, _resolve_command_stt_provider_config, is_stt_enabled + from tools.transcription_tools import ( + _get_provider, + _load_stt_config, + _resolve_command_stt_provider_config, + is_stt_enabled, + ) stt_config = _load_stt_config() stt_enabled = is_stt_enabled(stt_config) stt_provider = _get_provider(stt_config) - stt_available = stt_enabled and stt_provider != "none" + native_stt_available = stt_provider in { + "local", + "local_command", + "groq", + "openai", + "mistral", + "xai", + "elevenlabs", + } + command_stt_config = None + plugin_stt_available = False + if stt_enabled and not native_stt_available: + command_stt_config = _resolve_command_stt_provider_config( + stt_provider, stt_config, + ) + if command_stt_config is None: + plugin_stt_available = _check_plugin_stt_provider(stt_provider) + stt_available = stt_enabled and ( + native_stt_available + or command_stt_config is not None + or plugin_stt_available + ) missing: List[str] = [] termux_capture = _termux_voice_capture_available() @@ -1330,9 +1385,9 @@ def check_voice_requirements() -> Dict[str, Any]: details_parts.append("STT provider: OK (xAI Grok STT)") elif stt_provider == "elevenlabs": details_parts.append("STT provider: OK (ElevenLabs Scribe)") - elif _resolve_command_stt_provider_config(stt_provider, stt_config): + elif command_stt_config is not None: details_parts.append(f"STT provider: OK (command: {stt_provider})") - elif _check_plugin_stt_provider(stt_provider): + elif plugin_stt_available: details_parts.append(f"STT provider: OK (plugin: {stt_provider})") else: details_parts.append( From b2975f5cf807de117a3082d0c68a37da90a2f170 Mon Sep 17 00:00:00 2001 From: Damian Kluk Date: Sun, 14 Jun 2026 12:12:39 +0000 Subject: [PATCH 15/23] fix(stt): better error logging when faster-whisper lazy install fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Log lazy-install failures at WARNING instead of DEBUG, with actionable guidance about venv write-permission issues (the most common cause of silent STT failures). Salvaged from PR #46127 (transcription_tools half only — the gateway DM hunks are superseded by main's neutral-marker enrichment design, and the Docker/CI files were unrelated scope). (cherry picked from commit d3e07bdaaa, reduced) --- tools/transcription_tools.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index 8483a82d298a..c1735a80fac2 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -259,8 +259,20 @@ def _try_lazy_install_stt() -> bool: import importlib.util as _iu if _iu.find_spec("faster_whisper"): return True + logger.warning( + "faster-whisper was installed but importlib still cannot find it " + "(may require Python restart)" + ) except Exception as exc: - logger.debug("Lazy install of faster-whisper failed: %s", exc) + logger.warning( + "Lazy install of faster-whisper failed: %s. " + "This is often a permission issue: the Hermes process user cannot " + "write to the virtual environment. Try running manually as the " + "venv owner: `stat -c '%%u' '$(dirname $(dirname $(which python3)))'` " + "then `su - -c 'VIRTUAL_ENV=/opt/hermes/.venv " + "uv pip install faster-whisper==1.2.1'`", + exc, + ) return False From 617f770425575dbf09e1434e35462002143e0de7 Mon Sep 17 00:00:00 2001 From: Dennis <275702+dso2ng@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:30:17 -0700 Subject: [PATCH 16/23] fix(stt): preprocess .silk voice notes before transcription Decode WeChat/QQ SILK v3 voice notes to WAV inside transcribe_audio so any platform that caches a .silk file gets STT for free (same central- normalization philosophy as the outbound container repair). pilk is lazy-installed on first use (stt.silk in tools/lazy_deps.py) instead of being added to the voice extra. Fixes the inbound half of #32196. (cherry picked from commit e5db79369d; reworked to compose with the provider-scoped upload size cap and to lazy-dep pilk) --- tests/tools/test_transcription_tools.py | 66 ++++++++++++++ tools/lazy_deps.py | 3 + tools/transcription_tools.py | 109 +++++++++++++++++++++--- 3 files changed, 168 insertions(+), 10 deletions(-) diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index 9543fe7765f4..bc117b76e465 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -53,6 +53,14 @@ def sample_ogg(tmp_path): ogg_path.write_bytes(b"fake audio data") return str(ogg_path) +@pytest.fixture +def sample_silk(tmp_path): + """Create a fake WeChat .silk file for preprocessing tests.""" + silk_path = tmp_path / "voice.silk" + silk_path.write_bytes(b"\x02#!SILK_V3fake") + return str(silk_path) + + @pytest.fixture def oversized_wav(tmp_path): @@ -1208,6 +1216,64 @@ def test_model_override_passed_to_local(self, sample_ogg): assert mock_local.call_args[0][1] == "large-v3" + def test_converts_silk_before_dispatch(self, sample_silk): + with patch("tools.transcription_tools._prepare_audio_for_transcription", + return_value=("/tmp/converted.wav", "/tmp/hermes-silk-123", None), + create=True) as mock_prepare, \ + patch("tools.transcription_tools._validate_audio_file", return_value=None), \ + patch("tools.transcription_tools._load_stt_config", return_value={}), \ + patch("tools.transcription_tools._get_provider", return_value="local"), \ + patch("tools.transcription_tools._transcribe_local", + return_value={"success": True, "transcript": "hi"}) as mock_local, \ + patch("tools.transcription_tools.shutil.rmtree") as mock_rmtree: + from tools.transcription_tools import transcribe_audio + result = transcribe_audio(sample_silk) + + assert result["success"] is True + mock_prepare.assert_called_once_with(sample_silk) + mock_local.assert_called_once_with("/tmp/converted.wav", "base") + mock_rmtree.assert_called_once_with("/tmp/hermes-silk-123", ignore_errors=True) + + def test_silk_symlink_is_rejected_before_preprocessing(self, tmp_path): + """A Silk symlink must not reach the decoder before path safety validation.""" + if not hasattr(os, "symlink"): + pytest.skip("symlinks are not supported on this platform") + + target = tmp_path / "voice.silk" + target.write_bytes(b"\x02#!SILK_V3fake") + link = tmp_path / "linked.silk" + try: + os.symlink(target, link) + except (OSError, NotImplementedError) as exc: + pytest.skip(f"symlink creation unavailable: {exc}") + + with patch( + "tools.transcription_tools._prepare_audio_for_transcription", create=True + ) as mock_prepare: + from tools.transcription_tools import transcribe_audio + result = transcribe_audio(str(link)) + + assert result["success"] is False + assert "symbolic link" in result["error"] + mock_prepare.assert_not_called() + + def test_oversized_silk_is_rejected_before_preprocessing(self, tmp_path): + """A Silk source over the upload limit must not reach the decoder.""" + silk_path = tmp_path / "oversized.silk" + from tools.transcription_tools import MAX_FILE_SIZE + with silk_path.open("wb") as audio_file: + audio_file.truncate(MAX_FILE_SIZE + 1) + + with patch( + "tools.transcription_tools._prepare_audio_for_transcription", create=True + ) as mock_prepare: + from tools.transcription_tools import transcribe_audio + result = transcribe_audio(str(silk_path)) + + assert result["success"] is False + assert "File too large" in result["error"] + mock_prepare.assert_not_called() + def test_default_model_used_when_none(self, sample_ogg): with patch("tools.transcription_tools._load_stt_config", return_value={}), \ patch("tools.transcription_tools._get_provider", return_value="groq"), \ diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index 0cabac21b2d5..155e43ce6165 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -136,6 +136,9 @@ "sounddevice==0.5.5", "numpy==2.4.3", ), + # SILK voice-note decoding (WeChat/QQ .silk voice messages). pilk is a + # small silk-v3 codec binding; installed on first .silk transcription. + "stt.silk": ("pilk==0.2.4",), # ─── Image generation backends ───────────────────────────────────────── "image.fal": ("fal-client==0.13.1",), diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index c1735a80fac2..f7eb3c54af20 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -81,6 +81,7 @@ def _safe_find_spec(module_name: str) -> bool: _HAS_FASTER_WHISPER = _safe_find_spec("faster_whisper") _HAS_OPENAI = _safe_find_spec("openai") _HAS_MISTRAL = _safe_find_spec("mistralai") +_HAS_PILK = _safe_find_spec("pilk") # --------------------------------------------------------------------------- # Constants @@ -1130,12 +1131,12 @@ def _validate_audio_file_size(audio_path: Path) -> Optional[Dict[str, Any]]: return None -def _validate_audio_file( +def _validate_audio_source_file( file_path: str, *, enforce_size_limit: bool = True, ) -> Optional[Dict[str, Any]]: - """Validate the audio file. Returns an error dict or None if OK.""" + """Validate source path safety (and optionally size) before any decoder runs.""" audio_path = Path(file_path) if os.path.islink(audio_path): @@ -1144,12 +1145,6 @@ def _validate_audio_file( return {"success": False, "transcript": "", "error": f"Audio file not found: {file_path}"} if not audio_path.is_file(): return {"success": False, "transcript": "", "error": f"Path is not a file: {file_path}"} - if audio_path.suffix.lower() not in SUPPORTED_FORMATS: - return { - "success": False, - "transcript": "", - "error": f"Unsupported format: {audio_path.suffix}. Supported: {', '.join(sorted(SUPPORTED_FORMATS))}", - } if enforce_size_limit: return _validate_audio_file_size(audio_path) try: @@ -1158,6 +1153,69 @@ def _validate_audio_file( return {"success": False, "transcript": "", "error": f"Failed to access file: {e}"} return None + +def _validate_audio_file( + file_path: str, + *, + enforce_size_limit: bool = True, +) -> Optional[Dict[str, Any]]: + """Validate a supported, decoder-safe audio file.""" + source_error = _validate_audio_source_file( + file_path, enforce_size_limit=enforce_size_limit + ) + if source_error: + return source_error + + audio_path = Path(file_path) + if audio_path.suffix.lower() not in SUPPORTED_FORMATS: + return { + "success": False, + "transcript": "", + "error": f"Unsupported format: {audio_path.suffix}. Supported: {', '.join(sorted(SUPPORTED_FORMATS))}", + } + return None + + +def _prepare_audio_for_transcription( + file_path: str, +) -> tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]: + """Convert a decoder-safe .silk source to a temporary supported WAV file.""" + audio_path = Path(file_path) + if audio_path.suffix.lower() != ".silk": + return file_path, None, None + if not _HAS_PILK: + # pilk is a tiny silk-v3 codec binding — lazy-install it on first + # .silk voice note instead of bloating the base install. + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("stt.silk", prompt=False) + except Exception: + pass + if not _safe_find_spec("pilk"): + return None, None, { + "success": False, + "transcript": "", + "error": "Unsupported format: .silk. Install the optional 'pilk' dependency to enable WeChat voice transcription.", + } + + temp_dir = tempfile.mkdtemp(prefix="hermes-silk-") + converted_path = os.path.join(temp_dir, f"{audio_path.stem}.wav") + try: + import pilk + + pilk.silk_to_wav(file_path, converted_path) + if not Path(converted_path).is_file() or Path(converted_path).stat().st_size == 0: + raise RuntimeError("pilk did not produce a readable WAV file") + return converted_path, temp_dir, None + except Exception as exc: + shutil.rmtree(temp_dir, ignore_errors=True) + logger.error("Failed to convert .silk audio %s: %s", file_path, exc, exc_info=True) + return None, None, { + "success": False, + "transcript": "", + "error": f"Failed to convert .silk audio for transcription: {exc}", + } + # --------------------------------------------------------------------------- # Provider: local (faster-whisper) # --------------------------------------------------------------------------- @@ -1890,7 +1948,7 @@ def _transcribe_deepinfra(file_path: str, model_name: str) -> Dict[str, Any]: # --------------------------------------------------------------------------- -def transcribe_audio(file_path: str, model: Optional[str] = None) -> Dict[str, Any]: +def _transcribe_prepared_audio(file_path: str, model: Optional[str] = None) -> Dict[str, Any]: """ Transcribe an audio file using the configured STT provider. @@ -1910,7 +1968,8 @@ def transcribe_audio(file_path: str, model: Optional[str] = None) -> Dict[str, A - "provider" (str, optional): Which provider was used """ # Apply common path validation before provider resolution so invalid files - # cannot trigger provider setup or lazy installation. + # cannot trigger provider setup or lazy installation. The remote-upload + # size cap is enforced separately below, only for non-local providers. error = _validate_audio_file(file_path, enforce_size_limit=False) if error: return error @@ -2041,6 +2100,36 @@ def transcribe_audio(file_path: str, model: Optional[str] = None) -> Dict[str, A } +def transcribe_audio(file_path: str, model: Optional[str] = None) -> Dict[str, Any]: + """Safely validate, preprocess supported inputs, and dispatch transcription.""" + # Cap .silk sources before the decoder runs (decoder safety). For all + # other inputs the remote-upload size cap is provider-scoped and enforced + # in _transcribe_prepared_audio, so local whisper can handle big files. + is_silk = Path(file_path).suffix.lower() == ".silk" + source_error = _validate_audio_source_file(file_path, enforce_size_limit=is_silk) + if source_error: + return source_error + + prepared_path, cleanup_dir, prep_error = _prepare_audio_for_transcription(file_path) + if prep_error: + return prep_error + if prepared_path is None: + return { + "success": False, + "transcript": "", + "error": "Audio preprocessing did not produce a file for transcription.", + } + + try: + prepared_error = _validate_audio_file(prepared_path, enforce_size_limit=False) + if prepared_error: + return prepared_error + return _transcribe_prepared_audio(prepared_path, model) + finally: + if cleanup_dir: + shutil.rmtree(cleanup_dir, ignore_errors=True) + + def _resolve_openai_audio_client_config() -> tuple[str, str]: """Return direct OpenAI audio config or a managed gateway fallback.""" stt_config = _load_stt_config() From 4331d447da705c8515c2f4dff0d09e8bb4ab820c Mon Sep 17 00:00:00 2001 From: Carl Borg Date: Tue, 21 Jul 2026 17:34:43 +0200 Subject: [PATCH 17/23] transcription: transcode to m4a and retry when OpenAI STT rejects the audio container Newer OpenAI transcription models (gpt-4o-transcribe, gpt-4o-mini-transcribe) reject some containers the legacy whisper-1 endpoint accepted -- notably the Ogg/Opus voice notes messaging platforms deliver -- returning a 400 'corrupted or unsupported' error, so voice-note transcription fails for users on those models even though SUPPORTED_FORMATS still advertises .ogg/.aac/.flac. Wrap the OpenAI upload: on a format-related BadRequestError, transcode the source to a compact 16 kHz mono AAC .m4a via ffmpeg and retry once. This is model-agnostic (no per-model format table to maintain) and adds no cost for formats the endpoint already accepts. Fixes #68719 --- tools/transcription_tools.py | 66 +++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index f7eb3c54af20..dc0c447170ed 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -193,6 +193,38 @@ def _find_ffmpeg_binary() -> Optional[str]: return _find_binary("ffmpeg") +def _transcode_audio_for_stt(file_path: str, work_dir: str) -> tuple[Optional[str], Optional[str]]: + """Transcode ``file_path`` to a compact, broadly-accepted .m4a for STT upload. + + Newer OpenAI transcription models (``gpt-4o-transcribe``, + ``gpt-4o-mini-transcribe``) reject some containers the legacy ``whisper-1`` + endpoint accepted -- notably the Ogg/Opus voice notes messaging apps send -- + and gateway downloads occasionally arrive with a misleading extension. + Normalizing to 16 kHz mono AAC/m4a produces a small file the endpoints + accept. Returns ``(converted_path, None)`` on success or ``(None, error)``. + """ + ffmpeg = _find_ffmpeg_binary() + if not ffmpeg: + return None, "audio needs transcoding for the STT API, but ffmpeg was not found" + converted_path = os.path.join(work_dir, f"{Path(file_path).stem or 'audio'}-stt.m4a") + command = [ + ffmpeg, "-y", "-i", file_path, + "-vn", "-ac", "1", "-ar", "16000", + "-c:a", "aac", "-b:a", "32k", "-movflags", "+faststart", + converted_path, + ] + try: + subprocess.run(command, check=True, capture_output=True, text=True, timeout=120) + return converted_path, None + except subprocess.CalledProcessError as exc: + details = exc.stderr.strip() or exc.stdout.strip() or str(exc) + logger.error("ffmpeg STT transcode failed for %s: %s", file_path, details) + return None, f"failed to transcode audio for the STT API: {details}" + except Exception as exc: # noqa: BLE001 - transcode is best-effort + logger.error("unexpected STT transcode failure for %s: %s", file_path, exc, exc_info=True) + return None, f"failed to transcode audio for the STT API: {exc}" + + def _find_whisper_binary() -> Optional[str]: return _find_binary("whisper") @@ -1609,10 +1641,17 @@ def _transcribe_openai( model_name = DEFAULT_STT_MODEL try: - from openai import OpenAI, APIError, APIConnectionError, APITimeoutError + from openai import ( + OpenAI, + APIError, + APIConnectionError, + APITimeoutError, + BadRequestError, + ) client = OpenAI(api_key=api_key, base_url=base_url, timeout=30, max_retries=0) - try: - with open(file_path, "rb") as audio_file: + + def _create_transcription(path: str): + with open(path, "rb") as audio_file: create_kwargs = { "model": model_name, "file": audio_file, @@ -1621,8 +1660,27 @@ def _transcribe_openai( if language: create_kwargs["language"] = language logger.debug("Using language hint '%s' for OpenAI STT", language) + return client.audio.transcriptions.create(**create_kwargs) - transcription = client.audio.transcriptions.create(**create_kwargs) + try: + with tempfile.TemporaryDirectory(prefix="hermes-stt-") as work_dir: + try: + transcription = _create_transcription(file_path) + except BadRequestError as exc: + message = str(exc).lower() + if not any(k in message for k in ("unsupported", "corrupted", "invalid file")): + raise + # Newer models (e.g. gpt-4o-transcribe) reject some containers + # whisper-1 accepted (notably Ogg/Opus voice notes). Transcode + # to a compact .m4a and retry once. + converted_path, transcode_error = _transcode_audio_for_stt(file_path, work_dir) + if transcode_error: + return {"success": False, "transcript": "", "error": transcode_error} + logger.info( + "Retrying %s STT after transcoding %s to m4a (API rejected the original container)", + provider_label, Path(file_path).name, + ) + transcription = _create_transcription(converted_path) transcript_text = _extract_transcript_text(transcription) logger.info( From 1fc603c6688c90b8cdc2090618a9e99a514185d0 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:32:06 -0700 Subject: [PATCH 18/23] fix(stt): lock local model load; allow keyless local OpenAI-compatible STT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two small fresh fixes on top of the salvage wave: - Wrap the check-then-load of the module-global faster-whisper model in a double-checked threading.Lock so concurrent voice messages can't both download/load the model (#24767). - Treat an empty stt.openai.api_key as no-auth when stt.openai.base_url points at a loopback/RFC-1918/.local host, so local OpenAI-compatible STT servers (faster-whisper-server, speaches, vLLM whisper) work without a sham api_key value. Reimplements the idea from PR #25193 — credit @nnnet. Co-authored-by: nnnet --- tests/tools/test_transcription_tools.py | 99 +++++++++++++++++++++++++ tools/transcription_tools.py | 65 ++++++++++++---- 2 files changed, 151 insertions(+), 13 deletions(-) diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index bc117b76e465..db4482641942 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -1978,3 +1978,102 @@ def test_no_env_var_uses_list_mode(self, monkeypatch): monkeypatch.delenv(LOCAL_STT_COMMAND_ENV, raising=False) use_shell = bool(os.getenv(LOCAL_STT_COMMAND_ENV, "").strip()) assert use_shell is False + + +class TestLocalModelLock: + """#24767 — concurrent first-use must not double-load the whisper model.""" + + def test_lock_exists_and_is_a_lock(self): + import threading + from tools import transcription_tools + assert isinstance(transcription_tools._local_model_lock, type(threading.Lock())) + + def test_concurrent_transcribe_loads_model_once(self, tmp_path): + import threading + from tools import transcription_tools + from tools.transcription_tools import _transcribe_local + + audio = tmp_path / "test.ogg" + audio.write_bytes(b"fake") + + seg = MagicMock() + seg.text = "hello" + info = MagicMock() + info.language = "en" + info.duration = 1.0 + + load_count = 0 + load_started = threading.Event() + + def slow_load(model_name, device="auto", compute_type="auto"): + nonlocal load_count + load_count += 1 + load_started.set() + import time + time.sleep(0.05) + model = MagicMock() + model.transcribe.return_value = ([seg], info) + return model + + with patch("tools.transcription_tools._HAS_FASTER_WHISPER", True), \ + patch("tools.transcription_tools._load_stt_config", return_value={}), \ + patch("tools.transcription_tools._load_local_whisper_model", side_effect=slow_load), \ + patch("tools.transcription_tools._local_model", None), \ + patch("tools.transcription_tools._local_model_name", None): + threads = [ + threading.Thread(target=_transcribe_local, args=(str(audio), "base")) + for _ in range(4) + ] + for t in threads: + t.start() + for t in threads: + t.join(timeout=10) + + assert load_count == 1 + + +class TestLocalBaseUrlNoApiKey: + """#25193 — empty api_key with a local base_url should not raise.""" + + def test_local_base_url_returns_placeholder_key(self): + from tools.transcription_tools import _resolve_openai_audio_client_config + with patch( + "tools.transcription_tools._load_stt_config", + return_value={"openai": {"base_url": "http://localhost:8504/v1"}}, + ): + api_key, base_url = _resolve_openai_audio_client_config() + assert api_key == "not-needed" + assert base_url == "http://localhost:8504/v1" + + def test_private_ip_base_url_returns_placeholder_key(self): + from tools.transcription_tools import _resolve_openai_audio_client_config + with patch( + "tools.transcription_tools._load_stt_config", + return_value={"openai": {"base_url": "http://192.168.1.10:8000/v1"}}, + ): + api_key, base_url = _resolve_openai_audio_client_config() + assert api_key == "not-needed" + + def test_public_base_url_still_requires_key(self): + from tools.transcription_tools import _resolve_openai_audio_client_config + with patch( + "tools.transcription_tools._load_stt_config", + return_value={"openai": {"base_url": "https://api.example.com/v1"}}, + ), patch( + "tools.transcription_tools.resolve_openai_audio_api_key", return_value="", + ), patch( + "tools.transcription_tools.resolve_managed_tool_gateway", return_value=None, + ), patch( + "tools.transcription_tools.managed_nous_tools_enabled", return_value=False, + ): + with pytest.raises(ValueError): + _resolve_openai_audio_client_config() + + def test_is_local_or_private_url(self): + from tools.transcription_tools import _is_local_or_private_url + assert _is_local_or_private_url("http://localhost:8504/v1") + assert _is_local_or_private_url("http://127.0.0.1:9000") + assert _is_local_or_private_url("http://10.0.0.5/v1") + assert _is_local_or_private_url("http://stt.internal/v1") + assert not _is_local_or_private_url("https://api.openai.com/v1") + assert not _is_local_or_private_url("") diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index dc0c447170ed..548f45976b42 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -35,6 +35,7 @@ import shutil import subprocess import tempfile +import threading from pathlib import Path from typing import Optional, Dict, Any from urllib.parse import urljoin @@ -115,6 +116,10 @@ def _safe_find_spec(module_name: str) -> bool: # Singleton for the local model — loaded once, reused across calls _local_model: Optional[object] = None _local_model_name: Optional[str] = None +# Guards the check-then-load of the module-global model cache above. +# Without it, two concurrent voice messages can both see `_local_model is +# None` and download/load the whisper model twice (#24767). +_local_model_lock = threading.Lock() # --------------------------------------------------------------------------- # Config helpers @@ -1379,20 +1384,24 @@ def _transcribe_local(file_path: str, model_name: str) -> Dict[str, Any]: try: local_cfg = _load_stt_config().get("local", {}) - # Lazy-load the model (downloads on first use, ~150 MB for 'base') + # Lazy-load the model (downloads on first use, ~150 MB for 'base'). + # Double-checked lock: concurrent voice messages must not both + # download/load the model (#24767). if _local_model is None or _local_model_name != model_name: - logger.info("Loading faster-whisper model '%s' (first load downloads the model)...", model_name) - # Honour stt.local.device / stt.local.compute_type from config so - # users on hosts where ``auto`` mis-detects (NVIDIA libs present but - # not usable, etc.) can pin a working configuration (#9088). - # _load_local_whisper_model retains the CUDA→CPU fallback for the - # auto/CUDA paths. - _local_model = _load_local_whisper_model( - model_name, - device=local_cfg.get("device", "auto"), - compute_type=local_cfg.get("compute_type", "auto"), - ) - _local_model_name = model_name + with _local_model_lock: + if _local_model is None or _local_model_name != model_name: + logger.info("Loading faster-whisper model '%s' (first load downloads the model)...", model_name) + # Honour stt.local.device / stt.local.compute_type from config so + # users on hosts where ``auto`` mis-detects (NVIDIA libs present but + # not usable, etc.) can pin a working configuration (#9088). + # _load_local_whisper_model retains the CUDA→CPU fallback for the + # auto/CUDA paths. + _local_model = _load_local_whisper_model( + model_name, + device=local_cfg.get("device", "auto"), + compute_type=local_cfg.get("compute_type", "auto"), + ) + _local_model_name = model_name # Language: stt.local.language > stt.language > env var > auto-detect. stt_config = _load_stt_config() @@ -2188,6 +2197,31 @@ def transcribe_audio(file_path: str, model: Optional[str] = None) -> Dict[str, A shutil.rmtree(cleanup_dir, ignore_errors=True) +def _is_local_or_private_url(url: str) -> bool: + """True when *url* points at a loopback/RFC-1918/LAN-internal host. + + Used to decide whether an empty ``stt.openai.api_key`` is acceptable: + local OpenAI-compatible STT servers (faster-whisper-server, speaches, + vLLM whisper variants...) ignore the auth header, so users shouldn't + have to write a sham ``api_key: not-needed`` in config.yaml. + """ + try: + from urllib.parse import urlparse + import ipaddress + + host = (urlparse(url).hostname or "").lower() + if not host: + return False + if host == "localhost" or host.endswith((".local", ".lan", ".internal")): + return True + try: + return ipaddress.ip_address(host).is_private or ipaddress.ip_address(host).is_loopback + except ValueError: + return False + except Exception: + return False + + def _resolve_openai_audio_client_config() -> tuple[str, str]: """Return direct OpenAI audio config or a managed gateway fallback.""" stt_config = _load_stt_config() @@ -2197,6 +2231,11 @@ def _resolve_openai_audio_client_config() -> tuple[str, str]: if cfg_api_key: return cfg_api_key, (cfg_base_url or OPENAI_BASE_URL) + # A local OpenAI-compatible server needs no key — send a placeholder so + # the SDK doesn't refuse to construct a client (#25193, credit @nnnet). + if cfg_base_url and _is_local_or_private_url(cfg_base_url): + return "not-needed", cfg_base_url + direct_api_key = resolve_openai_audio_api_key() if direct_api_key: return direct_api_key, OPENAI_BASE_URL From d8d4d759327e2a0ab286ba9f3f71b7ce48ae1127 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:32:28 -0700 Subject: [PATCH 19/23] chore: add contributor email mappings for salvaged STT commits --- contributors/emails/anthony.ai.assistant@gmail.com | 1 + contributors/emails/carl@sempervirens.no | 1 + contributors/emails/damian.kluk.92@gmail.com | 1 + contributors/emails/richardhojunjang@gmail.com | 1 + contributors/emails/tusharanshu18@gmail.com | 1 + contributors/emails/zehuaw@mit.edu | 1 + 6 files changed, 6 insertions(+) create mode 100644 contributors/emails/anthony.ai.assistant@gmail.com create mode 100644 contributors/emails/carl@sempervirens.no create mode 100644 contributors/emails/damian.kluk.92@gmail.com create mode 100644 contributors/emails/richardhojunjang@gmail.com create mode 100644 contributors/emails/tusharanshu18@gmail.com create mode 100644 contributors/emails/zehuaw@mit.edu diff --git a/contributors/emails/anthony.ai.assistant@gmail.com b/contributors/emails/anthony.ai.assistant@gmail.com new file mode 100644 index 000000000000..adb75f75ea6a --- /dev/null +++ b/contributors/emails/anthony.ai.assistant@gmail.com @@ -0,0 +1 @@ +AnthonyFrancis diff --git a/contributors/emails/carl@sempervirens.no b/contributors/emails/carl@sempervirens.no new file mode 100644 index 000000000000..c06588475f76 --- /dev/null +++ b/contributors/emails/carl@sempervirens.no @@ -0,0 +1 @@ +carljborg diff --git a/contributors/emails/damian.kluk.92@gmail.com b/contributors/emails/damian.kluk.92@gmail.com new file mode 100644 index 000000000000..bb613c292762 --- /dev/null +++ b/contributors/emails/damian.kluk.92@gmail.com @@ -0,0 +1 @@ +damiankluk diff --git a/contributors/emails/richardhojunjang@gmail.com b/contributors/emails/richardhojunjang@gmail.com new file mode 100644 index 000000000000..5d4e26548035 --- /dev/null +++ b/contributors/emails/richardhojunjang@gmail.com @@ -0,0 +1 @@ +RichardHojunJang diff --git a/contributors/emails/tusharanshu18@gmail.com b/contributors/emails/tusharanshu18@gmail.com new file mode 100644 index 000000000000..7703147912d6 --- /dev/null +++ b/contributors/emails/tusharanshu18@gmail.com @@ -0,0 +1 @@ +tusharui diff --git a/contributors/emails/zehuaw@mit.edu b/contributors/emails/zehuaw@mit.edu new file mode 100644 index 000000000000..90f1e4db4a94 --- /dev/null +++ b/contributors/emails/zehuaw@mit.edu @@ -0,0 +1 @@ +zehuaw1 From 222662d37537a43430544a2318d75934f82d41ea Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:34:46 -0700 Subject: [PATCH 20/23] test: align dispatch tests with provider-scoped validation and named registration errors Follow-ups for the salvaged wave: the auto-detect legacy-error test now stubs the split validators, the unknown-command-provider test expects the new provider_not_registered error, and _transcribe_local tolerates a null stt.local config section again. --- tests/tools/test_transcription_command_providers.py | 5 ++++- tests/tools/test_transcription_plugin_dispatch.py | 2 ++ tools/transcription_tools.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/tools/test_transcription_command_providers.py b/tests/tools/test_transcription_command_providers.py index 10d6e9b332a8..f496bfe3bfe7 100644 --- a/tests/tools/test_transcription_command_providers.py +++ b/tests/tools/test_transcription_command_providers.py @@ -528,7 +528,10 @@ def test_unknown_provider_no_command_falls_through_to_error(self, tmp_path): with patch("tools.transcription_tools._load_stt_config", return_value=cfg): result = transcribe_audio(str(audio)) assert result["success"] is False - assert "No STT provider available" in result["error"] + # Explicitly-configured unknown providers now get a named + # registration error instead of the generic legacy message. + assert result["error_type"] == "provider_not_registered" + assert "unknown-cli" in result["error"] # --------------------------------------------------------------------------- diff --git a/tests/tools/test_transcription_plugin_dispatch.py b/tests/tools/test_transcription_plugin_dispatch.py index 7bfd311de1a8..fa8b6db4c9fc 100644 --- a/tests/tools/test_transcription_plugin_dispatch.py +++ b/tests/tools/test_transcription_plugin_dispatch.py @@ -256,6 +256,8 @@ def test_auto_detect_failure_keeps_legacy_no_provider_message(self): from unittest.mock import patch with patch("tools.transcription_tools._validate_audio_file", return_value=None), \ + patch("tools.transcription_tools._validate_audio_source_file", return_value=None), \ + patch("tools.transcription_tools._validate_audio_file_size", return_value=None), \ patch("tools.transcription_tools._load_stt_config", return_value={}), \ patch("tools.transcription_tools.is_stt_enabled", return_value=True), \ patch("tools.transcription_tools._get_provider", return_value="none"): diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index 548f45976b42..ccaa3315423c 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -1383,7 +1383,7 @@ def _transcribe_local(file_path: str, model_name: str) -> Dict[str, Any]: return {"success": False, "transcript": "", "error": "faster-whisper not installed"} try: - local_cfg = _load_stt_config().get("local", {}) + local_cfg = _load_stt_config().get("local") or {} # Lazy-load the model (downloads on first use, ~150 MB for 'base'). # Double-checked lock: concurrent voice messages must not both # download/load the model (#24767). From 213fbd25ad2bebd15bfd55480af7a991e940905a Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:36:17 -0700 Subject: [PATCH 21/23] test: add BadRequestError to the fake openai module fixture _transcribe_openai now imports BadRequestError for the container-retry path; the managed-gateway fake module needs to provide it. --- tests/tools/test_managed_media_gateways.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tools/test_managed_media_gateways.py b/tests/tools/test_managed_media_gateways.py index d7d35e4b7b69..6dc76374d5ab 100644 --- a/tests/tools/test_managed_media_gateways.py +++ b/tests/tools/test_managed_media_gateways.py @@ -169,6 +169,7 @@ def close(self): APIError=Exception, APIConnectionError=Exception, APITimeoutError=Exception, + BadRequestError=type("BadRequestError", (Exception,), {}), ) sys.modules["openai"] = fake_module From b569b6bfe485557d55c69f3f2d519958086f5993 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:29:46 -0700 Subject: [PATCH 22/23] fix: explicit utf-8 encoding on ffmpeg STT transcode subprocess (Windows footgun lint) --- tools/transcription_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index ccaa3315423c..72ce4e3572af 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -219,7 +219,7 @@ def _transcode_audio_for_stt(file_path: str, work_dir: str) -> tuple[Optional[st converted_path, ] try: - subprocess.run(command, check=True, capture_output=True, text=True, timeout=120) + subprocess.run(command, check=True, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=120) return converted_path, None except subprocess.CalledProcessError as exc: details = exc.stderr.strip() or exc.stdout.strip() or str(exc) From fa5bf441876272809e6805cafc6ba3a44ed376b5 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:49:08 -0700 Subject: [PATCH 23/23] fix: stdin=DEVNULL + windows_hide_flags on STT transcode subprocess (guard test) --- tools/transcription_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/transcription_tools.py b/tools/transcription_tools.py index 72ce4e3572af..b329039b5035 100644 --- a/tools/transcription_tools.py +++ b/tools/transcription_tools.py @@ -219,7 +219,7 @@ def _transcode_audio_for_stt(file_path: str, work_dir: str) -> tuple[Optional[st converted_path, ] try: - subprocess.run(command, check=True, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=120) + subprocess.run(command, check=True, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=120, stdin=subprocess.DEVNULL, creationflags=windows_hide_flags()) return converted_path, None except subprocess.CalledProcessError as exc: details = exc.stderr.strip() or exc.stdout.strip() or str(exc)