From 648c9d9e4b3e29b626d479d3b708e3c01cfe3b4e Mon Sep 17 00:00:00 2001 From: Bailey Dixon Date: Wed, 20 May 2026 11:34:01 -0400 Subject: [PATCH] feat: add API server audio endpoints --- gateway/platforms/api_server.py | 203 ++++++++++++++++++++++++++++++- tests/gateway/test_api_server.py | 199 +++++++++++++++++++++++++++++- 2 files changed, 400 insertions(+), 2 deletions(-) diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index 0668896e170f7..a7cacb11340b4 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -8,6 +8,9 @@ - DELETE /v1/responses/{response_id} — Delete a stored response - GET /v1/models — lists hermes-agent as an available model - GET /v1/capabilities — machine-readable API capabilities for external UIs +- GET /api/audio/capabilities — voice/STT/TTS capability metadata +- POST /api/audio/transcriptions — multipart audio upload to configured STT +- POST /api/audio/speech — text-to-speech via configured TTS - POST /v1/runs — start a run, returns run_id immediately (202) - GET /v1/runs/{run_id} — retrieve current run status - GET /v1/runs/{run_id}/events — SSE stream of structured lifecycle events @@ -33,8 +36,10 @@ import socket as _socket import re import sqlite3 +import tempfile import time import uuid +from pathlib import Path from typing import Any, Dict, List, Optional try: @@ -57,7 +62,8 @@ DEFAULT_HOST = "127.0.0.1" DEFAULT_PORT = 8642 MAX_STORED_RESPONSES = 100 -MAX_REQUEST_BYTES = 10_000_000 # 10 MB — accommodates long agent conversations with tool calls +MAX_AUDIO_UPLOAD_BYTES = 25 * 1024 * 1024 # 25 MB — matches core STT upload limit +MAX_REQUEST_BYTES = MAX_AUDIO_UPLOAD_BYTES # accommodates long conversations and audio uploads CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS = 30.0 MAX_NORMALIZED_TEXT_LENGTH = 65_536 # 64 KB cap for normalized content parts MAX_CONTENT_LIST_SIZE = 1_000 # Max items when content is an array @@ -1000,6 +1006,8 @@ async def _handle_capabilities(self, request: "web.Request") -> "web.Response": "run_events_sse": True, "run_stop": True, "run_approval_response": True, + "audio_transcription": True, + "audio_speech": True, "tool_progress_events": True, "approval_events": True, "session_continuity_header": "X-Hermes-Session-Id", @@ -1010,6 +1018,9 @@ async def _handle_capabilities(self, request: "web.Request") -> "web.Response": "health": {"method": "GET", "path": "/health"}, "health_detailed": {"method": "GET", "path": "/health/detailed"}, "models": {"method": "GET", "path": "/v1/models"}, + "audio_capabilities": {"method": "GET", "path": "/api/audio/capabilities"}, + "audio_transcriptions": {"method": "POST", "path": "/api/audio/transcriptions"}, + "audio_speech": {"method": "POST", "path": "/api/audio/speech"}, "chat_completions": {"method": "POST", "path": "/v1/chat/completions"}, "responses": {"method": "POST", "path": "/v1/responses"}, "runs": {"method": "POST", "path": "/v1/runs"}, @@ -1020,6 +1031,190 @@ async def _handle_capabilities(self, request: "web.Request") -> "web.Response": }, }) + def _build_audio_capabilities(self) -> Dict[str, Any]: + """Return client-safe audio capability metadata without exposing config.""" + transcription_available = False + transcription_provider = "none" + try: + from tools.transcription_tools import _get_provider, _load_stt_config, is_stt_enabled + + stt_config = _load_stt_config() + if is_stt_enabled(stt_config): + transcription_provider = _get_provider(stt_config) + transcription_available = transcription_provider != "none" + except Exception as exc: + logger.debug("Failed to inspect STT capabilities: %s", exc, exc_info=True) + + speech_available = False + try: + from tools.tts_tool import check_tts_requirements + + speech_available = bool(check_tts_requirements()) + except Exception as exc: + logger.debug("Failed to inspect TTS capabilities: %s", exc, exc_info=True) + + return { + "object": "hermes.api_server.audio_capabilities", + "auth": { + "type": "bearer", + "required": bool(self._api_key), + }, + "transcription": { + "available": bool(transcription_available), + "provider": transcription_provider if transcription_available else None, + "endpoint": "/api/audio/transcriptions", + "method": "POST", + "request": "multipart/form-data; field name 'file'", + "max_upload_bytes": MAX_AUDIO_UPLOAD_BYTES, + }, + "speech": { + "available": bool(speech_available), + "endpoint": "/api/audio/speech", + "method": "POST", + "request": "application/json; {'text': '...'}", + "response_content_type": "audio/mpeg", + }, + "aliases": { + "capabilities": "/voice/config", + "transcriptions": "/voice/transcribe", + "speech": "/voice/synthesize", + }, + "realtime": { + "available": False, + "description": "HTTP STT/TTS only; WebRTC and provider-native realtime sessions are not implemented.", + }, + } + + async def _handle_audio_capabilities(self, request: "web.Request") -> "web.Response": + """GET /api/audio/capabilities — advertise safe voice/STT/TTS metadata.""" + auth_err = self._check_auth(request) + if auth_err: + return auth_err + return web.json_response(self._build_audio_capabilities()) + + async def _handle_audio_transcription(self, request: "web.Request") -> "web.Response": + """POST /api/audio/transcriptions — multipart upload to configured STT.""" + auth_err = self._check_auth(request) + if auth_err: + return auth_err + + try: + reader = await request.multipart() + except Exception: + return web.json_response( + _openai_error("Expected multipart/form-data with a 'file' audio field", param="file"), + status=400, + ) + + file_field = None + model: Optional[str] = None + while True: + part = await reader.next() + if part is None: + break + if part.name == "model": + raw_model = await part.text() + model = raw_model.strip() or None + continue + if part.name in {"file", "audio"}: + file_field = part + break + + if file_field is None: + return web.json_response( + _openai_error("Missing multipart audio file field named 'file'", param="file"), + status=400, + ) + + filename = file_field.filename or "audio.wav" + suffix = Path(filename).suffix.lower() or ".wav" + temp_path: Optional[str] = None + bytes_written = 0 + try: + with tempfile.NamedTemporaryFile(prefix="hermes_api_audio_", suffix=suffix, delete=False) as tmp: + temp_path = tmp.name + while True: + chunk = await file_field.read_chunk(size=1024 * 1024) + if not chunk: + break + bytes_written += len(chunk) + if bytes_written > MAX_AUDIO_UPLOAD_BYTES: + return web.json_response( + _openai_error("Audio upload too large.", param="file", code="audio_too_large"), + status=413, + ) + tmp.write(chunk) + + if bytes_written == 0: + return web.json_response(_openai_error("Audio file is empty", param="file"), status=400) + + from tools.transcription_tools import transcribe_audio + + result = await asyncio.to_thread(transcribe_audio, temp_path, model) + finally: + if temp_path: + try: + os.unlink(temp_path) + except FileNotFoundError: + pass + except Exception as exc: + logger.debug("Failed to remove temp audio upload %s: %s", temp_path, exc) + + if not result.get("success"): + message = str(result.get("error") or "Transcription failed") + status = 400 if "unsupported format" in message.lower() else 503 + return web.json_response(_openai_error(message, code="transcription_failed"), status=status) + + transcript = str(result.get("transcript") or result.get("text") or "") + return web.json_response({ + "object": "audio.transcription", + "text": transcript, + "transcript": transcript, + "provider": result.get("provider"), + }) + + async def _handle_audio_speech(self, request: "web.Request") -> "web.Response": + """POST /api/audio/speech — synthesize text with configured Hermes TTS.""" + auth_err = self._check_auth(request) + if auth_err: + return auth_err + + try: + body = await request.json() + except (json.JSONDecodeError, Exception): + return web.json_response(_openai_error("Invalid JSON in request body"), status=400) + + text = body.get("text") if isinstance(body, dict) else None + if text is None and isinstance(body, dict): + text = body.get("input") + if not isinstance(text, str) or not text.strip(): + return web.json_response(_openai_error("Missing or empty 'text' field", param="text"), status=400) + + from tools.tts_tool import text_to_speech_tool + + result_raw = await asyncio.to_thread(text_to_speech_tool, text=text.strip()) + try: + result = json.loads(result_raw) if isinstance(result_raw, str) else dict(result_raw) + except Exception: + return web.json_response(_openai_error("TTS returned an invalid response", code="tts_invalid_response"), status=502) + + if not result.get("success"): + message = str(result.get("error") or "TTS generation failed") + return web.json_response(_openai_error(message, code="tts_failed"), status=503) + + file_path = result.get("file_path") + if not file_path: + return web.json_response(_openai_error("TTS response did not include an audio file", code="tts_missing_file"), status=502) + + try: + audio_bytes = Path(str(file_path)).read_bytes() + except Exception as exc: + logger.error("Failed to read TTS output %s: %s", file_path, exc, exc_info=True) + return web.json_response(_openai_error("Failed to read generated speech audio", code="tts_read_failed"), status=502) + + headers = {"X-Hermes-TTS-Provider": str(result.get("provider") or "")} + return web.Response(body=audio_bytes, content_type="audio/mpeg", headers=headers) + async def _handle_chat_completions(self, request: "web.Request") -> "web.Response": """POST /v1/chat/completions — OpenAI Chat Completions format.""" auth_err = self._check_auth(request) @@ -3402,6 +3597,12 @@ async def connect(self) -> bool: self._app.router.add_get("/v1/health", self._handle_health) self._app.router.add_get("/v1/models", self._handle_models) self._app.router.add_get("/v1/capabilities", self._handle_capabilities) + self._app.router.add_get("/api/audio/capabilities", self._handle_audio_capabilities) + self._app.router.add_post("/api/audio/transcriptions", self._handle_audio_transcription) + self._app.router.add_post("/api/audio/speech", self._handle_audio_speech) + self._app.router.add_get("/voice/config", self._handle_audio_capabilities) + self._app.router.add_post("/voice/transcribe", self._handle_audio_transcription) + self._app.router.add_post("/voice/synthesize", self._handle_audio_speech) self._app.router.add_post("/v1/chat/completions", self._handle_chat_completions) self._app.router.add_post("/v1/responses", self._handle_responses) self._app.router.add_get("/v1/responses/{response_id}", self._handle_get_response) diff --git a/tests/gateway/test_api_server.py b/tests/gateway/test_api_server.py index aae5f55053207..7203cc2fe110c 100644 --- a/tests/gateway/test_api_server.py +++ b/tests/gateway/test_api_server.py @@ -19,7 +19,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest -from aiohttp import web +from aiohttp import FormData, web from aiohttp.test_utils import AioHTTPTestCase, TestClient, TestServer from gateway.config import GatewayConfig, Platform, PlatformConfig @@ -380,6 +380,12 @@ def _create_app(adapter: APIServerAdapter) -> web.Application: app.router.add_get("/v1/health", adapter._handle_health) app.router.add_get("/v1/models", adapter._handle_models) app.router.add_get("/v1/capabilities", adapter._handle_capabilities) + app.router.add_get("/api/audio/capabilities", adapter._handle_audio_capabilities) + app.router.add_post("/api/audio/transcriptions", adapter._handle_audio_transcription) + app.router.add_post("/api/audio/speech", adapter._handle_audio_speech) + app.router.add_get("/voice/config", adapter._handle_audio_capabilities) + app.router.add_post("/voice/transcribe", adapter._handle_audio_transcription) + app.router.add_post("/voice/synthesize", adapter._handle_audio_speech) app.router.add_post("/v1/chat/completions", adapter._handle_chat_completions) app.router.add_post("/v1/responses", adapter._handle_responses) app.router.add_get("/v1/responses/{response_id}", adapter._handle_get_response) @@ -640,6 +646,197 @@ async def test_capabilities_requires_auth_when_key_configured(self, auth_adapter data = await authed.json() assert data["auth"]["required"] is True + @pytest.mark.asyncio + async def test_capabilities_advertises_audio_endpoints(self, adapter): + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.get("/v1/capabilities") + assert resp.status == 200 + data = await resp.json() + assert data["features"]["audio_transcription"] is True + assert data["features"]["audio_speech"] is True + assert data["endpoints"]["audio_capabilities"] == { + "method": "GET", + "path": "/api/audio/capabilities", + } + assert data["endpoints"]["audio_transcriptions"]["path"] == "/api/audio/transcriptions" + assert data["endpoints"]["audio_speech"]["path"] == "/api/audio/speech" + + +# --------------------------------------------------------------------------- +# /api/audio endpoints +# --------------------------------------------------------------------------- + + +class TestAudioEndpoints: + @pytest.mark.asyncio + async def test_audio_capabilities_returns_safe_shape(self, adapter): + app = _create_app(adapter) + with ( + patch("tools.transcription_tools._load_stt_config", return_value={"enabled": True, "provider": "local"}), + patch("tools.transcription_tools._get_provider", return_value="local"), + patch("tools.tts_tool.check_tts_requirements", return_value=True), + ): + async with TestClient(TestServer(app)) as cli: + resp = await cli.get("/api/audio/capabilities") + assert resp.status == 200 + data = await resp.json() + + assert data["object"] == "hermes.api_server.audio_capabilities" + assert data["auth"]["required"] is False + assert data["transcription"]["available"] is True + assert data["transcription"]["endpoint"] == "/api/audio/transcriptions" + assert data["transcription"]["max_upload_bytes"] == 25 * 1024 * 1024 + assert data["speech"]["available"] is True + assert data["speech"]["endpoint"] == "/api/audio/speech" + assert "api_key" not in json.dumps(data).lower() + + @pytest.mark.asyncio + async def test_audio_capabilities_requires_auth_when_key_configured(self, auth_adapter): + app = _create_app(auth_adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.get("/api/audio/capabilities") + assert resp.status == 401 + authed = await cli.get( + "/api/audio/capabilities", + headers={"Authorization": "Bearer sk-secret"}, + ) + assert authed.status == 200 + data = await authed.json() + assert data["auth"]["required"] is True + + @pytest.mark.asyncio + async def test_audio_capabilities_marks_unavailable_without_providers(self, adapter): + app = _create_app(adapter) + with ( + patch("tools.transcription_tools._load_stt_config", return_value={"enabled": False}), + patch("tools.transcription_tools._get_provider", return_value="none"), + patch("tools.tts_tool.check_tts_requirements", return_value=False), + ): + async with TestClient(TestServer(app)) as cli: + resp = await cli.get("/api/audio/capabilities") + assert resp.status == 200 + data = await resp.json() + + assert data["transcription"]["available"] is False + assert data["transcription"]["provider"] is None + assert data["speech"]["available"] is False + assert data["realtime"]["available"] is False + + @pytest.mark.asyncio + async def test_transcription_upload_calls_existing_stt_helper(self, adapter): + app = _create_app(adapter) + form = FormData() + form.add_field("file", b"fake wav bytes", filename="clip.wav", content_type="audio/wav") + + with patch( + "tools.transcription_tools.transcribe_audio", + return_value={"success": True, "transcript": "hello world", "provider": "test-stt"}, + ) as transcribe: + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/audio/transcriptions", data=form) + assert resp.status == 200 + data = await resp.json() + + assert data == { + "object": "audio.transcription", + "text": "hello world", + "transcript": "hello world", + "provider": "test-stt", + } + transcribe.assert_called_once() + assert transcribe.call_args.args[0].endswith(".wav") + + @pytest.mark.asyncio + async def test_transcription_rejects_missing_file(self, adapter): + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/audio/transcriptions", data={"model": "whisper-1"}) + assert resp.status == 400 + data = await resp.json() + assert "file" in data["error"]["message"].lower() + + @pytest.mark.asyncio + async def test_transcription_requires_auth_when_key_configured(self, auth_adapter): + app = _create_app(auth_adapter) + form = FormData() + form.add_field("file", b"fake wav bytes", filename="clip.wav", content_type="audio/wav") + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/audio/transcriptions", data=form) + assert resp.status == 401 + + @pytest.mark.asyncio + async def test_speech_returns_audio_mpeg_from_existing_tts_helper(self, adapter, tmp_path): + audio_path = tmp_path / "speech.mp3" + audio_path.write_bytes(b"mp3-bytes") + app = _create_app(adapter) + + with patch( + "tools.tts_tool.text_to_speech_tool", + return_value=json.dumps({"success": True, "file_path": str(audio_path), "provider": "test-tts"}), + ) as synthesize: + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/audio/speech", json={"text": "Say hello"}) + assert resp.status == 200 + assert resp.headers["Content-Type"].startswith("audio/mpeg") + assert resp.headers["X-Hermes-TTS-Provider"] == "test-tts" + body = await resp.read() + + assert body == b"mp3-bytes" + synthesize.assert_called_once() + assert synthesize.call_args.kwargs["text"] == "Say hello" + + @pytest.mark.asyncio + async def test_speech_rejects_empty_text(self, adapter): + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/audio/speech", json={"text": " "}) + assert resp.status == 400 + data = await resp.json() + assert "text" in data["error"]["message"].lower() + + @pytest.mark.asyncio + async def test_speech_requires_auth_when_key_configured(self, auth_adapter): + app = _create_app(auth_adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/audio/speech", json={"text": "hello"}) + assert resp.status == 401 + + @pytest.mark.asyncio + async def test_speech_reports_tts_failure_without_leaking_config(self, adapter): + app = _create_app(adapter) + with patch( + "tools.tts_tool.text_to_speech_tool", + return_value=json.dumps({"success": False, "error": "No TTS provider available"}), + ): + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/api/audio/speech", json={"text": "hello"}) + assert resp.status == 503 + data = await resp.json() + assert data["error"]["message"] == "No TTS provider available" + assert "api_key" not in json.dumps(data).lower() + + @pytest.mark.asyncio + async def test_voice_aliases_delegate_to_audio_handlers(self, adapter, tmp_path): + audio_path = tmp_path / "speech.mp3" + audio_path.write_bytes(b"alias-bytes") + app = _create_app(adapter) + form = FormData() + form.add_field("file", b"fake wav bytes", filename="clip.wav", content_type="audio/wav") + + with ( + patch("tools.transcription_tools.transcribe_audio", return_value={"success": True, "transcript": "alias", "provider": "test-stt"}), + patch("tools.tts_tool.text_to_speech_tool", return_value=json.dumps({"success": True, "file_path": str(audio_path), "provider": "test-tts"})), + ): + async with TestClient(TestServer(app)) as cli: + config_resp = await cli.get("/voice/config") + transcribe_resp = await cli.post("/voice/transcribe", data=form) + synth_resp = await cli.post("/voice/synthesize", json={"text": "hello"}) + assert config_resp.status == 200 + assert transcribe_resp.status == 200 + assert synth_resp.status == 200 + assert await synth_resp.read() == b"alias-bytes" + # --------------------------------------------------------------------------- # /v1/chat/completions endpoint