diff --git a/agent/models_dev.py b/agent/models_dev.py index 590f77806abf..011c6bc61e00 100644 --- a/agent/models_dev.py +++ b/agent/models_dev.py @@ -480,7 +480,8 @@ def get_model_capabilities(provider: str, model: str) -> Optional[ModelCapabilit else: input_mods = None if isinstance(input_mods, list): - supports_vision = "image" in input_mods + input_mod_values = {str(mod).strip().lower() for mod in input_mods} + supports_vision = "image" in input_mod_values else: supports_vision = bool(entry.get("attachment", False)) supports_reasoning = bool(entry.get("reasoning", False)) diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index 99b2faa1165c..b184f40d179f 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -4,6 +4,7 @@ Exposes an HTTP server with endpoints: - POST /v1/chat/completions — OpenAI Chat Completions format (stateless; opt-in session continuity via X-Hermes-Session-Id header; opt-in long-term memory scoping via X-Hermes-Session-Key header) - POST /v1/responses — OpenAI Responses API format (stateful via previous_response_id; X-Hermes-Session-Key supported) +- POST /v1/audio/transcriptions — STT utility endpoint; does not create or mutate chat sessions - GET /v1/responses/{response_id} — Retrieve a stored response - DELETE /v1/responses/{response_id} — Delete a stored response - GET /v1/models — lists hermes-agent as an available model @@ -46,10 +47,13 @@ from typing import Any, Dict, List, Optional try: - from aiohttp import web + from aiohttp import ClientSession, ClientTimeout, FormData, web AIOHTTP_AVAILABLE = True except ImportError: AIOHTTP_AVAILABLE = False + ClientSession = None # type: ignore[assignment] + ClientTimeout = None # type: ignore[assignment] + FormData = None # type: ignore[assignment] web = None # type: ignore[assignment] from gateway.config import Platform, PlatformConfig @@ -90,10 +94,33 @@ def _hermes_version() -> str: DEFAULT_PORT = 8642 MAX_STORED_RESPONSES = 100 MAX_REQUEST_BYTES = 10_000_000 # 10 MB — accommodates long agent conversations with tool calls +AUDIO_TRANSCRIPTION_MAX_BYTES = 25 * 1024 * 1024 +# Multipart bodies include field framing and headers in addition to the audio +# bytes. The handler enforces AUDIO_TRANSCRIPTION_MAX_BYTES on the file itself. +AUDIO_TRANSCRIPTION_REQUEST_MAX_BYTES = AUDIO_TRANSCRIPTION_MAX_BYTES + 1_048_576 +AUDIO_TRANSCRIPTION_FORMATS = ("wav", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "webm", "flac") +AUDIO_TRANSCRIPTION_TIMEOUT_SECONDS = 120 CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS = 30.0 MAX_NORMALIZED_TEXT_LENGTH = 65_536 # 64 KB cap for normalized content parts MAX_CONTENT_LIST_SIZE = 1_000 # Max items when content is an array +_AUDIO_TRANSCRIPTION_MIME_TO_FORMAT = { + "audio/wav": "wav", + "audio/wave": "wav", + "audio/x-wav": "wav", + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/mp4": "mp4", + "audio/x-m4a": "m4a", + "audio/mpga": "mpga", + "audio/ogg": "ogg", + "audio/opus": "ogg", + "audio/webm": "webm", + "video/webm": "webm", + "audio/flac": "flac", + "audio/aac": "m4a", +} + def _coerce_port(value: Any, default: int = DEFAULT_PORT) -> int: """Parse a listen port without letting malformed env/config values crash startup.""" @@ -202,6 +229,7 @@ def _normalize_chat_content( # rest of the agent pipeline already understands. _TEXT_PART_TYPES = frozenset({"text", "input_text", "output_text"}) _IMAGE_PART_TYPES = frozenset({"image_url", "input_image"}) +_AUDIO_PART_TYPES = frozenset({"input_audio", "audio"}) _FILE_PART_TYPES = frozenset({"file", "input_file"}) @@ -218,6 +246,7 @@ def _normalize_multimodal_content(content: Any) -> Any: * ``unsupported_content_type`` — file/input_file/file_id parts, or non-image ``data:`` URLs. * ``invalid_image_url`` — missing URL or unsupported scheme. + * ``unsupported_audio_input`` — raw audio parts sent to chat/session APIs. * ``invalid_content_part`` — malformed text/image objects. Callers translate the ValueError into a 400 response. @@ -298,6 +327,12 @@ def _normalize_multimodal_content(content: Any) -> Any: normalized_parts.append(image_part) continue + if part_type in _AUDIO_PART_TYPES: + raise ValueError( + "unsupported_audio_input:Audio input must be transcribed first with " + "/v1/audio/transcriptions, then sent to chat as text." + ) + if part_type in _FILE_PART_TYPES: raise ValueError( "unsupported_content_type:Inline image inputs are supported, " @@ -324,7 +359,7 @@ def _normalize_multimodal_content(content: Any) -> Any: def _content_has_visible_payload(content: Any) -> bool: - """True when content has any text or image attachment. Used to reject empty turns.""" + """True when content has any text, image, or audio attachment. Used to reject empty turns.""" if isinstance(content, str): return bool(content.strip()) if isinstance(content, list): @@ -335,6 +370,25 @@ def _content_has_visible_payload(content: Any) -> bool: return True if ptype in _IMAGE_PART_TYPES: return True + if ptype in _AUDIO_PART_TYPES: + return True + return False + + +def _content_has_audio_parts_deep(content: Any) -> bool: + stack = [content] + while stack: + current = stack.pop() + if not isinstance(current, list): + continue + for part in current: + if not isinstance(part, dict): + continue + ptype = str(part.get("type") or "").strip().lower() + if ptype in _AUDIO_PART_TYPES: + return True + if "content" in part: + stack.append(part.get("content")) return False @@ -364,6 +418,85 @@ def _session_chat_user_message(body: Dict[str, Any], *, param: str = "message") return None, _multimodal_validation_error(exc, param=param) +def _audio_format_from_mime(content_type: str) -> Optional[str]: + return _AUDIO_TRANSCRIPTION_MIME_TO_FORMAT.get(str(content_type or "").split(";", 1)[0].strip().lower()) + + +def _audio_format_for_upload(filename: str, content_type: str) -> Optional[str]: + suffix = Path(str(filename or "")).suffix.lower().lstrip(".") + if suffix in AUDIO_TRANSCRIPTION_FORMATS: + return suffix + return _audio_format_from_mime(content_type) + + +def _transcription_upstream_url(base_url: str) -> str: + return f"{str(base_url or '').strip().rstrip('/')}/audio/transcriptions" + + +def _codex_transcription_upstream_url(base_url: str) -> str: + normalized = (base_url or "").strip().rstrip("/") + if not normalized: + normalized = "https://chatgpt.com/backend-api/codex" + if normalized.endswith("/transcribe"): + return normalized + if normalized.endswith("/codex"): + normalized = normalized[: -len("/codex")] + if normalized.endswith("/backend-api"): + return normalized + "/transcribe" + return normalized + "/backend-api/transcribe" + + +def _extract_transcription_text(payload: Any) -> str: + if isinstance(payload, dict): + value = payload.get("text") + if isinstance(value, str): + return value.strip() + if isinstance(payload, str): + return payload.strip() + return "" + + +def _transcription_error_message_from_payload(payload: Any, fallback: str) -> str: + if isinstance(payload, dict): + err = payload.get("error") + if isinstance(err, dict): + message = err.get("message") + if isinstance(message, str) and message.strip(): + return message.strip() + elif isinstance(err, str) and err.strip(): + return err.strip() + message = payload.get("message") + if isinstance(message, str) and message.strip(): + return message.strip() + return fallback + + +def _redact_codex_transcription_error_text(value: Any, *, limit: int = 500) -> str: + text = _redact_api_error_text(value) + text = re.sub( + r"(?i)[\"']?\basset_pointer\b[\"']?\s*[:=]\s*[\"']?[^,}\s\"']+[\"']?", + "asset metadata [redacted]", + text, + ) + text = re.sub(r"(?i)\basset_pointer\b", "asset metadata", text) + text = re.sub(r"\bptr_[A-Za-z0-9._:-]+", "[redacted-asset-pointer]", text) + return text[:limit] + + +def _codex_transcription_error_message(status: int, payload: Any, raw_body: str) -> str: + if status == 403: + return "ChatGPT transcription rejected request." + + fallback = f"ChatGPT transcription request failed with HTTP {status}." + if isinstance(payload, dict): + message = _transcription_error_message_from_payload(payload, "") + if message: + return f"ChatGPT transcription request failed: {_redact_codex_transcription_error_text(message)}" + return fallback + + return fallback + + def check_api_server_requirements() -> bool: """Check if API server dependencies are available.""" return AIOHTTP_AVAILABLE @@ -592,6 +725,19 @@ def _openai_error(message: str, err_type: str = "invalid_request_error", param: } +def _codex_audio_auth_error_response(exc: Any) -> "web.Response": + code = getattr(exc, "code", None) or "codex_auth_required" + status = 429 if "rate" in code or "limit" in code or "quota" in code else 401 + return web.json_response( + _openai_error( + str(exc), + err_type="authentication_error", + code=code, + ), + status=status, + ) + + if AIOHTTP_AVAILABLE: @web.middleware async def body_limit_middleware(request, handler): @@ -600,7 +746,12 @@ async def body_limit_middleware(request, handler): cl = request.headers.get("Content-Length") if cl is not None: try: - if int(cl) > MAX_REQUEST_BYTES: + limit = ( + AUDIO_TRANSCRIPTION_REQUEST_MAX_BYTES + if request.path == "/v1/audio/transcriptions" + else MAX_REQUEST_BYTES + ) + if int(cl) > limit: return web.json_response(_openai_error("Request body too large.", code="body_too_large"), status=413) except ValueError: return web.json_response(_openai_error("Invalid Content-Length header.", code="invalid_content_length"), status=400) @@ -1138,6 +1289,315 @@ def _create_agent( ) return agent + def _reject_unsupported_audio_content(self, content: Any, *, param: str) -> Optional["web.Response"]: + if not _content_has_audio_parts_deep(content): + return None + return web.json_response( + _openai_error( + "Audio input must be transcribed first with /v1/audio/transcriptions, " + "then sent to chat as text.", + code="unsupported_audio_input", + param=param, + ), + status=400, + ) + + def _resolve_audio_transcription_runtime(self) -> tuple[Optional[Dict[str, str]], Optional["web.Response"]]: + try: + from hermes_cli.auth import AuthError + from hermes_cli.runtime_provider import resolve_requested_provider, resolve_runtime_provider + + requested_provider = resolve_requested_provider() + except Exception as exc: + return None, web.json_response( + _openai_error( + f"Could not resolve runtime provider for audio transcription: {exc}", + code="provider_config_error", + ), + status=503, + ) + + provider = str(requested_provider or "").strip().lower() + if provider == "openai-codex": + try: + from hermes_cli.auth import resolve_codex_runtime_credentials + + creds = resolve_codex_runtime_credentials() + except AuthError as exc: + return None, _codex_audio_auth_error_response(exc) + except Exception as exc: + return None, web.json_response( + _openai_error( + f"Could not resolve Codex credentials for audio transcription: {exc}", + err_type="authentication_error", + code="codex_auth_error", + ), + status=401, + ) + runtime_kwargs = { + "provider": "openai-codex", + "base_url": creds.get("base_url"), + "api_key": creds.get("api_key"), + } + else: + try: + runtime_kwargs = resolve_runtime_provider( + requested=requested_provider, + allow_auto_codex_fallback=False, + ) + except AuthError as exc: + if ( + getattr(exc, "provider", None) == "openai-codex" + or str(getattr(exc, "code", "") or "").startswith("codex_") + ): + return None, _codex_audio_auth_error_response(exc) + return None, web.json_response( + _openai_error( + f"Could not resolve runtime provider for audio transcription: {exc}", + code="provider_config_error", + ), + status=503, + ) + except Exception as exc: + return None, web.json_response( + _openai_error( + f"Could not resolve runtime provider for audio transcription: {exc}", + code="provider_config_error", + ), + status=503, + ) + + base_url = str(runtime_kwargs.get("base_url") or "").strip().rstrip("/") + api_key = str(runtime_kwargs.get("api_key") or "").strip() + provider = str(runtime_kwargs.get("provider") or provider or "").strip() + if not provider or not base_url or not api_key: + return None, web.json_response( + _openai_error( + "Audio transcription requires a configured runtime provider, base_url, and API key.", + code="provider_config_error", + ), + status=503, + ) + + return {"provider": provider, "base_url": base_url, "api_key": api_key}, None + + async def _post_audio_transcription( + self, + *, + runtime: Dict[str, str], + model: str, + filename: str, + content_type: str, + data: bytes, + ) -> tuple[Optional[str], Optional["web.Response"]]: + if ClientSession is None or ClientTimeout is None or FormData is None: + return None, web.json_response( + _openai_error("aiohttp is required for audio transcription.", code="missing_dependency"), + status=500, + ) + + provider = runtime["provider"] + api_key = runtime["api_key"] + headers = {"Authorization": f"Bearer {api_key}"} + if provider == "openai-codex": + try: + from agent.auxiliary_client import _codex_cloudflare_headers + + headers.update({ + "Accept": "application/json", + "Origin": "https://chatgpt.com", + "Referer": "https://chatgpt.com/", + }) + headers.update(_codex_cloudflare_headers(api_key)) + except Exception as exc: + return None, web.json_response( + _openai_error( + f"Could not build Codex transcription headers: {exc}", + err_type="server_error", + code="codex_transcription_header_error", + ), + status=500, + ) + + form = FormData() + if provider != "openai-codex": + form.add_field("model", model) + form.add_field( + "file", + data, + filename=filename or "audio", + content_type=content_type or "application/octet-stream", + ) + + url = ( + _codex_transcription_upstream_url(runtime["base_url"]) + if provider == "openai-codex" + else _transcription_upstream_url(runtime["base_url"]) + ) + timeout = ClientTimeout(total=AUDIO_TRANSCRIPTION_TIMEOUT_SECONDS) + try: + async with ClientSession(timeout=timeout) as session: + async with session.post(url, data=form, headers=headers) as resp: + raw_body = await resp.text() + try: + payload = json.loads(raw_body) if raw_body else {} + except json.JSONDecodeError: + payload = raw_body + + if resp.status >= 400: + if provider == "openai-codex": + message = _codex_transcription_error_message(resp.status, payload, raw_body) + else: + fallback = raw_body or f"Upstream transcription request failed with HTTP {resp.status}." + message = _transcription_error_message_from_payload(payload, fallback) + status = resp.status if resp.status in {400, 401, 403, 404, 413, 429} else 502 + error_message = ( + message + if provider == "openai-codex" + else f"Upstream transcription failed: {message}" + ) + return None, web.json_response( + _openai_error( + error_message, + err_type="server_error", + code="upstream_transcription_failed", + ), + status=status, + ) + + transcript = _extract_transcription_text(payload) + if not transcript: + return None, web.json_response( + _openai_error( + "Transcription response did not include text.", + err_type="server_error", + code="invalid_transcription_response", + ), + status=502, + ) + return transcript, None + except asyncio.TimeoutError: + return None, web.json_response( + _openai_error( + "Upstream transcription timed out.", + err_type="server_error", + code="upstream_transcription_timeout", + ), + status=504, + ) + except Exception as exc: + logger.debug("Audio transcription upstream request failed", exc_info=True) + return None, web.json_response( + _openai_error( + f"Upstream transcription request failed: {exc}", + err_type="server_error", + code="upstream_transcription_failed", + ), + status=502, + ) + + async def _handle_audio_transcriptions(self, request: "web.Request") -> "web.Response": + """POST /v1/audio/transcriptions — utility STT endpoint.""" + auth_err = self._check_auth(request) + if auth_err: + return auth_err + + if not str(request.content_type or "").lower().startswith("multipart/"): + return web.json_response( + _openai_error("Expected multipart/form-data.", code="invalid_content_type"), + status=400, + ) + + request = request.clone(client_max_size=AUDIO_TRANSCRIPTION_REQUEST_MAX_BYTES) + + model = "" + file_bytes: Optional[bytes] = None + filename = "" + content_type = "" + + try: + reader = await request.multipart() + async for part in reader: + if part.name == "model": + model = (await part.text()).strip() + continue + if part.name != "file": + continue + + filename = str(part.filename or "").strip() or "audio" + content_type = str(part.headers.get("Content-Type") or getattr(part, "content_type", "") or "").strip() + if _audio_format_for_upload(filename, content_type) is None: + return web.json_response( + _openai_error( + "Unsupported audio format. Supported formats: " + + ", ".join(AUDIO_TRANSCRIPTION_FORMATS), + code="unsupported_content_type", + param="file", + ), + status=400, + ) + + chunks = bytearray() + while True: + chunk = await part.read_chunk() + if not chunk: + break + chunks.extend(chunk) + if len(chunks) > AUDIO_TRANSCRIPTION_MAX_BYTES: + return web.json_response( + _openai_error( + f"Audio file exceeds {AUDIO_TRANSCRIPTION_MAX_BYTES} bytes.", + code="audio_too_large", + param="file", + ), + status=413, + ) + file_bytes = bytes(chunks) + except Exception as exc: + return web.json_response( + _openai_error(f"Invalid multipart request: {exc}", code="invalid_multipart"), + status=400, + ) + + if not model: + return web.json_response( + _openai_error("Missing required multipart field 'model'.", code="missing_model", param="model"), + status=400, + ) + if file_bytes is None: + return web.json_response( + _openai_error("Missing required multipart file field 'file'.", code="missing_file", param="file"), + status=400, + ) + if not file_bytes: + return web.json_response( + _openai_error("Audio file is empty.", code="invalid_audio", param="file"), + status=400, + ) + + runtime, runtime_err = self._resolve_audio_transcription_runtime() + if runtime_err is not None: + return runtime_err + assert runtime is not None + + transcript, upstream_err = await self._post_audio_transcription( + runtime=runtime, + model=model, + filename=filename, + content_type=content_type, + data=file_bytes, + ) + if upstream_err is not None: + return upstream_err + + return web.json_response( + { + "text": transcript or "", + "model": model, + "provider": runtime["provider"], + } + ) + # ------------------------------------------------------------------ # HTTP Handlers # ------------------------------------------------------------------ @@ -1221,6 +1681,13 @@ async def _handle_capabilities(self, request: "web.Request") -> "web.Response": if auth_err: return auth_err + audio_caps = { + "transcription": True, + "native_model": False, + "max_bytes": AUDIO_TRANSCRIPTION_MAX_BYTES, + "formats": list(AUDIO_TRANSCRIPTION_FORMATS), + } + return web.json_response({ "object": "hermes.api_server.capabilities", "platform": "hermes-agent", @@ -1259,12 +1726,13 @@ async def _handle_capabilities(self, request: "web.Request") -> "web.Response": "jobs_admin": False, "memory_write_api": False, "skills_api": True, - "audio_api": False, + "audio_api": True, "realtime_voice": False, "session_continuity_header": "X-Hermes-Session-Id", "session_key_header": "X-Hermes-Session-Key", "cors": bool(self._cors_origins), }, + "audio": audio_caps, "endpoints": { "health": {"method": "GET", "path": "/health"}, "health_detailed": {"method": "GET", "path": "/health/detailed"}, @@ -1272,6 +1740,7 @@ async def _handle_capabilities(self, request: "web.Request") -> "web.Response": "chat_completions": {"method": "POST", "path": "/v1/chat/completions"}, "responses": {"method": "POST", "path": "/v1/responses"}, "runs": {"method": "POST", "path": "/v1/runs"}, + "audio_transcriptions": {"method": "POST", "path": "/v1/audio/transcriptions"}, "run_status": {"method": "GET", "path": "/v1/runs/{run_id}"}, "run_events": {"method": "GET", "path": "/v1/runs/{run_id}/events"}, "run_approval": {"method": "POST", "path": "/v1/runs/{run_id}/approval"}, @@ -1650,7 +2119,13 @@ async def _handle_session_chat(self, request: "web.Request") -> "web.Response": system_prompt = body.get("system_message") or body.get("instructions") if system_prompt is not None and not isinstance(system_prompt, str): return web.json_response(_openai_error("system_message must be a string", code="invalid_system_message"), status=400) + audio_err = self._reject_unsupported_audio_content(user_message, param="message") + if audio_err is not None: + return audio_err history = self._conversation_history_for_session(session_id) + audio_err = self._reject_unsupported_audio_content(history, param="conversation_history") + if audio_err is not None: + return audio_err result, usage = await self._run_agent( user_message=user_message, conversation_history=history, @@ -1694,6 +2169,13 @@ async def _handle_session_chat_stream(self, request: "web.Request") -> "web.Stre system_prompt = body.get("system_message") or body.get("instructions") if system_prompt is not None and not isinstance(system_prompt, str): return web.json_response(_openai_error("system_message must be a string", code="invalid_system_message"), status=400) + audio_err = self._reject_unsupported_audio_content(user_message, param="message") + if audio_err is not None: + return audio_err + history = self._conversation_history_for_session(session_id) + audio_err = self._reject_unsupported_audio_content(history, param="conversation_history") + if audio_err is not None: + return audio_err loop = asyncio.get_running_loop() queue: "asyncio.Queue[Optional[tuple[str, Dict[str, Any]]]]" = asyncio.Queue() @@ -1772,7 +2254,6 @@ async def _run_and_signal() -> None: try: await queue.put(_event_payload("run.started", {"user_message": {"role": "user", "content": user_message}})) await queue.put(_event_payload("message.started", {"message": {"id": message_id, "role": "assistant"}})) - history = self._conversation_history_for_session(session_id) result, usage = await self._run_agent( user_message=user_message, conversation_history=history, @@ -1877,7 +2358,7 @@ async def _handle_chat_completions(self, request: "web.Request") -> "web.Respons # Extract system message (becomes ephemeral system prompt layered ON TOP of core) system_prompt = None - conversation_messages: List[Dict[str, str]] = [] + conversation_messages: List[Dict[str, Any]] = [] for idx, msg in enumerate(messages): role = msg.get("role", "") @@ -1909,6 +2390,9 @@ async def _handle_chat_completions(self, request: "web.Request") -> "web.Respons {"error": {"message": "No user message found in messages", "type": "invalid_request_error"}}, status=400, ) + audio_err = self._reject_unsupported_audio_content(conversation_messages, param="messages") + if audio_err is not None: + return audio_err # Allow caller to scope long-term memory (e.g. Honcho) with a # stable per-channel identifier via X-Hermes-Session-Key. This @@ -1965,6 +2449,9 @@ async def _handle_chat_completions(self, request: "web.Request") -> "web.Respons except Exception as e: logger.warning("Failed to load session history for %s: %s", session_id, e) history = [] + audio_err = self._reject_unsupported_audio_content(history, param="conversation_history") + if audio_err is not None: + return audio_err else: # Derive a stable session ID from the conversation fingerprint so # that consecutive messages from the same Open WebUI (or similar) @@ -3061,7 +3548,8 @@ async def _handle_responses(self, request: "web.Request") -> "web.Response": entry_content = _normalize_multimodal_content(entry["content"]) except ValueError as exc: return _multimodal_validation_error(exc, param=f"conversation_history[{i}].content") - conversation_history.append({"role": str(entry["role"]), "content": entry_content}) + entry_role = str(entry["role"]) + conversation_history.append({"role": entry_role, "content": entry_content}) if previous_response_id: logger.debug("Both conversation_history and previous_response_id provided; using conversation_history") @@ -3075,6 +3563,9 @@ async def _handle_responses(self, request: "web.Request") -> "web.Response": # If no instructions provided, carry forward from previous if instructions is None: instructions = stored.get("instructions") + audio_err = self._reject_unsupported_audio_content(conversation_history, param="previous_response_id") + if audio_err is not None: + return audio_err # Append new input messages to history (all but the last become history) for msg in input_messages[:-1]: @@ -3084,6 +3575,12 @@ async def _handle_responses(self, request: "web.Request") -> "web.Response": user_message: Any = input_messages[-1].get("content", "") if input_messages else "" if not _content_has_visible_payload(user_message): return web.json_response(_openai_error("No user message found in input"), status=400) + audio_err = self._reject_unsupported_audio_content( + conversation_history + [{"role": "user", "content": user_message}], + param="input", + ) + if audio_err is not None: + return audio_err # Truncation support if body.get("truncation") == "auto" and len(conversation_history) > 100: @@ -3799,8 +4296,8 @@ def _bind_api_server_session( async def _run_agent( self, - user_message: str, - conversation_history: List[Dict[str, str]], + user_message: Any, + conversation_history: List[Dict[str, Any]], ephemeral_system_prompt: Optional[str] = None, session_id: Optional[str] = None, stream_delta_callback=None, @@ -3964,8 +4461,25 @@ async def _handle_runs(self, request: "web.Request") -> "web.Response": if not raw_input: return web.json_response(_openai_error("Missing 'input' field"), status=400) - user_message = raw_input if isinstance(raw_input, str) else (raw_input[-1].get("content", "") if isinstance(raw_input, list) else "") - if not user_message: + input_messages: List[Dict[str, Any]] = [] + if isinstance(raw_input, str): + input_messages = [{"role": "user", "content": raw_input}] + elif isinstance(raw_input, list): + for idx, item in enumerate(raw_input): + if isinstance(item, str): + input_messages.append({"role": "user", "content": item}) + elif isinstance(item, dict): + role = item.get("role", "user") + try: + content = _normalize_multimodal_content(item.get("content", "")) + except ValueError as exc: + return _multimodal_validation_error(exc, param=f"input[{idx}].content") + input_messages.append({"role": role, "content": content}) + else: + return web.json_response(_openai_error("'input' must be a string or array"), status=400) + + user_message: Any = input_messages[-1].get("content", "") if input_messages else "" + if not _content_has_visible_payload(user_message): return web.json_response(_openai_error("No user message found in input"), status=400) instructions = body.get("instructions") @@ -3973,7 +4487,7 @@ async def _handle_runs(self, request: "web.Request") -> "web.Response": # Accept explicit conversation_history from the request body. # Precedence: explicit conversation_history > previous_response_id. - conversation_history: List[Dict[str, str]] = [] + conversation_history: List[Dict[str, Any]] = [] raw_history = body.get("conversation_history") if raw_history: if not isinstance(raw_history, list): @@ -3987,7 +4501,12 @@ async def _handle_runs(self, request: "web.Request") -> "web.Response": _openai_error(f"conversation_history[{i}] must have 'role' and 'content' fields"), status=400, ) - conversation_history.append({"role": str(entry["role"]), "content": str(entry["content"])}) + try: + entry_content = _normalize_multimodal_content(entry["content"]) + except ValueError as exc: + return _multimodal_validation_error(exc, param=f"conversation_history[{i}].content") + entry_role = str(entry["role"]) + conversation_history.append({"role": entry_role, "content": entry_content}) if previous_response_id: logger.debug("Both conversation_history and previous_response_id provided; using conversation_history") @@ -3999,21 +4518,22 @@ async def _handle_runs(self, request: "web.Request") -> "web.Response": stored_session_id = stored.get("session_id") if instructions is None: instructions = stored.get("instructions") + audio_err = self._reject_unsupported_audio_content(conversation_history, param="previous_response_id") + if audio_err is not None: + return audio_err # When input is a multi-message array, extract all but the last # message as conversation history (the last becomes user_message). # Only fires when no explicit history was provided. - if not conversation_history and isinstance(raw_input, list) and len(raw_input) > 1: - for msg in raw_input[:-1]: - if isinstance(msg, dict) and msg.get("role") and msg.get("content"): - content = msg["content"] - if isinstance(content, list): - # Flatten multi-part content blocks to text - content = " ".join( - part.get("text", "") for part in content - if isinstance(part, dict) and part.get("type") == "text" - ) - conversation_history.append({"role": msg["role"], "content": str(content)}) + if not conversation_history and len(input_messages) > 1: + conversation_history.extend(input_messages[:-1]) + + audio_err = self._reject_unsupported_audio_content( + conversation_history + [{"role": "user", "content": user_message}], + param="input", + ) + if audio_err is not None: + return audio_err run_id = f"run_{uuid.uuid4().hex}" session_id = body.get("session_id") or stored_session_id or run_id @@ -4489,6 +5009,7 @@ async def connect(self, *, is_reconnect: bool = False) -> bool: self._app.router.add_get("/v1/capabilities", self._handle_capabilities) self._app.router.add_get("/v1/skills", self._handle_skills) self._app.router.add_get("/v1/toolsets", self._handle_toolsets) + self._app.router.add_post("/v1/audio/transcriptions", self._handle_audio_transcriptions) # Session/client control surface (thin wrappers over SessionDB + _run_agent) self._app.router.add_get("/api/sessions", self._handle_list_sessions) self._app.router.add_post("/api/sessions", self._handle_create_session) diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index a30bdcc3a17c..5f02aecf026c 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -1431,6 +1431,7 @@ def resolve_runtime_provider( explicit_api_key: Optional[str] = None, explicit_base_url: Optional[str] = None, target_model: Optional[str] = None, + allow_auto_codex_fallback: bool = True, ) -> Dict[str, Any]: """Resolve runtime provider credentials for agent execution. @@ -1441,6 +1442,11 @@ def resolve_runtime_provider( api_mode is derived from the model they are switching TO, not the stale persisted default. Other callers can leave it None to preserve existing behavior (api_mode derived from config). + + allow_auto_codex_fallback: When False, an auto-detected Codex provider with + invalid credentials raises instead of falling through to another provider. + The normal chat path keeps the historical fallback behavior; utility + endpoints that must stay inside the Codex auth boundary can opt out. """ requested_provider = resolve_requested_provider(requested) @@ -1662,7 +1668,7 @@ def resolve_runtime_provider( "requested_provider": requested_provider, } except AuthError: - if requested_provider != "auto": + if requested_provider != "auto" or not allow_auto_codex_fallback: raise # Auto-detected Codex but credentials are stale/revoked — # fall through to env-var providers (e.g. OpenRouter). diff --git a/run_agent.py b/run_agent.py index bca815da3c69..cc997eadac1c 100644 --- a/run_agent.py +++ b/run_agent.py @@ -1690,13 +1690,15 @@ def _flush_messages_to_session_db(self, messages: List[Dict], conversation_histo if _is_multimodal_tool_result(content): content = _multimodal_text_summary(content) elif isinstance(content, list): - # List of OpenAI-style content parts: strip images, keep text. + # List of OpenAI-style content parts: strip media, keep text. _txt = [] for p in content: if isinstance(p, dict) and p.get("type") == "text": _txt.append(str(p.get("text", ""))) elif isinstance(p, dict) and p.get("type") in {"image", "image_url", "input_image"}: _txt.append("[screenshot]") + elif isinstance(p, dict) and p.get("type") in {"audio", "input_audio"}: + _txt.append("[audio]") content = "\n".join(_txt) if _txt else None tool_calls_data = None if hasattr(msg, "tool_calls") and isinstance(msg.tool_calls, list) and msg.tool_calls: diff --git a/tests/agent/test_models_dev.py b/tests/agent/test_models_dev.py index b4bbbf753dfa..4c8b984ed850 100644 --- a/tests/agent/test_models_dev.py +++ b/tests/agent/test_models_dev.py @@ -401,3 +401,8 @@ def test_model_not_found_returns_none(self): with patch("agent.models_dev.fetch_models_dev", return_value=CAPS_REGISTRY): caps = get_model_capabilities("anthropic", "nonexistent-model") assert caps is None + + def test_provider_not_found_returns_none(self): + with patch("agent.models_dev.fetch_models_dev", return_value=CAPS_REGISTRY): + caps = get_model_capabilities("nonexistent-provider", "gemma-4-31b-it") + assert caps is None diff --git a/tests/gateway/test_api_server.py b/tests/gateway/test_api_server.py index c0a2f52d6c70..10438402dce1 100644 --- a/tests/gateway/test_api_server.py +++ b/tests/gateway/test_api_server.py @@ -534,8 +534,10 @@ def _create_app(adapter: APIServerAdapter) -> web.Application: app.router.add_get("/v1/capabilities", adapter._handle_capabilities) app.router.add_get("/v1/skills", adapter._handle_skills) app.router.add_get("/v1/toolsets", adapter._handle_toolsets) + app.router.add_post("/v1/audio/transcriptions", adapter._handle_audio_transcriptions) app.router.add_post("/v1/chat/completions", adapter._handle_chat_completions) app.router.add_post("/v1/responses", adapter._handle_responses) + app.router.add_post("/v1/runs", adapter._handle_runs) app.router.add_get("/v1/responses/{response_id}", adapter._handle_get_response) app.router.add_delete("/v1/responses/{response_id}", adapter._handle_delete_response) return app @@ -798,11 +800,35 @@ async def test_capabilities_advertises_plugin_safe_contract(self, adapter): assert data["features"]["chat_completions"] is True assert data["features"]["run_status"] is True assert data["features"]["run_events_sse"] is True + assert data["features"]["audio_api"] is True + assert data["audio"]["transcription"] is True + assert data["audio"]["native_model"] is False + assert data["audio"]["max_bytes"] == 26214400 + assert "ogg" in data["audio"]["formats"] assert data["features"]["session_continuity_header"] == "X-Hermes-Session-Id" assert data["endpoints"]["run_status"]["path"] == "/v1/runs/{run_id}" + assert data["endpoints"]["audio_transcriptions"] == { + "method": "POST", + "path": "/v1/audio/transcriptions", + } assert data["endpoints"]["skills"] == {"method": "GET", "path": "/v1/skills"} assert data["endpoints"]["toolsets"] == {"method": "GET", "path": "/v1/toolsets"} + @pytest.mark.asyncio + async def test_capabilities_never_reports_native_audio_from_runtime(self, adapter): + app = _create_app(adapter) + with patch("gateway.run._load_gateway_config", return_value={}), \ + patch("gateway.run._resolve_runtime_agent_kwargs", return_value={"provider": "openai"}), \ + patch("gateway.run._resolve_gateway_model", return_value="gpt-audio"): + async with TestClient(TestServer(app)) as cli: + resp = await cli.get("/v1/capabilities") + assert resp.status == 200 + data = await resp.json() + + assert data["features"]["audio_api"] is True + assert data["audio"]["transcription"] is True + assert data["audio"]["native_model"] is False + @pytest.mark.asyncio async def test_capabilities_requires_auth_when_key_configured(self, auth_adapter): app = _create_app(auth_adapter) @@ -819,6 +845,69 @@ async def test_capabilities_requires_auth_when_key_configured(self, auth_adapter assert data["auth"]["required"] is True +# --------------------------------------------------------------------------- +# /v1/runs audio preflight +# --------------------------------------------------------------------------- + + +class TestRunsAudioPreflight: + @pytest.mark.asyncio + async def test_runs_rejects_unsupported_audio_before_run_state(self, adapter): + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.post( + "/v1/runs", + json={ + "input": [ + { + "role": "user", + "content": [ + {"type": "input_audio", "input_audio": {"data": "ZmFrZQ==", "format": "ogg"}}, + ], + } + ], + }, + ) + data = await resp.json() + + assert resp.status == 400 + assert data["error"]["code"] == "unsupported_audio_input" + assert adapter._run_streams == {} + assert adapter._run_statuses == {} + + @pytest.mark.asyncio + async def test_runs_rejects_previous_response_assistant_audio_before_run_state(self, adapter): + adapter._response_store.put( + "resp_prev", + { + "conversation_history": [ + { + "role": "assistant", + "content": [ + {"type": "input_audio", "input_audio": {"data": "ZmFrZQ==", "format": "ogg"}}, + ], + }, + ], + "session_id": "session_prev", + "instructions": None, + }, + ) + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + resp = await cli.post( + "/v1/runs", + json={"previous_response_id": "resp_prev", "input": "next"}, + ) + data = await resp.json() + + assert resp.status == 400 + assert data["error"]["code"] == "unsupported_audio_input" + assert data["error"]["param"] == "previous_response_id" + assert adapter._run_streams == {} + assert adapter._run_statuses == {} + + # --------------------------------------------------------------------------- # /v1/skills and /v1/toolsets endpoints # --------------------------------------------------------------------------- diff --git a/tests/gateway/test_api_server_audio_transcriptions.py b/tests/gateway/test_api_server_audio_transcriptions.py new file mode 100644 index 000000000000..e216fc06f93d --- /dev/null +++ b/tests/gateway/test_api_server_audio_transcriptions.py @@ -0,0 +1,552 @@ +"""Tests for the API-server STT utility endpoint.""" + +import base64 +import json +from unittest.mock import AsyncMock, patch + +import pytest +from aiohttp import FormData, web +from aiohttp.test_utils import TestClient, TestServer + +from gateway.config import PlatformConfig +from gateway.platforms import api_server as api_server_mod +from gateway.platforms.api_server import APIServerAdapter +from hermes_cli.auth import AuthError + + +def _make_adapter(api_key: str = "") -> APIServerAdapter: + extra = {"key": api_key} if api_key else {} + return APIServerAdapter(PlatformConfig(enabled=True, extra=extra)) + + +def _create_app(adapter: APIServerAdapter) -> web.Application: + app = web.Application() + app.router.add_post("/v1/audio/transcriptions", adapter._handle_audio_transcriptions) + app.router.add_post("/v1/chat/completions", adapter._handle_chat_completions) + return app + + +def _multipart( + *, + model: str | None = "whisper-1", + audio: bytes | None = b"fake-audio", + filename: str = "voice.ogg", + content_type: str = "audio/ogg", + file_field: str = "file", +) -> FormData: + form = FormData() + if model is not None: + form.add_field("model", model) + if audio is not None: + form.add_field(file_field, audio, filename=filename, content_type=content_type) + return form + + +def _jwt_with_account_id(account_id: str) -> str: + def encode(payload): + raw = json.dumps(payload, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + return ".".join( + ( + encode({"alg": "none"}), + encode({"https://api.openai.com/auth": {"chatgpt_account_id": account_id}}), + "signature", + ) + ) + + +class _FakeFormData: + def __init__(self): + self.fields = [] + + def add_field(self, name, value, **kwargs): + self.fields.append({"name": name, "value": value, **kwargs}) + + +def _install_fake_upstream(monkeypatch, *, status: int = 200, body: str = '{"text":"hello"}') -> dict: + captured = {} + + class FakeResponse: + def __init__(self): + self.status = status + + async def __aenter__(self): + return self + + async def __aexit__(self, *_exc): + return None + + async def text(self): + return body + + class FakeClientSession: + def __init__(self, *, timeout=None, **_kwargs): + captured["timeout"] = timeout + + async def __aenter__(self): + return self + + async def __aexit__(self, *_exc): + return None + + def post(self, url, *, data=None, headers=None): + captured["url"] = url + captured["fields"] = list(getattr(data, "fields", [])) + captured["headers"] = dict(headers or {}) + return FakeResponse() + + monkeypatch.setattr(api_server_mod, "ClientSession", FakeClientSession) + monkeypatch.setattr(api_server_mod, "FormData", _FakeFormData) + return captured + + +def test_codex_transcription_url_uses_backend_api_sibling(): + assert ( + api_server_mod._codex_transcription_upstream_url("https://chatgpt.com/backend-api/codex") + == "https://chatgpt.com/backend-api/transcribe" + ) + assert ( + api_server_mod._codex_transcription_upstream_url("https://chatgpt.example/backend-api/codex") + == "https://chatgpt.example/backend-api/transcribe" + ) + + +@pytest.mark.asyncio +async def test_transcription_succeeds_with_codex_auth(): + adapter = _make_adapter() + captured = {} + + async def fake_post(**kwargs): + captured.update(kwargs) + return "hello world", None + + app = _create_app(adapter) + with patch("hermes_cli.runtime_provider.resolve_requested_provider", return_value="openai-codex"), \ + patch( + "hermes_cli.auth.resolve_codex_runtime_credentials", + return_value={ + "provider": "openai-codex", + "base_url": "https://chatgpt.example/backend-api/codex", + "api_key": "codex-token", + }, + ), \ + patch.object(adapter, "_post_audio_transcription", side_effect=fake_post), \ + patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + async with TestClient(TestServer(app)) as cli: + resp = await cli.post("/v1/audio/transcriptions", data=_multipart(model="gpt-4o-transcribe")) + body = await resp.json() + + assert resp.status == 200 + assert body == { + "text": "hello world", + "model": "gpt-4o-transcribe", + "provider": "openai-codex", + } + assert captured["runtime"]["provider"] == "openai-codex" + assert captured["runtime"]["base_url"] == "https://chatgpt.example/backend-api/codex" + assert captured["runtime"]["api_key"] == "codex-token" + assert captured["model"] == "gpt-4o-transcribe" + assert captured["data"] == b"fake-audio" + mock_run.assert_not_called() + assert adapter._session_db is None + + +@pytest.mark.asyncio +async def test_post_audio_transcription_uses_codex_transcribe_request(monkeypatch): + adapter = _make_adapter() + captured = _install_fake_upstream(monkeypatch, body='{"text":"hello"}') + token = _jwt_with_account_id("acct_123") + + transcript, err = await adapter._post_audio_transcription( + runtime={ + "provider": "openai-codex", + "base_url": "https://chatgpt.com/backend-api/codex", + "api_key": token, + }, + model="should-not-forward", + filename="voice.ogg", + content_type="audio/ogg", + data=b"fake-audio", + ) + + assert err is None + assert transcript == "hello" + assert captured["url"] == "https://chatgpt.com/backend-api/transcribe" + assert captured["headers"]["Authorization"] == f"Bearer {token}" + assert captured["headers"]["Accept"] == "application/json" + assert captured["headers"]["Origin"] == "https://chatgpt.com" + assert captured["headers"]["Referer"] == "https://chatgpt.com/" + assert captured["headers"]["originator"] == "codex_cli_rs" + assert captured["headers"]["ChatGPT-Account-ID"] == "acct_123" + assert [field["name"] for field in captured["fields"]] == ["file"] + assert captured["fields"][0]["filename"] == "voice.ogg" + assert captured["fields"][0]["content_type"] == "audio/ogg" + assert captured["fields"][0]["value"] == b"fake-audio" + + +@pytest.mark.asyncio +async def test_codex_transcription_403_html_error_is_sanitized(monkeypatch): + adapter = _make_adapter() + token = _jwt_with_account_id("acct_123") + captured = _install_fake_upstream( + monkeypatch, + status=403, + body="cf challenge super-secret-token asset_pointer ptr_123", + ) + + transcript, err = await adapter._post_audio_transcription( + runtime={ + "provider": "openai-codex", + "base_url": "https://chatgpt.com/backend-api/codex", + "api_key": token, + }, + model="should-not-forward", + filename="voice.ogg", + content_type="audio/ogg", + data=b"fake-audio", + ) + + assert transcript is None + assert err is not None + assert err.status == 403 + body = json.loads(err.text) + message = body["error"]["message"] + assert message == "ChatGPT transcription rejected request." + assert "super-secret-token" not in message + assert "asset_pointer" not in message + assert "