From aec8f78048af6800b462edd73ad442323dd99eb3 Mon Sep 17 00:00:00 2001 From: qbit-mirror-bot Date: Thu, 2 Jul 2026 11:42:30 +0000 Subject: [PATCH] feat(api-server): inline MEDIA: image tags as data URLs for remote frontends (#2696 salvage) --- gateway/platforms/api_server.py | 173 +++++++++++++----- scripts/release.py | 2 + .../test_api_server_media_data_urls.py | 77 ++++++++ 3 files changed, 203 insertions(+), 49 deletions(-) create mode 100644 tests/gateway/test_api_server_media_data_urls.py diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index ea91aea4329b..fa2f14267bbb 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -572,6 +572,55 @@ async def cors_middleware(request, handler): cors_middleware = None # type: ignore[assignment] +_MEDIA_IMG_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"} +_MEDIA_MIME = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".bmp": "image/bmp", +} +_MEDIA_TAG_RE = re.compile( + r"[`\"']?MEDIA:\s*(`[^`\n]+`|\"[^\"\n]+\"|'[^'\n]+'|\S+)[`\"']?" +) +_MEDIA_DATA_URL_MAX_BYTES = 5 * 1024 * 1024 # skip images larger than 5MB + + +def _resolve_media_to_data_urls(text: str) -> str: + """Replace ``MEDIA:`` image tags with inline base64 data URLs. + + Remote OpenAI-compatible frontends can't read local file paths, so + ``MEDIA:`` tags referencing images on the server are useless to them. + Inline small local images as markdown data URLs; non-image or unreadable + paths are left untouched. + """ + if not text or "MEDIA:" not in text: + return text + import base64 + + def _to_data_url(path_str: str) -> Optional[str]: + p = Path(path_str.strip().strip("`\"'")).expanduser() + suffix = p.suffix.lower() + if suffix not in _MEDIA_IMG_EXT: + return None + try: + if not p.is_file() or p.stat().st_size > _MEDIA_DATA_URL_MAX_BYTES: + return None + b64 = base64.b64encode(p.read_bytes()).decode() + except OSError: + return None + return f"![image](data:{_MEDIA_MIME[suffix]};base64,{b64})" + + def _repl(m: "re.Match[str]") -> str: + return _to_data_url(m.group(1)) or m.group(0) + + try: + return _MEDIA_TAG_RE.sub(_repl, text) + except Exception: + return text + + def _redact_api_error_text(value: Any, *, limit: int | None = None) -> str: """Redact API-bound error text before it crosses the HTTP boundary.""" redacted = redact_sensitive_text(str(value), force=True) @@ -1108,6 +1157,18 @@ def _create_agent( reasoning_config = GatewayRunner._load_reasoning_config() model = _resolve_gateway_model() + # When the primary provider's auth fails (expired token / 429 quota + # cap), _resolve_runtime_agent_kwargs() falls through to the fallback + # provider chain, whose runtime dict carries its own ``model`` key. + # Pop it and let it override the config model, mirroring the native + # gateway path (_resolve_session_agent_runtime in run.py). Otherwise + # the explicit ``model=model`` below collides with the ``**runtime_kwargs`` + # spread → "got multiple values for keyword argument 'model'", 500ing + # every /v1/chat/completions request while a fallback is active. + runtime_model = runtime_kwargs.pop("model", None) + if runtime_model: + model = runtime_model + user_config = _load_gateway_config() enabled_toolsets = sorted(_get_platform_tools(user_config, "api_server")) @@ -1153,8 +1214,12 @@ async def _handle_health_detailed(self, request: "web.Request") -> "web.Response Returns gateway state, connected platforms, PID, and uptime so the dashboard can display full status without needing a shared PID file or - /proc access. No authentication required. + /proc access. Requires the same Bearer auth as other API routes. """ + auth_err = self._check_auth(request) + if auth_err: + return auth_err + from gateway.status import ( derive_gateway_busy, derive_gateway_drainable, @@ -1659,7 +1724,7 @@ async def _handle_session_chat(self, request: "web.Request") -> "web.Response": gateway_session_key=gateway_session_key, ) effective_session_id = result.get("session_id") if isinstance(result, dict) else session_id - final_response = result.get("final_response", "") if isinstance(result, dict) else "" + final_response = _resolve_media_to_data_urls(result.get("final_response", "") if isinstance(result, dict) else "") headers = {"X-Hermes-Session-Id": effective_session_id or session_id} if gateway_session_key: headers["X-Hermes-Session-Key"] = gateway_session_key @@ -1749,7 +1814,7 @@ async def _run_and_signal() -> None: tool_progress_callback=_tool_progress, gateway_session_key=gateway_session_key, ) - final_response = result.get("final_response", "") if isinstance(result, dict) else "" + final_response = _resolve_media_to_data_urls(result.get("final_response", "") if isinstance(result, dict) else "") effective_session_id = result.get("session_id", session_id) if isinstance(result, dict) else session_id turn_messages = self._turn_transcript_messages(history, user_message, result) if isinstance(result, dict) else [] await queue.put(_event_payload("assistant.completed", { @@ -2071,7 +2136,7 @@ async def _compute_completion(): status=500, ) - final_response = result.get("final_response") or "" + final_response = _resolve_media_to_data_urls(result.get("final_response") or "") is_partial = bool(result.get("partial")) is_failed = bool(result.get("failed")) completed = bool(result.get("completed", True)) @@ -3170,7 +3235,7 @@ async def _compute_response(): status=500, ) - final_response = result.get("final_response", "") + final_response = _resolve_media_to_data_urls(result.get("final_response", "")) if not final_response: final_response = _redact_api_error_text(result.get("error", "(No response generated)")) @@ -3982,7 +4047,12 @@ async def _handle_runs(self, request: "web.Request") -> "web.Response": run_id = f"run_{uuid.uuid4().hex}" session_id = body.get("session_id") or stored_session_id or run_id - approval_session_key = gateway_session_key or session_id or run_id + # Approval queues gate host-side tool execution and must be isolated + # per API run. Client-provided session IDs and memory session keys are + # conversation/memory scopes, not authorization namespaces: multiple + # concurrent runs can intentionally share them, and resolving an + # approval for one run must not unblock another run's dangerous command. + approval_session_key = run_id ephemeral_system_prompt = instructions loop = asyncio.get_running_loop() q: "asyncio.Queue[Optional[Dict]]" = asyncio.Queue() @@ -4437,12 +4507,60 @@ async def _sweep_orphaned_runs(self) -> None: # BasePlatformAdapter interface # ------------------------------------------------------------------ + def _api_key_passes_startup_guard(self) -> bool: + """Return True when API_SERVER_KEY is present and strong enough to start.""" + if not self._api_key: + logger.error( + "[%s] Refusing to start: API_SERVER_KEY is required for the API server, " + "including loopback-only binds on %s.", + self.name, self._host, + ) + return False + + try: + from hermes_cli.auth import has_usable_secret + if not has_usable_secret(self._api_key, min_length=16): + logger.error( + "[%s] Refusing to start: API_SERVER_KEY is a " + "placeholder or too short (<16 chars). This endpoint " + "dispatches terminal-capable agent work — a guessable " + "key is remote code execution. Generate a strong secret " + "(e.g. `openssl rand -hex 32`) and set API_SERVER_KEY " + "before starting the API server on %s.", + self.name, self._host, + ) + return False + except ImportError: + pass + return True + + def _port_is_available(self) -> bool: + """Return True when the configured listen port is free.""" + try: + with _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) as _s: + _s.settimeout(1) + _s.connect(('127.0.0.1', self._port)) + logger.error( + "[%s] Port %d already in use. Set a different port in config.yaml: " + "platforms.api_server.port", + self.name, self._port, + ) + return False + except (ConnectionRefusedError, OSError): + return True + async def connect(self, *, is_reconnect: bool = False) -> bool: """Start the aiohttp web server.""" if not AIOHTTP_AVAILABLE: logger.warning("[%s] aiohttp not installed", self.name) return False + if not self._api_key_passes_startup_guard(): + return False + + if not self._port_is_available(): + return False + try: mws = [mw for mw in (cors_middleware, body_limit_middleware, security_headers_middleware) if mw is not None] self._app = web.Application(middlewares=mws, client_max_size=MAX_REQUEST_BYTES) @@ -4503,39 +4621,6 @@ async def connect(self, *, is_reconnect: bool = False) -> bool: if hasattr(sweep_task, "add_done_callback"): sweep_task.add_done_callback(self._background_tasks.discard) - # Refuse to start without authentication. The API server can - # dispatch terminal-capable agent work, so every deployment needs - # an explicit API_SERVER_KEY regardless of bind address. - if not self._api_key: - logger.error( - "[%s] Refusing to start: API_SERVER_KEY is required for the API server, " - "including loopback-only binds on %s.", - self.name, self._host, - ) - return False - - # Refuse to start network-accessible with a placeholder or weak key. - # Ported from openclaw/openclaw#64586; entropy floor raised to 16 in - # the June 2026 hermes-0day hardening (an 8-char key dispatching - # terminal-capable agent work on a public bind is brute-forceable). - if is_network_accessible(self._host) and self._api_key: - try: - from hermes_cli.auth import has_usable_secret - if not has_usable_secret(self._api_key, min_length=16): - logger.error( - "[%s] Refusing to start: API_SERVER_KEY is a " - "placeholder or too short (<16 chars) for a " - "network-accessible bind. This endpoint dispatches " - "terminal-capable agent work — a guessable key is " - "remote code execution. Generate a strong secret " - "(e.g. `openssl rand -hex 32`) and set " - "API_SERVER_KEY before exposing it on %s.", - self.name, self._host, - ) - return False - except ImportError: - pass - # Loud warning when a network-accessible API server runs against an # unsandboxed local terminal backend. The API server can drive the # agent's terminal/file tools as the host user; on a public bind @@ -4564,16 +4649,6 @@ async def connect(self, *, is_reconnect: bool = False) -> bool: self.name, self._host, ) - # Port conflict detection — fail fast if port is already in use - try: - with _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) as _s: - _s.settimeout(1) - _s.connect(('127.0.0.1', self._port)) - logger.error('[%s] Port %d already in use. Set a different port in config.yaml: platforms.api_server.port', self.name, self._port) - return False - except (ConnectionRefusedError, OSError): - pass # port is free - self._runner = web.AppRunner(self._app) await self._runner.setup() self._site = web.TCPSite(self._runner, self._host, self._port) diff --git a/scripts/release.py b/scripts/release.py index 90ef9e230d67..1a580983e736 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -45,6 +45,7 @@ # Auto-extracted from noreply emails + manual overrides AUTHOR_MAP = { + "tarunravi@gmail.com": "tarunravi", # PR #2696 salvage (api-server: inline MEDIA: image tags as base64 data URLs in final responses so remote OpenAI-compatible frontends can render server-local screenshots; the PR's tool-progress-streaming and SSE-sentinel pieces were independently superseded on main) "aqdrgg19@gmail.com": "VolodymyrBg", # PR #2861 salvage (webhook: drop the unused full request payload from retained _delivery_info entries — up to ~1MB dead weight per delivery for the 1h idempotency TTL) "ohyes9711@gmail.com": "CharmingGroot", # PR #2794 salvage (email: guard msg_data[0][1] against malformed IMAP fetch structures so one bad response can't abort the batch and permanently lose seen-marked messages; Message-ID domain falls back to localhost when EMAIL_ADDRESS lacks '@') "sahibzada@fastino.ai": "sahibzada-allahyar", # PR #39227 salvage (desktop: configured terminal.cwd overrides a stale remembered workspace-cwd localStorage value when no session is active; #38855) @@ -919,6 +920,7 @@ "fr@tecompanytea.com": "ifrederico", "cdanis@gmail.com": "cdanis", "samherring99@gmail.com": "samherring99", + "sampiyonyus@gmail.com": "crazywriter1", "desaiaum08@gmail.com": "Aum08Desai", "shannon.sands.1979@gmail.com": "shannonsands", "shannon@nousresearch.com": "shannonsands", diff --git a/tests/gateway/test_api_server_media_data_urls.py b/tests/gateway/test_api_server_media_data_urls.py new file mode 100644 index 000000000000..960f4b194f84 --- /dev/null +++ b/tests/gateway/test_api_server_media_data_urls.py @@ -0,0 +1,77 @@ +"""MEDIA: tag → base64 data-URL resolution for the API server (salvage of #2696). + +Remote OpenAI-compatible frontends can't read local file paths, so +``MEDIA:`` image tags in final responses are inlined as markdown +data URLs before crossing the HTTP boundary. +""" + +import base64 +import unittest + +import pytest + +pytest.importorskip("aiohttp") + +from gateway.platforms.api_server import _resolve_media_to_data_urls # noqa: E402 + +# 1x1 transparent PNG +_PNG_BYTES = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQAB" + "h6FO1AAAAABJRU5ErkJggg==" +) + + +class TestResolveMediaToDataUrls(unittest.TestCase): + def _write_png(self, tmpdir_name="hermes_media_test"): + import tempfile + from pathlib import Path + + d = Path(tempfile.mkdtemp(prefix=tmpdir_name)) + p = d / "shot.png" + p.write_bytes(_PNG_BYTES) + return p + + def test_media_tag_inlined(self): + p = self._write_png() + out = _resolve_media_to_data_urls(f"Here you go: MEDIA:{p}") + self.assertIn("data:image/png;base64,", out) + self.assertNotIn("MEDIA:", out) + + def test_backtick_wrapped_tag(self): + p = self._write_png() + out = _resolve_media_to_data_urls(f"See `MEDIA:{p}` above") + self.assertIn("data:image/png;base64,", out) + + def test_missing_file_left_untouched(self): + text = "MEDIA:/nonexistent/path/shot.png" + self.assertEqual(_resolve_media_to_data_urls(text), text) + + def test_non_image_left_untouched(self): + text = "MEDIA:/tmp/archive.zip" + self.assertEqual(_resolve_media_to_data_urls(text), text) + + def test_text_without_media_passthrough(self): + self.assertEqual(_resolve_media_to_data_urls("plain text"), "plain text") + self.assertEqual(_resolve_media_to_data_urls(""), "") + + def test_oversized_image_skipped(self): + from gateway.platforms import api_server as mod + + p = self._write_png() + orig = mod._MEDIA_DATA_URL_MAX_BYTES + mod._MEDIA_DATA_URL_MAX_BYTES = 1 + try: + text = f"MEDIA:{p}" + self.assertEqual(_resolve_media_to_data_urls(text), text) + finally: + mod._MEDIA_DATA_URL_MAX_BYTES = orig + + def test_multiple_tags(self): + p1 = self._write_png() + p2 = self._write_png("hermes_media_test2") + out = _resolve_media_to_data_urls(f"MEDIA:{p1}\nand MEDIA:{p2}") + self.assertEqual(out.count("data:image/png;base64,"), 2) + + +if __name__ == "__main__": + unittest.main()