feat(api-server): inline MEDIA: image tags as data URLs for remote frontends (#2696 salvage)

gateway/platforms/api_server.py

@@ -572,6 +572,55 @@ async def cors_middleware(request, handler):
     cors_middleware = None  # type: ignore[assignment]
 
 
+_MEDIA_IMG_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"}
+_MEDIA_MIME = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".gif": "image/gif",
+    ".webp": "image/webp",
+    ".bmp": "image/bmp",
+}
+_MEDIA_TAG_RE = re.compile(
+    r"[`\"']?MEDIA:\s*(`[^`\n]+`|\"[^\"\n]+\"|'[^'\n]+'|\S+)[`\"']?"
+)
+_MEDIA_DATA_URL_MAX_BYTES = 5 * 1024 * 1024  # skip images larger than 5MB
+
+
+def _resolve_media_to_data_urls(text: str) -> str:
+    """Replace ``MEDIA:<path>`` image tags with inline base64 data URLs.
+
+    Remote OpenAI-compatible frontends can't read local file paths, so
+    ``MEDIA:`` tags referencing images on the server are useless to them.
+    Inline small local images as markdown data URLs; non-image or unreadable
+    paths are left untouched.
+    """
+    if not text or "MEDIA:" not in text:
+        return text
+    import base64
+
+    def _to_data_url(path_str: str) -> Optional[str]:
+        p = Path(path_str.strip().strip("`\"'")).expanduser()
+        suffix = p.suffix.lower()
+        if suffix not in _MEDIA_IMG_EXT:
+            return None
+        try:
+            if not p.is_file() or p.stat().st_size > _MEDIA_DATA_URL_MAX_BYTES:
+                return None
+            b64 = base64.b64encode(p.read_bytes()).decode()
+        except OSError:
+            return None
+        return f"![image](data:{_MEDIA_MIME[suffix]};base64,{b64})"
+
+    def _repl(m: "re.Match[str]") -> str:
+        return _to_data_url(m.group(1)) or m.group(0)
+
+    try:
+        return _MEDIA_TAG_RE.sub(_repl, text)
+    except Exception:
+        return text
+
+
 def _redact_api_error_text(value: Any, *, limit: int | None = None) -> str:
     """Redact API-bound error text before it crosses the HTTP boundary."""
     redacted = redact_sensitive_text(str(value), force=True)
@@ -1675,7 +1724,7 @@ async def _handle_session_chat(self, request: "web.Request") -> "web.Response":
             gateway_session_key=gateway_session_key,
         )
         effective_session_id = result.get("session_id") if isinstance(result, dict) else session_id
-        final_response = result.get("final_response", "") if isinstance(result, dict) else ""
+        final_response = _resolve_media_to_data_urls(result.get("final_response", "") if isinstance(result, dict) else "")
         headers = {"X-Hermes-Session-Id": effective_session_id or session_id}
         if gateway_session_key:
             headers["X-Hermes-Session-Key"] = gateway_session_key
@@ -1765,7 +1814,7 @@ async def _run_and_signal() -> None:
                     tool_progress_callback=_tool_progress,
                     gateway_session_key=gateway_session_key,
                 )
-                final_response = result.get("final_response", "") if isinstance(result, dict) else ""
+                final_response = _resolve_media_to_data_urls(result.get("final_response", "") if isinstance(result, dict) else "")
                 effective_session_id = result.get("session_id", session_id) if isinstance(result, dict) else session_id
                 turn_messages = self._turn_transcript_messages(history, user_message, result) if isinstance(result, dict) else []
                 await queue.put(_event_payload("assistant.completed", {
@@ -2087,7 +2136,7 @@ async def _compute_completion():
                     status=500,
                 )
 
-        final_response = result.get("final_response") or ""
+        final_response = _resolve_media_to_data_urls(result.get("final_response") or "")
         is_partial = bool(result.get("partial"))
         is_failed = bool(result.get("failed"))
         completed = bool(result.get("completed", True))
@@ -3186,7 +3235,7 @@ async def _compute_response():
                     status=500,
                 )
 
-        final_response = result.get("final_response", "")
+        final_response = _resolve_media_to_data_urls(result.get("final_response", ""))
         if not final_response:
             final_response = _redact_api_error_text(result.get("error", "(No response generated)"))
 

scripts/release.py

@@ -45,6 +45,7 @@
 
 # Auto-extracted from noreply emails + manual overrides
 AUTHOR_MAP = {
+    "tarunravi@gmail.com": "tarunravi",  # PR #2696 salvage (api-server: inline MEDIA:<path> image tags as base64 data URLs in final responses so remote OpenAI-compatible frontends can render server-local screenshots; the PR's tool-progress-streaming and SSE-sentinel pieces were independently superseded on main)
     "aqdrgg19@gmail.com": "VolodymyrBg",  # PR #2861 salvage (webhook: drop the unused full request payload from retained _delivery_info entries — up to ~1MB dead weight per delivery for the 1h idempotency TTL)
     "ohyes9711@gmail.com": "CharmingGroot",  # PR #2794 salvage (email: guard msg_data[0][1] against malformed IMAP fetch structures so one bad response can't abort the batch and permanently lose seen-marked messages; Message-ID domain falls back to localhost when EMAIL_ADDRESS lacks '@')
     "sahibzada@fastino.ai": "sahibzada-allahyar",  # PR #39227 salvage (desktop: configured terminal.cwd overrides a stale remembered workspace-cwd localStorage value when no session is active; #38855)

tests/gateway/test_api_server_media_data_urls.py

@@ -0,0 +1,77 @@
+"""MEDIA: tag → base64 data-URL resolution for the API server (salvage of #2696).
+
+Remote OpenAI-compatible frontends can't read local file paths, so
+``MEDIA:<path>`` image tags in final responses are inlined as markdown
+data URLs before crossing the HTTP boundary.
+"""
+
+import base64
+import unittest
+
+import pytest
+
+pytest.importorskip("aiohttp")
+
+from gateway.platforms.api_server import _resolve_media_to_data_urls  # noqa: E402
+
+# 1x1 transparent PNG
+_PNG_BYTES = base64.b64decode(
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQAB"
+    "h6FO1AAAAABJRU5ErkJggg=="
+)
+
+
+class TestResolveMediaToDataUrls(unittest.TestCase):
+    def _write_png(self, tmpdir_name="hermes_media_test"):
+        import tempfile
+        from pathlib import Path
+
+        d = Path(tempfile.mkdtemp(prefix=tmpdir_name))
+        p = d / "shot.png"
+        p.write_bytes(_PNG_BYTES)
+        return p
+
+    def test_media_tag_inlined(self):
+        p = self._write_png()
+        out = _resolve_media_to_data_urls(f"Here you go: MEDIA:{p}")
+        self.assertIn("data:image/png;base64,", out)
+        self.assertNotIn("MEDIA:", out)
+
+    def test_backtick_wrapped_tag(self):
+        p = self._write_png()
+        out = _resolve_media_to_data_urls(f"See `MEDIA:{p}` above")
+        self.assertIn("data:image/png;base64,", out)
+
+    def test_missing_file_left_untouched(self):
+        text = "MEDIA:/nonexistent/path/shot.png"
+        self.assertEqual(_resolve_media_to_data_urls(text), text)
+
+    def test_non_image_left_untouched(self):
+        text = "MEDIA:/tmp/archive.zip"
+        self.assertEqual(_resolve_media_to_data_urls(text), text)
+
+    def test_text_without_media_passthrough(self):
+        self.assertEqual(_resolve_media_to_data_urls("plain text"), "plain text")
+        self.assertEqual(_resolve_media_to_data_urls(""), "")
+
+    def test_oversized_image_skipped(self):
+        from gateway.platforms import api_server as mod
+
+        p = self._write_png()
+        orig = mod._MEDIA_DATA_URL_MAX_BYTES
+        mod._MEDIA_DATA_URL_MAX_BYTES = 1
+        try:
+            text = f"MEDIA:{p}"
+            self.assertEqual(_resolve_media_to_data_urls(text), text)
+        finally:
+            mod._MEDIA_DATA_URL_MAX_BYTES = orig
+
+    def test_multiple_tags(self):
+        p1 = self._write_png()
+        p2 = self._write_png("hermes_media_test2")
+        out = _resolve_media_to_data_urls(f"MEDIA:{p1}\nand MEDIA:{p2}")
+        self.assertEqual(out.count("data:image/png;base64,"), 2)
+
+
+if __name__ == "__main__":
+    unittest.main()