From 1a8916cb6c98d28eb6ae5233ad3b6295fb3fd027 Mon Sep 17 00:00:00 2001 From: Kain Date: Tue, 14 Apr 2026 11:50:01 +0200 Subject: [PATCH 1/2] test(mcp): cover image-only content blocks --- tests/tools/test_mcp_structured_content.py | 29 +++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/tools/test_mcp_structured_content.py b/tests/tools/test_mcp_structured_content.py index 520872e8a542..00154dab7a1f 100644 --- a/tests/tools/test_mcp_structured_content.py +++ b/tests/tools/test_mcp_structured_content.py @@ -11,13 +11,22 @@ class _FakeContentBlock: - """Minimal content block with .text and .type attributes.""" + """Minimal text content block with .text and .type attributes.""" def __init__(self, text: str, block_type: str = "text"): self.text = text self.type = block_type +class _FakeImageContentBlock: + """Minimal image content block with .data and .mimeType attributes.""" + + def __init__(self, data, mime_type: str = "image/png"): + self.data = data + self.mimeType = mime_type + self.type = "image" + + class _FakeCallToolResult: """Minimal CallToolResult stand-in. @@ -129,3 +138,21 @@ def test_empty_text_with_structured_content(self, _patch_mcp_server): raw = handler({}) data = json.loads(raw) assert data["result"] == payload + + def test_image_block_is_preserved_as_media(self, _patch_mcp_server): + """When an MCP tool returns image blocks, Hermes should cache them and emit MEDIA tags.""" + session = _patch_mcp_server + png_b64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO7r3h8AAAAASUVORK5CYII=" + ) + session.call_tool = AsyncMock( + return_value=_FakeCallToolResult( + content=[_FakeImageContentBlock(png_b64, "image/png")], + ) + ) + handler = mcp_tool._make_tool_handler("test-server", "my-tool", 30.0) + raw = handler({}) + data = json.loads(raw) + assert data["result"].startswith("MEDIA:") + assert data["media"] + assert data["media"][0].endswith(".png") From d3eb01475cf82330691b328311a9665569f6aec7 Mon Sep 17 00:00:00 2001 From: Kain Date: Tue, 14 Apr 2026 11:50:21 +0200 Subject: [PATCH 2/2] fix(mcp): preserve image content blocks in tool results --- tools/mcp_tool.py | 64 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/tools/mcp_tool.py b/tools/mcp_tool.py index 2356830c465d..3899baea8ac5 100644 --- a/tools/mcp_tool.py +++ b/tools/mcp_tool.py @@ -70,6 +70,7 @@ """ import asyncio +import base64 import concurrent.futures import inspect import json @@ -82,6 +83,8 @@ import time from typing import Any, Dict, List, Optional +from gateway.platforms.base import cache_image_from_bytes + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -1307,11 +1310,56 @@ async def _call(): ) }) - # Collect text from content blocks + # Collect text and binary media from content blocks. parts: List[str] = [] + media_paths: List[str] = [] + + def _image_ext_from_mime(mime: str) -> str: + mime = (mime or "").lower() + if "png" in mime: + return ".png" + if "jpeg" in mime or "jpg" in mime: + return ".jpg" + if "webp" in mime: + return ".webp" + if "gif" in mime: + return ".gif" + if "bmp" in mime: + return ".bmp" + return ".png" + for block in (result.content or []): if hasattr(block, "text"): parts.append(block.text) + elif hasattr(block, "data") and hasattr(block, "mimeType"): + mime = str(getattr(block, "mimeType", "") or "") + raw_data = getattr(block, "data", b"") + if isinstance(raw_data, str): + # MCP image payloads are often base64 strings; accept + # either plain base64 or data URIs. + if raw_data.startswith("data:") and "," in raw_data: + raw_data = raw_data.split(",", 1)[1] + try: + raw_bytes = base64.b64decode(raw_data) + except Exception: + raw_bytes = raw_data.encode("utf-8", errors="ignore") + else: + raw_bytes = bytes(raw_data) + try: + image_path = cache_image_from_bytes(raw_bytes, ext=_image_ext_from_mime(mime)) + media_paths.append(image_path) + parts.append(f"MEDIA:{image_path}") + except Exception as exc: + logger.warning( + "MCP image block could not be cached for %s/%s: %s", + server_name, tool_name, exc, + ) + parts.append(f"[image:{mime or 'unknown'}]") + else: + logger.warning( + "MCP tool %s/%s returned unsupported content block: %r", + server_name, tool_name, block, + ) text_result = "\n".join(parts) if parts else "" # Combine content + structuredContent when both are present. @@ -1319,14 +1367,14 @@ async def _call(): # is machine-oriented (JSON metadata). For an AI agent, content # is the primary payload; structuredContent supplements it. structured = getattr(result, "structuredContent", None) + payload = {"result": text_result} + if media_paths: + payload["media"] = media_paths if structured is not None: - if text_result: - return json.dumps({ - "result": text_result, - "structuredContent": structured, - }) - return json.dumps({"result": structured}) - return json.dumps({"result": text_result}) + payload["structuredContent"] = structured + if not text_result and not media_paths: + return json.dumps({"result": structured}) + return json.dumps(payload) try: return _run_on_mcp_loop(_call(), timeout=tool_timeout)