feat(multimodal): implement native image content flow across gateway, agent, and read_file, with safer context budgeting

agent/anthropic_adapter.py

@@ -1027,9 +1027,14 @@ def convert_messages_to_anthropic(
 
         if role == "tool":
             # Sanitize tool_use_id and ensure non-empty content
-            result_content = content if isinstance(content, str) else json.dumps(content)
-            if not result_content:
-                result_content = "(no output)"
+            if isinstance(content, list):
+                result_content = _convert_content_to_anthropic(content)
+                if not result_content:
+                    result_content = [{"type": "text", "text": "(no output)"}]
+            else:
+                result_content = content if isinstance(content, str) else json.dumps(content)
+                if not result_content:
+                    result_content = "(no output)"
             tool_result = {
                 "type": "tool_result",
                 "tool_use_id": _sanitize_tool_id(m.get("tool_call_id", "")),
@@ -1319,4 +1324,4 @@ def normalize_anthropic_response(
             reasoning_details=None,
         ),
         finish_reason,
-    )
+    )

agent/context_compressor.py

@@ -17,6 +17,7 @@
 from typing import Any, Dict, List, Optional
 
 from agent.auxiliary_client import call_llm
+from agent.message_content import content_to_text
 from agent.model_metadata import (
     get_model_context_length,
     estimate_messages_tokens_rough,
@@ -171,7 +172,11 @@ def _prune_old_tool_results(
             msg = result[i]
             if msg.get("role") != "tool":
                 continue
-            content = msg.get("content", "")
+            content = content_to_text(
+                msg.get("content", ""),
+                image_placeholder="[image]",
+                fallback_json=True,
+            )
             if not content or content == _PRUNED_TOOL_PLACEHOLDER:
                 continue
             # Only prune if the content is substantial (>200 chars)
@@ -206,7 +211,11 @@ def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str:
         parts = []
         for msg in turns:
             role = msg.get("role", "unknown")
-            content = msg.get("content") or ""
+            content = content_to_text(
+                msg.get("content"),
+                image_placeholder="[image]",
+                fallback_json=True,
+            )
 
             # Tool results: keep more content than before (3000 chars)
             if role == "tool":
@@ -510,7 +519,11 @@ def _find_tail_cut_by_tokens(
 
         for i in range(n - 1, head_end - 1, -1):
             msg = messages[i]
-            content = msg.get("content") or ""
+            content = content_to_text(
+                msg.get("content"),
+                image_placeholder="[image]",
+                fallback_json=True,
+            )
             msg_tokens = len(content) // _CHARS_PER_TOKEN + 10  # +10 for role/metadata
             # Include tool call arguments in estimate
             for tc in msg.get("tool_calls") or []:
@@ -617,7 +630,7 @@ def compress(self, messages: List[Dict[str, Any]], current_tokens: int = None) -
             msg = messages[i].copy()
             if i == 0 and msg.get("role") == "system" and self.compression_count == 0:
                 msg["content"] = (
-                    (msg.get("content") or "")
+                    content_to_text(msg.get("content"), image_placeholder="[image]", fallback_json=True)
                     + "\n\n[Note: Some earlier conversation turns have been compacted into a handoff summary to preserve context space. The current session state may still reflect earlier work, so build on that summary and state rather than re-doing work.]"
                 )
             compressed.append(msg)
@@ -653,7 +666,11 @@ def compress(self, messages: List[Dict[str, Any]], current_tokens: int = None) -
         for i in range(compress_end, n_messages):
             msg = messages[i].copy()
             if _merge_summary_into_tail and i == compress_end:
-                original = msg.get("content") or ""
+                original = content_to_text(
+                    msg.get("content"),
+                    image_placeholder="[image]",
+                    fallback_json=True,
+                )
                 msg["content"] = summary + "\n\n" + original
                 _merge_summary_into_tail = False
             compressed.append(msg)

agent/display.py

@@ -767,7 +767,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 # Cute tool message (completion line that replaces the spinner)
 # =========================================================================
 
-def _detect_tool_failure(tool_name: str, result: str | None) -> tuple[bool, str]:
+def _detect_tool_failure(tool_name: str, result: object | None) -> tuple[bool, str]:
     """Inspect a tool result string for signs of failure.
 
     Returns ``(is_failure, suffix)`` where *suffix* is an informational tag
@@ -777,9 +777,17 @@ def _detect_tool_failure(tool_name: str, result: str | None) -> tuple[bool, str]
     if result is None:
         return False, ""
 
+    if isinstance(result, str):
+        result_text = result
+    else:
+        try:
+            result_text = json.dumps(result, ensure_ascii=False)
+        except (TypeError, ValueError):
+            result_text = str(result)
+
     if tool_name == "terminal":
         try:
-            data = json.loads(result)
+            data = json.loads(result_text)
             exit_code = data.get("exit_code")
             if exit_code is not None and exit_code != 0:
                 return True, f" [exit {exit_code}]"
@@ -790,15 +798,15 @@ def _detect_tool_failure(tool_name: str, result: str | None) -> tuple[bool, str]
     # Memory-specific: distinguish "full" from real errors
     if tool_name == "memory":
         try:
-            data = json.loads(result)
+            data = json.loads(result_text)
             if data.get("success") is False and "exceed the limit" in data.get("error", ""):
                 return True, " [full]"
         except (json.JSONDecodeError, TypeError, AttributeError):
             logger.debug("Could not parse memory result as JSON for capacity check")
 
     # Generic heuristic for non-terminal tools
-    lower = result[:500].lower()
-    if '"error"' in lower or '"failed"' in lower or result.startswith("Error"):
+    lower = result_text[:500].lower()
+    if '"error"' in lower or '"failed"' in lower or result_text.startswith("Error"):
         return True, " [error]"
 
     return False, ""

agent/message_content.py

@@ -0,0 +1,151 @@
+"""Helpers for Hermes message content blocks.
+
+Hermes historically treated ``message["content"]`` as plain text. Modern chat
+APIs also allow structured content arrays that mix text and images. This module
+provides a small set of helpers so the rest of the codebase can preserve
+multimodal payloads internally while still deriving text for logging, search,
+and rough token estimation.
+"""
+
+from __future__ import annotations
+
+import json
+import base64
+import mimetypes
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+_TEXT_PART_TYPES = frozenset({"text", "input_text", "output_text"})
+_IMAGE_PART_TYPES = frozenset({"image_url", "input_image", "image"})
+
+
+def image_path_to_data_url(path: str, media_type: str = "") -> Optional[str]:
+    """Read a local image file and return a data URL."""
+    try:
+        raw = Path(path).read_bytes()
+    except OSError:
+        return None
+    mime = media_type if isinstance(media_type, str) and media_type.startswith("image/") else ""
+    if not mime:
+        mime = mimetypes.guess_type(path)[0] or "image/jpeg"
+    encoded = base64.b64encode(raw).decode("ascii")
+    return f"data:{mime};base64,{encoded}"
+
+
+def content_has_image_parts(content: Any) -> bool:
+    if not isinstance(content, list):
+        return False
+    for part in content:
+        if isinstance(part, dict) and part.get("type") in _IMAGE_PART_TYPES:
+            return True
+    return False
+
+
+def content_to_text(
+    content: Any,
+    *,
+    image_placeholder: str = "[image]",
+    fallback_json: bool = False,
+) -> str:
+    """Extract human-readable text from a structured content payload."""
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: List[str] = []
+        for part in content:
+            if isinstance(part, str):
+                if part:
+                    parts.append(part)
+                continue
+            if not isinstance(part, dict):
+                if fallback_json:
+                    parts.append(str(part))
+                continue
+            ptype = part.get("type")
+            if ptype in _TEXT_PART_TYPES:
+                text = part.get("text")
+                if text is None and ptype == "output_text":
+                    text = part.get("content")
+                if isinstance(text, str) and text:
+                    parts.append(text)
+            elif ptype in _IMAGE_PART_TYPES and image_placeholder:
+                parts.append(image_placeholder)
+            elif fallback_json:
+                text = part.get("text")
+                if isinstance(text, str) and text:
+                    parts.append(text)
+                else:
+                    parts.append(json.dumps(part, ensure_ascii=False))
+        return "\n".join(p for p in parts if p)
+    if isinstance(content, dict):
+        text = content.get("text")
+        if isinstance(text, str):
+            return text
+        nested = content.get("content")
+        if nested is not None:
+            return content_to_text(
+                nested,
+                image_placeholder=image_placeholder,
+                fallback_json=fallback_json,
+            )
+        return json.dumps(content, ensure_ascii=False) if fallback_json else ""
+    return str(content)
+
+
+def serialize_message_content(content: Any) -> tuple[Optional[str], Optional[str]]:
+    """Return ``(content_text, content_json)`` for state.db storage."""
+    if content is None:
+        return None, None
+    if isinstance(content, str):
+        return content, None
+    return (
+        content_to_text(content, image_placeholder="[image]", fallback_json=False),
+        json.dumps(content, ensure_ascii=False),
+    )
+
+
+def deserialize_message_content(content_text: Any, content_json: Any) -> Any:
+    """Restore structured content from state.db columns."""
+    if isinstance(content_json, str) and content_json:
+        try:
+            return json.loads(content_json)
+        except (json.JSONDecodeError, TypeError):
+            pass
+    return content_text
+
+
+def convert_content_to_responses_input(content: Any) -> Any:
+    """Convert OpenAI chat-style content blocks to Responses input blocks."""
+    if isinstance(content, str):
+        return content
+    if not isinstance(content, list):
+        return content_to_text(content, image_placeholder="", fallback_json=True)
+
+    converted: List[Dict[str, Any]] = []
+    for part in content:
+        if isinstance(part, str):
+            if part:
+                converted.append({"type": "input_text", "text": part})
+            continue
+        if not isinstance(part, dict):
+            continue
+        ptype = part.get("type", "")
+        if ptype == "text":
+            converted.append({"type": "input_text", "text": part.get("text", "")})
+        elif ptype == "image_url":
+            image_data = part.get("image_url", {})
+            url = image_data.get("url", "") if isinstance(image_data, dict) else str(image_data or "")
+            entry: Dict[str, Any] = {"type": "input_image", "image_url": url}
+            if isinstance(image_data, dict) and image_data.get("detail"):
+                entry["detail"] = image_data["detail"]
+            converted.append(entry)
+        elif ptype in {"input_text", "input_image"}:
+            converted.append(dict(part))
+        else:
+            text = part.get("text", "")
+            if text:
+                converted.append({"type": "input_text", "text": text})
+
+    return converted or ""