diff --git a/agent/anthropic_adapter.py b/agent/anthropic_adapter.py index 2fae12dde86f3..6d3afbddf7236 100644 --- a/agent/anthropic_adapter.py +++ b/agent/anthropic_adapter.py @@ -1027,9 +1027,14 @@ def convert_messages_to_anthropic( if role == "tool": # Sanitize tool_use_id and ensure non-empty content - result_content = content if isinstance(content, str) else json.dumps(content) - if not result_content: - result_content = "(no output)" + if isinstance(content, list): + result_content = _convert_content_to_anthropic(content) + if not result_content: + result_content = [{"type": "text", "text": "(no output)"}] + else: + result_content = content if isinstance(content, str) else json.dumps(content) + if not result_content: + result_content = "(no output)" tool_result = { "type": "tool_result", "tool_use_id": _sanitize_tool_id(m.get("tool_call_id", "")), @@ -1319,4 +1324,4 @@ def normalize_anthropic_response( reasoning_details=None, ), finish_reason, - ) \ No newline at end of file + ) diff --git a/agent/context_compressor.py b/agent/context_compressor.py index 6fdb38b29b388..cf8baa1ef4cf7 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -17,6 +17,7 @@ from typing import Any, Dict, List, Optional from agent.auxiliary_client import call_llm +from agent.message_content import content_to_text from agent.model_metadata import ( get_model_context_length, estimate_messages_tokens_rough, @@ -171,7 +172,11 @@ def _prune_old_tool_results( msg = result[i] if msg.get("role") != "tool": continue - content = msg.get("content", "") + content = content_to_text( + msg.get("content", ""), + image_placeholder="[image]", + fallback_json=True, + ) if not content or content == _PRUNED_TOOL_PLACEHOLDER: continue # Only prune if the content is substantial (>200 chars) @@ -206,7 +211,11 @@ def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str: parts = [] for msg in turns: role = msg.get("role", "unknown") - content = msg.get("content") or "" + content = content_to_text( + msg.get("content"), + image_placeholder="[image]", + fallback_json=True, + ) # Tool results: keep more content than before (3000 chars) if role == "tool": @@ -510,7 +519,11 @@ def _find_tail_cut_by_tokens( for i in range(n - 1, head_end - 1, -1): msg = messages[i] - content = msg.get("content") or "" + content = content_to_text( + msg.get("content"), + image_placeholder="[image]", + fallback_json=True, + ) msg_tokens = len(content) // _CHARS_PER_TOKEN + 10 # +10 for role/metadata # Include tool call arguments in estimate for tc in msg.get("tool_calls") or []: @@ -617,7 +630,7 @@ def compress(self, messages: List[Dict[str, Any]], current_tokens: int = None) - msg = messages[i].copy() if i == 0 and msg.get("role") == "system" and self.compression_count == 0: msg["content"] = ( - (msg.get("content") or "") + content_to_text(msg.get("content"), image_placeholder="[image]", fallback_json=True) + "\n\n[Note: Some earlier conversation turns have been compacted into a handoff summary to preserve context space. The current session state may still reflect earlier work, so build on that summary and state rather than re-doing work.]" ) compressed.append(msg) @@ -653,7 +666,11 @@ def compress(self, messages: List[Dict[str, Any]], current_tokens: int = None) - for i in range(compress_end, n_messages): msg = messages[i].copy() if _merge_summary_into_tail and i == compress_end: - original = msg.get("content") or "" + original = content_to_text( + msg.get("content"), + image_placeholder="[image]", + fallback_json=True, + ) msg["content"] = summary + "\n\n" + original _merge_summary_into_tail = False compressed.append(msg) diff --git a/agent/display.py b/agent/display.py index 94259fa80a899..97aab746fad52 100644 --- a/agent/display.py +++ b/agent/display.py @@ -767,7 +767,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): # Cute tool message (completion line that replaces the spinner) # ========================================================================= -def _detect_tool_failure(tool_name: str, result: str | None) -> tuple[bool, str]: +def _detect_tool_failure(tool_name: str, result: object | None) -> tuple[bool, str]: """Inspect a tool result string for signs of failure. Returns ``(is_failure, suffix)`` where *suffix* is an informational tag @@ -777,9 +777,17 @@ def _detect_tool_failure(tool_name: str, result: str | None) -> tuple[bool, str] if result is None: return False, "" + if isinstance(result, str): + result_text = result + else: + try: + result_text = json.dumps(result, ensure_ascii=False) + except (TypeError, ValueError): + result_text = str(result) + if tool_name == "terminal": try: - data = json.loads(result) + data = json.loads(result_text) exit_code = data.get("exit_code") if exit_code is not None and exit_code != 0: return True, f" [exit {exit_code}]" @@ -790,15 +798,15 @@ def _detect_tool_failure(tool_name: str, result: str | None) -> tuple[bool, str] # Memory-specific: distinguish "full" from real errors if tool_name == "memory": try: - data = json.loads(result) + data = json.loads(result_text) if data.get("success") is False and "exceed the limit" in data.get("error", ""): return True, " [full]" except (json.JSONDecodeError, TypeError, AttributeError): logger.debug("Could not parse memory result as JSON for capacity check") # Generic heuristic for non-terminal tools - lower = result[:500].lower() - if '"error"' in lower or '"failed"' in lower or result.startswith("Error"): + lower = result_text[:500].lower() + if '"error"' in lower or '"failed"' in lower or result_text.startswith("Error"): return True, " [error]" return False, "" diff --git a/agent/message_content.py b/agent/message_content.py new file mode 100644 index 0000000000000..f2ee9aec70084 --- /dev/null +++ b/agent/message_content.py @@ -0,0 +1,151 @@ +"""Helpers for Hermes message content blocks. + +Hermes historically treated ``message["content"]`` as plain text. Modern chat +APIs also allow structured content arrays that mix text and images. This module +provides a small set of helpers so the rest of the codebase can preserve +multimodal payloads internally while still deriving text for logging, search, +and rough token estimation. +""" + +from __future__ import annotations + +import json +import base64 +import mimetypes +from pathlib import Path +from typing import Any, Dict, List, Optional + +_TEXT_PART_TYPES = frozenset({"text", "input_text", "output_text"}) +_IMAGE_PART_TYPES = frozenset({"image_url", "input_image", "image"}) + + +def image_path_to_data_url(path: str, media_type: str = "") -> Optional[str]: + """Read a local image file and return a data URL.""" + try: + raw = Path(path).read_bytes() + except OSError: + return None + mime = media_type if isinstance(media_type, str) and media_type.startswith("image/") else "" + if not mime: + mime = mimetypes.guess_type(path)[0] or "image/jpeg" + encoded = base64.b64encode(raw).decode("ascii") + return f"data:{mime};base64,{encoded}" + + +def content_has_image_parts(content: Any) -> bool: + if not isinstance(content, list): + return False + for part in content: + if isinstance(part, dict) and part.get("type") in _IMAGE_PART_TYPES: + return True + return False + + +def content_to_text( + content: Any, + *, + image_placeholder: str = "[image]", + fallback_json: bool = False, +) -> str: + """Extract human-readable text from a structured content payload.""" + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + parts: List[str] = [] + for part in content: + if isinstance(part, str): + if part: + parts.append(part) + continue + if not isinstance(part, dict): + if fallback_json: + parts.append(str(part)) + continue + ptype = part.get("type") + if ptype in _TEXT_PART_TYPES: + text = part.get("text") + if text is None and ptype == "output_text": + text = part.get("content") + if isinstance(text, str) and text: + parts.append(text) + elif ptype in _IMAGE_PART_TYPES and image_placeholder: + parts.append(image_placeholder) + elif fallback_json: + text = part.get("text") + if isinstance(text, str) and text: + parts.append(text) + else: + parts.append(json.dumps(part, ensure_ascii=False)) + return "\n".join(p for p in parts if p) + if isinstance(content, dict): + text = content.get("text") + if isinstance(text, str): + return text + nested = content.get("content") + if nested is not None: + return content_to_text( + nested, + image_placeholder=image_placeholder, + fallback_json=fallback_json, + ) + return json.dumps(content, ensure_ascii=False) if fallback_json else "" + return str(content) + + +def serialize_message_content(content: Any) -> tuple[Optional[str], Optional[str]]: + """Return ``(content_text, content_json)`` for state.db storage.""" + if content is None: + return None, None + if isinstance(content, str): + return content, None + return ( + content_to_text(content, image_placeholder="[image]", fallback_json=False), + json.dumps(content, ensure_ascii=False), + ) + + +def deserialize_message_content(content_text: Any, content_json: Any) -> Any: + """Restore structured content from state.db columns.""" + if isinstance(content_json, str) and content_json: + try: + return json.loads(content_json) + except (json.JSONDecodeError, TypeError): + pass + return content_text + + +def convert_content_to_responses_input(content: Any) -> Any: + """Convert OpenAI chat-style content blocks to Responses input blocks.""" + if isinstance(content, str): + return content + if not isinstance(content, list): + return content_to_text(content, image_placeholder="", fallback_json=True) + + converted: List[Dict[str, Any]] = [] + for part in content: + if isinstance(part, str): + if part: + converted.append({"type": "input_text", "text": part}) + continue + if not isinstance(part, dict): + continue + ptype = part.get("type", "") + if ptype == "text": + converted.append({"type": "input_text", "text": part.get("text", "")}) + elif ptype == "image_url": + image_data = part.get("image_url", {}) + url = image_data.get("url", "") if isinstance(image_data, dict) else str(image_data or "") + entry: Dict[str, Any] = {"type": "input_image", "image_url": url} + if isinstance(image_data, dict) and image_data.get("detail"): + entry["detail"] = image_data["detail"] + converted.append(entry) + elif ptype in {"input_text", "input_image"}: + converted.append(dict(part)) + else: + text = part.get("text", "") + if text: + converted.append({"type": "input_text", "text": text}) + + return converted or "" diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 7486afb048f4e..44c8c263e8542 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -8,6 +8,9 @@ import os import re import time +import base64 +import math +import struct from pathlib import Path from typing import Any, Dict, List, Optional from urllib.parse import urlparse @@ -15,6 +18,7 @@ import requests import yaml +from agent.message_content import content_to_text from hermes_constants import OPENROUTER_MODELS_URL logger = logging.getLogger(__name__) @@ -79,6 +83,7 @@ def _strip_provider_prefix(model: str) -> str: # Default context length when no detection method succeeds. DEFAULT_FALLBACK_CONTEXT = CONTEXT_PROBE_TIERS[0] +_ROUGH_IMAGE_TOKEN_FALLBACK = 1600 # Thin fallback defaults โ€” only broad model family patterns. # These fire only when provider is unknown AND models.dev/OpenRouter/Anthropic @@ -386,11 +391,15 @@ def fetch_model_metadata(force_refresh: bool = False) -> Dict[str, Dict[str, Any cache = {} for model in data.get("data", []): model_id = model.get("id", "") + arch = model.get("architecture", {}) + input_mods = arch.get("input_modalities", []) if isinstance(arch, dict) else [] entry = { "context_length": model.get("context_length", 128000), "max_completion_tokens": model.get("top_provider", {}).get("max_completion_tokens", 4096), "name": model.get("name", model_id), "pricing": model.get("pricing", {}), + "supports_vision": "image" in input_mods, + "supports_audio": "audio" in input_mods, } _add_model_aliases(cache, model_id, entry) canonical = model.get("canonical_slug", "") @@ -896,16 +905,224 @@ def get_model_context_length( def estimate_tokens_rough(text: str) -> int: - """Rough token estimate (~4 chars/token) for pre-flight checks.""" + """Conservative rough token estimate for mixed-language text. + + ASCII-heavy text is estimated at roughly 4 chars/token. Non-ASCII text is + treated more conservatively at roughly 1 char/token to avoid severe + undercounting for CJK, Persian, Arabic, and similar scripts. + """ if not text: return 0 - return len(text) // 4 + ascii_chars = sum(1 for ch in text if ord(ch) < 128) + non_ascii_chars = len(text) - ascii_chars + estimate = max(len(text) // 4, (ascii_chars // 4) + non_ascii_chars) + return max(1, estimate) + + +def _parse_png_dimensions(data: bytes) -> Optional[tuple[int, int]]: + if len(data) < 24 or not data.startswith(b"\x89PNG\r\n\x1a\n"): + return None + try: + width, height = struct.unpack(">II", data[16:24]) + except struct.error: + return None + return (width, height) if width > 0 and height > 0 else None + + +def _parse_gif_dimensions(data: bytes) -> Optional[tuple[int, int]]: + if len(data) < 10 or not data.startswith((b"GIF87a", b"GIF89a")): + return None + width, height = struct.unpack(" 0 and height > 0 else None + + +def _parse_bmp_dimensions(data: bytes) -> Optional[tuple[int, int]]: + if len(data) < 26 or not data.startswith(b"BM"): + return None + dib_header_size = struct.unpack("= 40 and len(data) >= 26: + width = struct.unpack("= 26: + width, height = struct.unpack(" 0 and height > 0 else None + + +def _parse_webp_dimensions(data: bytes) -> Optional[tuple[int, int]]: + if len(data) < 30 or data[:4] != b"RIFF" or data[8:12] != b"WEBP": + return None + chunk = data[12:16] + try: + if chunk == b"VP8 " and len(data) >= 30: + width, height = struct.unpack("= 25: + bits = struct.unpack("> 14) & 0x3FFF) + 1 + return (width, height) + if chunk == b"VP8X" and len(data) >= 30: + width = 1 + int.from_bytes(data[24:27], "little") + height = 1 + int.from_bytes(data[27:30], "little") + return (width, height) + except Exception: + return None + return None + + +def _parse_jpeg_dimensions(data: bytes) -> Optional[tuple[int, int]]: + if len(data) < 4 or not data.startswith(b"\xff\xd8"): + return None + i = 2 + size = len(data) + while i + 9 < size: + if data[i] != 0xFF: + i += 1 + continue + while i < size and data[i] == 0xFF: + i += 1 + if i >= size: + break + marker = data[i] + i += 1 + if marker in {0xD8, 0xD9}: + continue + if i + 1 >= size: + break + seglen = struct.unpack(">H", data[i:i + 2])[0] + if seglen < 2 or i + seglen > size: + break + if marker in { + 0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, + 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF, + } and seglen >= 7: + height = struct.unpack(">H", data[i + 3:i + 5])[0] + width = struct.unpack(">H", data[i + 5:i + 7])[0] + return (width, height) if width > 0 and height > 0 else None + i += seglen + return None + + +def _image_dimensions_from_bytes(data: bytes) -> Optional[tuple[int, int]]: + for parser in ( + _parse_png_dimensions, + _parse_jpeg_dimensions, + _parse_gif_dimensions, + _parse_bmp_dimensions, + _parse_webp_dimensions, + ): + dims = parser(data) + if dims: + return dims + return None + + +def _load_image_dimensions(image_ref: str) -> Optional[tuple[int, int]]: + ref = str(image_ref or "").strip() + if not ref: + return None + try: + if ref.startswith("data:"): + _, _, encoded = ref.partition(",") + if not encoded: + return None + return _image_dimensions_from_bytes(base64.b64decode(encoded, validate=False)) + path = Path(ref).expanduser() + if path.exists() and path.is_file(): + return _image_dimensions_from_bytes(path.read_bytes()) + except Exception: + return None + return None + + +def _estimate_gpt54_image_tokens(width: int, height: int, detail: str = "auto") -> int: + if width <= 0 or height <= 0: + return _ROUGH_IMAGE_TOKEN_FALLBACK + + detail_norm = str(detail or "auto").strip().lower() + if detail_norm == "low": + return 256 + + patch_budget = 10_000 if detail_norm == "original" else 2_500 + max_dimension = 6_000 if detail_norm == "original" else 2_048 + + scale = min(1.0, max_dimension / max(width, height)) + scaled_width = max(1, int(math.floor(width * scale))) + scaled_height = max(1, int(math.floor(height * scale))) + + patch_count = math.ceil(scaled_width / 32) * math.ceil(scaled_height / 32) + if patch_count > patch_budget: + shrink_factor = math.sqrt((32 * 32 * patch_budget) / (scaled_width * scaled_height)) + width_ratio = (scaled_width * shrink_factor) / 32 + height_ratio = (scaled_height * shrink_factor) / 32 + adjusted = shrink_factor * min( + math.floor(width_ratio) / width_ratio if width_ratio else 1.0, + math.floor(height_ratio) / height_ratio if height_ratio else 1.0, + ) + scaled_width = max(1, int(math.floor(scaled_width * adjusted))) + scaled_height = max(1, int(math.floor(scaled_height * adjusted))) + patch_count = math.ceil(scaled_width / 32) * math.ceil(scaled_height / 32) + + return min(patch_count, patch_budget) + + +def estimate_image_tokens_rough(image_ref: str, detail: str = "auto") -> int: + dims = _load_image_dimensions(image_ref) + if not dims: + return _ROUGH_IMAGE_TOKEN_FALLBACK + return _estimate_gpt54_image_tokens(dims[0], dims[1], detail=detail) + + +def _estimate_content_image_tokens(content: Any) -> int: + if not isinstance(content, list): + return 0 + total = 0 + for part in content: + if not isinstance(part, dict): + continue + ptype = part.get("type") + if ptype == "image_url": + image_data = part.get("image_url", {}) + image_ref = image_data.get("url", "") if isinstance(image_data, dict) else str(image_data or "") + detail = image_data.get("detail", "auto") if isinstance(image_data, dict) else "auto" + total += estimate_image_tokens_rough(image_ref, detail) + elif ptype == "input_image": + image_ref = part.get("image_url") or part.get("file_id") or "" + total += estimate_image_tokens_rough(str(image_ref or ""), str(part.get("detail", "auto"))) + elif ptype == "image": + source = part.get("source", {}) if isinstance(part.get("source"), dict) else {} + if source.get("type") == "base64" and source.get("data"): + media_type = source.get("media_type") or "image/jpeg" + image_ref = f"data:{media_type};base64,{source['data']}" + total += estimate_image_tokens_rough(image_ref, "auto") + else: + total += _ROUGH_IMAGE_TOKEN_FALLBACK + return total def estimate_messages_tokens_rough(messages: List[Dict[str, Any]]) -> int: """Rough token estimate for a message list (pre-flight only).""" - total_chars = sum(len(str(msg)) for msg in messages) - return total_chars // 4 + total_text_parts: List[str] = [] + total_image_tokens = 0 + for msg in messages: + if not isinstance(msg, dict): + total_text_parts.append(str(msg)) + continue + msg_copy = dict(msg) + content = msg_copy.get("content") + if content is not None and not isinstance(content, str): + total_image_tokens += _estimate_content_image_tokens(content) + msg_copy["content"] = content_to_text( + content, + image_placeholder="[image]", + fallback_json=True, + ) + total_text_parts.append(str(msg_copy)) + return estimate_tokens_rough("".join(total_text_parts)) + total_image_tokens def estimate_request_tokens_rough( @@ -921,11 +1138,40 @@ def estimate_request_tokens_rough( tools enabled, schemas alone can add 20-30K tokens โ€” a significant blind spot when only counting messages. """ - total_chars = 0 + text_parts: List[str] = [] if system_prompt: - total_chars += len(system_prompt) + text_parts.append(system_prompt) if messages: - total_chars += sum(len(str(msg)) for msg in messages) + for msg in messages: + if isinstance(msg, dict) and not isinstance(msg.get("content"), str): + text_parts.append( + str( + { + **msg, + "content": content_to_text( + msg.get("content"), + image_placeholder="[image]", + fallback_json=True, + ), + } + ) + ) + else: + text_parts.append(str(msg)) if tools: - total_chars += len(str(tools)) - return total_chars // 4 + text_parts.append(str(tools)) + return estimate_tokens_rough("".join(text_parts)) + + +def get_model_capabilities(model: str, provider: Optional[str] = None) -> Dict[str, bool]: + """Get multimodal capabilities for a model. + + Returns dict with keys: supports_vision, supports_audio + Checks OpenRouter cache first, falls back to False if not found. + """ + metadata = fetch_model_metadata() + entry = metadata.get(model.lower(), {}) + return { + "supports_vision": entry.get("supports_vision", False), + "supports_audio": entry.get("supports_audio", False), + } diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py index 2059a1aa6bcc5..20826f053771f 100644 --- a/gateway/platforms/api_server.py +++ b/gateway/platforms/api_server.py @@ -26,6 +26,8 @@ import uuid from typing import Any, Dict, List, Optional +from agent.message_content import content_to_text + try: from aiohttp import web AIOHTTP_AVAILABLE = True @@ -470,28 +472,29 @@ async def _handle_chat_completions(self, request: "web.Request") -> "web.Respons # Extract system message (becomes ephemeral system prompt layered ON TOP of core) system_prompt = None - conversation_messages: List[Dict[str, str]] = [] + conversation_messages: List[Dict[str, Any]] = [] for msg in messages: role = msg.get("role", "") content = msg.get("content", "") if role == "system": # Accumulate system messages + content_text = content_to_text(content, image_placeholder="[image]", fallback_json=True) if system_prompt is None: - system_prompt = content + system_prompt = content_text else: - system_prompt = system_prompt + "\n" + content + system_prompt = system_prompt + "\n" + content_text elif role in ("user", "assistant"): conversation_messages.append({"role": role, "content": content}) # Extract the last user message as the primary input - user_message = "" + user_message: Any = "" history = [] if conversation_messages: user_message = conversation_messages[-1].get("content", "") history = conversation_messages[:-1] - if not user_message: + if not content_to_text(user_message, image_placeholder="[image]", fallback_json=True).strip(): return web.json_response( {"error": {"message": "No user message found in messages", "type": "invalid_request_error"}}, status=400, @@ -760,7 +763,7 @@ async def _handle_responses(self, request: "web.Request") -> "web.Response": # No error if conversation doesn't exist yet โ€” it's a new conversation # Normalize input to message list - input_messages: List[Dict[str, str]] = [] + input_messages: List[Dict[str, Any]] = [] if isinstance(raw_input, str): input_messages = [{"role": "user", "content": raw_input}] elif isinstance(raw_input, list): @@ -770,23 +773,12 @@ async def _handle_responses(self, request: "web.Request") -> "web.Response": elif isinstance(item, dict): role = item.get("role", "user") content = item.get("content", "") - # Handle content that may be a list of content parts - if isinstance(content, list): - text_parts = [] - for part in content: - if isinstance(part, dict) and part.get("type") == "input_text": - text_parts.append(part.get("text", "")) - elif isinstance(part, dict) and part.get("type") == "output_text": - text_parts.append(part.get("text", "")) - elif isinstance(part, str): - text_parts.append(part) - content = "\n".join(text_parts) input_messages.append({"role": role, "content": content}) else: return web.json_response(_openai_error("'input' must be a string or array"), status=400) # Reconstruct conversation history from previous_response_id - conversation_history: List[Dict[str, str]] = [] + conversation_history: List[Dict[str, Any]] = [] if previous_response_id: stored = self._response_store.get(previous_response_id) if stored is None: @@ -802,7 +794,7 @@ async def _handle_responses(self, request: "web.Request") -> "web.Response": # Last input message is the user_message user_message = input_messages[-1].get("content", "") if input_messages else "" - if not user_message: + if not content_to_text(user_message, image_placeholder="[image]", fallback_json=True).strip(): return web.json_response(_openai_error("No user message found in input"), status=400) # Truncation support @@ -1220,8 +1212,8 @@ def _extract_output_items(result: Dict[str, Any]) -> List[Dict[str, Any]]: async def _run_agent( self, - user_message: str, - conversation_history: List[Dict[str, str]], + user_message: Any, + conversation_history: List[Dict[str, Any]], ephemeral_system_prompt: Optional[str] = None, session_id: Optional[str] = None, stream_delta_callback=None, diff --git a/gateway/run.py b/gateway/run.py index b440ee71c5bd7..94292162d22d9 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -30,6 +30,8 @@ from datetime import datetime from typing import Dict, Optional, Any, List +from agent.message_content import content_to_text, image_path_to_data_url + # --------------------------------------------------------------------------- # SSL certificate auto-detection for NixOS and other non-standard systems. # Must run BEFORE any HTTP library (discord, aiohttp, etc.) is imported. @@ -673,7 +675,7 @@ def _flush_memories_for_session( msgs = [ {"role": m.get("role"), "content": m.get("content")} for m in history - if m.get("role") in ("user", "assistant") and m.get("content") + if m.get("role") in ("user", "assistant") and content_to_text(m.get("content")) ] # Read live memory state from disk so the flush agent can see @@ -2478,20 +2480,20 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str): context_prompt += f"\n\n{vc_context}" # ----------------------------------------------------------------- - # Auto-analyze images sent by the user + # Image input handling # - # If the user attached image(s), we run the vision tool eagerly so - # the conversation model always receives a text description. The - # local file path is also included so the model can re-examine the - # image later with a more targeted question via vision_analyze. + # If the resolved turn model supports vision, preserve the user's + # message as native multimodal content. Otherwise fall back to eager + # vision analysis so non-vision models still receive a text summary. # # We filter to image paths only (by media_type) so that non-image # attachments (documents, audio, etc.) are not sent to the vision - # tool even when they appear in the same message. + # path even when they appear in the same message. # ----------------------------------------------------------------- - message_text = event.text or "" + message_text: Any = event.text or "" if event.media_urls: image_paths = [] + image_payloads = [] for i, path in enumerate(event.media_urls): # Check media_types if available; otherwise infer from message type mtype = event.media_types[i] if i < len(event.media_types) else "" @@ -2501,10 +2503,69 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str): ) if is_image: image_paths.append(path) + image_payloads.append((path, mtype)) if image_paths: - message_text = await self._enrich_message_with_vision( - message_text, image_paths - ) + resolved_turn_model = None + try: + user_config = _load_gateway_config() + model = _resolve_gateway_model(user_config) + runtime_kwargs = _resolve_runtime_agent_kwargs() + route_probe_parts: List[Dict[str, Any]] = [] + if event.text: + route_probe_parts.append({"type": "text", "text": event.text}) + route_probe_parts.extend( + {"type": "image_url", "image_url": {"url": path}} + for path in image_paths + ) + route_probe_message = ( + route_probe_parts if route_probe_parts else (event.text or "") + ) + resolved_turn_model = ( + self._resolve_turn_agent_config( + content_to_text( + route_probe_message, + image_placeholder="[image]", + fallback_json=True, + ), + model, + runtime_kwargs, + ).get("model") + or model + ) + + from agent.model_metadata import get_model_capabilities + + supports_vision = bool( + get_model_capabilities(resolved_turn_model).get("supports_vision") + ) + except Exception: + supports_vision = False + try: + resolved_turn_model = _resolve_gateway_model(_load_gateway_config()) + except Exception: + resolved_turn_model = None + + if supports_vision: + multimodal_parts: List[Dict[str, Any]] = [] + if event.text: + multimodal_parts.append({"type": "text", "text": event.text}) + for path, mtype in image_payloads: + data_url = image_path_to_data_url(path, mtype) + if not data_url: + continue + multimodal_parts.append( + { + "type": "image_url", + "image_url": {"url": data_url, "detail": "auto"}, + } + ) + if multimodal_parts: + message_text = multimodal_parts + else: + message_text = await self._enrich_message_with_vision( + content_to_text(message_text), + image_paths, + ) # ----------------------------------------------------------------- # Auto-transcribe voice/audio messages sent by the user @@ -2532,7 +2593,8 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str): "can't listen", "VOICE_TOOLS_OPENAI_KEY", ) - if any(m in message_text for m in _stt_fail_markers): + _message_text_plain = content_to_text(message_text, fallback_json=True) + if any(m in _message_text_plain for m in _stt_fail_markers): _stt_adapter = self.adapters.get(source.platform) _stt_meta = {"thread_id": source.thread_id} if source.thread_id else None if _stt_adapter: @@ -2585,7 +2647,10 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str): f"The file is saved at: {path}. " f"Ask the user what they'd like you to do with it.]" ) - message_text = f"{context_note}\n\n{message_text}" + if isinstance(message_text, list): + message_text = [{"type": "text", "text": context_note}] + message_text + else: + message_text = f"{context_note}\n\n{message_text}" # ----------------------------------------------------------------- # Inject reply context when user replies to a message not in history. @@ -2597,12 +2662,16 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str): if getattr(event, 'reply_to_text', None) and event.reply_to_message_id: reply_snippet = event.reply_to_text[:500] found_in_history = any( - reply_snippet[:200] in (msg.get("content") or "") + reply_snippet[:200] in content_to_text(msg.get("content")) for msg in history if msg.get("role") in ("assistant", "user", "tool") ) if not found_in_history: - message_text = f'[Replying to: "{reply_snippet}"]\n\n{message_text}' + reply_note = f'[Replying to: "{reply_snippet}"]' + if isinstance(message_text, list): + message_text = [{"type": "text", "text": reply_note}] + message_text + else: + message_text = f"{reply_note}\n\n{message_text}" try: # Emit agent:start hook @@ -2610,18 +2679,27 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str): "platform": source.platform.value if source.platform else "", "user_id": source.user_id, "session_id": session_entry.session_id, - "message": message_text[:500], + "message": content_to_text(message_text, fallback_json=True)[:500], } await self.hooks.emit("agent:start", hook_ctx) # Expand @ context references (@file:, @folder:, @diff, etc.) - if "@" in message_text: + if isinstance(message_text, str) and "@" in message_text: try: from agent.context_references import preprocess_context_references_async from agent.model_metadata import get_model_context_length _msg_cwd = os.environ.get("MESSAGING_CWD", os.path.expanduser("~")) + _msg_model = _resolve_gateway_model(_load_gateway_config()) + _msg_base_url = "" + try: + _msg_runtime = _resolve_runtime_agent_kwargs() + _msg_base_url = _msg_runtime.get("base_url") or "" + except Exception: + pass _msg_ctx_len = get_model_context_length( - self._model, base_url=self._base_url or "") + _msg_model, + base_url=_msg_base_url, + ) _ctx_result = await preprocess_context_references_async( message_text, cwd=_msg_cwd, context_length=_msg_ctx_len, allowed_root=_msg_cwd) @@ -4396,7 +4474,7 @@ async def _handle_compress_command(self, event: MessageEvent) -> str: msgs = [ {"role": m.get("role"), "content": m.get("content")} for m in history - if m.get("role") in ("user", "assistant") and m.get("content") + if m.get("role") in ("user", "assistant") and content_to_text(m.get("content")) ] original_count = len(msgs) approx_tokens = estimate_messages_tokens_rough(msgs) @@ -4596,7 +4674,7 @@ async def _handle_usage_command(self, event: MessageEvent) -> str: history = self.session_store.load_transcript(session_entry.session_id) if history: from agent.model_metadata import estimate_messages_tokens_rough - msgs = [m for m in history if m.get("role") in ("user", "assistant") and m.get("content")] + msgs = [m for m in history if m.get("role") in ("user", "assistant") and content_to_text(m.get("content"))] approx = estimate_messages_tokens_rough(msgs) return ( f"๐Ÿ“Š **Session Info**\n" @@ -5071,10 +5149,10 @@ async def _enrich_message_with_vision( Auto-analyze user-attached images with the vision tool and prepend the descriptions to the message text. - Each image is analyzed with a general-purpose prompt. The resulting - description *and* the local cache path are injected so the model can: + Each image is analyzed with a general-purpose prompt. The resulting + description and local cache path are injected so non-vision models can: 1. Immediately understand what the user sent (no extra tool call). - 2. Re-examine the image with vision_analyze if it needs more detail. + 2. Re-open the original image with read_file if they need the file again. Args: user_text: The user's original caption / message text. @@ -5105,21 +5183,21 @@ async def _enrich_message_with_vision( description = result.get("analysis", "") enriched_parts.append( f"[The user sent an image~ Here's what I can see:\n{description}]\n" - f"[If you need a closer look, use vision_analyze with " - f"image_url: {path} ~]" + f"[If you need the original file again, use read_file with " + f"path: {path} ~]" ) else: enriched_parts.append( "[The user sent an image but I couldn't quite see it " "this time (>_<) You can try looking at it yourself " - f"with vision_analyze using image_url: {path}]" + f"with read_file using path: {path}]" ) except Exception as e: logger.error("Vision auto-analysis error: %s", e) enriched_parts.append( f"[The user sent an image but something went wrong when I " f"tried to look at it~ You can try examining it yourself " - f"with vision_analyze using image_url: {path}]" + f"with read_file using path: {path}]" ) # Combine: vision descriptions first, then the user's original text @@ -5359,7 +5437,7 @@ def _evict_cached_agent(self, session_key: str) -> None: async def _run_agent( self, - message: str, + message: Any, context_prompt: str, history: List[Dict[str, Any]], source: SessionSource, @@ -5680,7 +5758,11 @@ def run_sync(): except Exception as _sc_err: logger.debug("Could not set up stream consumer: %s", _sc_err) - turn_route = self._resolve_turn_agent_config(message, model, runtime_kwargs) + turn_route = self._resolve_turn_agent_config( + content_to_text(message, image_placeholder="[image]", fallback_json=True), + model, + runtime_kwargs, + ) # Check agent cache โ€” reuse the AIAgent from the previous message # in this session to preserve the frozen system prompt and tool @@ -5798,11 +5880,17 @@ def _bg_review_send(message: str) -> None: else: # Simple text message - just need role and content content = msg.get("content") - if content: + if content_to_text(content): # Tag cross-platform mirror messages so the agent knows their origin if msg.get("mirror"): mirror_src = msg.get("mirror_source", "another session") - content = f"[Delivered from {mirror_src}] {content}" + if isinstance(content, list): + content = [ + {"type": "text", "text": f"[Delivered from {mirror_src}]"}, + *content, + ] + else: + content = f"[Delivered from {mirror_src}] {content}" entry = {"role": role, "content": content} # Preserve reasoning fields on assistant messages so # multi-turn reasoning context survives session reload. @@ -5822,7 +5910,7 @@ def _bg_review_send(message: str) -> None: _history_media_paths: set = set() for _hm in agent_history: if _hm.get("role") in ("tool", "function"): - _hc = _hm.get("content", "") + _hc = content_to_text(_hm.get("content"), fallback_json=True) if "MEDIA:" in _hc: for _match in re.finditer(r'MEDIA:(\S+)', _hc): _p = _match.group(1).strip().rstrip('",}') @@ -5879,7 +5967,7 @@ def _bg_review_send(message: str) -> None: has_voice_directive = False for msg in result.get("messages", []): if msg.get("role") in ("tool", "function"): - content = msg.get("content", "") + content = content_to_text(msg.get("content"), fallback_json=True) if "MEDIA:" in content: for match in re.finditer(r'MEDIA:(\S+)', content): path = match.group(1).strip().rstrip('",}') diff --git a/hermes_cli/models.py b/hermes_cli/models.py index df58df02f8bf5..15f7526e72ad1 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -14,6 +14,9 @@ from difflib import get_close_matches from typing import Any, Optional +# In-memory cache for full model catalog data with capabilities +_MODEL_CATALOG_CACHE: dict[str, list[dict[str, Any]]] = {} + COPILOT_BASE_URL = "https://api.githubcopilot.com" COPILOT_MODELS_URL = f"{COPILOT_BASE_URL}/models" COPILOT_EDITOR_VERSION = "vscode/1.104.1" @@ -1240,3 +1243,28 @@ def validate_requested_model( f"If the service isn't down, this model may not be valid." ), } + + +def get_model_capabilities(provider: Optional[str], model: str) -> dict[str, bool]: + """Query model capabilities from cached catalog data. + + Returns dict with keys: supports_vision, supports_audio + Falls back to empty capabilities if not found in cache. + """ + normalized = normalize_provider(provider) + catalog = _MODEL_CATALOG_CACHE.get(normalized, []) + + for item in catalog: + item_id = str(item.get("id", "")).lower() + if item_id == model.lower(): + # Extract from OpenRouter-style architecture.input_modalities + arch = item.get("architecture", {}) + if isinstance(arch, dict): + mods = arch.get("input_modalities", []) + if isinstance(mods, list): + return { + "supports_vision": "image" in mods, + "supports_audio": "audio" in mods, + } + + return {"supports_vision": False, "supports_audio": False} diff --git a/hermes_state.py b/hermes_state.py index af74ed6ff78d5..002e20021ace3 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -23,6 +23,7 @@ import threading import time from pathlib import Path +from agent.message_content import deserialize_message_content, serialize_message_content from hermes_constants import get_hermes_home from typing import Any, Callable, Dict, List, Optional, TypeVar @@ -32,7 +33,7 @@ DEFAULT_DB_PATH = get_hermes_home() / "state.db" -SCHEMA_VERSION = 6 +SCHEMA_VERSION = 7 SCHEMA_SQL = """ CREATE TABLE IF NOT EXISTS schema_version ( @@ -74,6 +75,7 @@ session_id TEXT NOT NULL REFERENCES sessions(id), role TEXT NOT NULL, content TEXT, + content_json TEXT, tool_call_id TEXT, tool_calls TEXT, tool_name TEXT, @@ -330,6 +332,12 @@ def _init_schema(self): except sqlite3.OperationalError: pass # Column already exists cursor.execute("UPDATE schema_version SET version = 6") + if current_version < 7: + try: + cursor.execute('ALTER TABLE messages ADD COLUMN "content_json" TEXT') + except sqlite3.OperationalError: + pass + cursor.execute("UPDATE schema_version SET version = 7") # Unique title index โ€” always ensure it exists (safe to run after migrations # since the title column is guaranteed to exist at this point) @@ -859,7 +867,7 @@ def append_message( self, session_id: str, role: str, - content: str = None, + content: Any = None, tool_name: str = None, tool_calls: Any = None, tool_call_id: str = None, @@ -876,6 +884,7 @@ def append_message( if role is 'tool' or tool_calls is present). """ # Serialize structured fields to JSON before entering the write txn + content_text, content_json = serialize_message_content(content) reasoning_details_json = ( json.dumps(reasoning_details) if reasoning_details else None @@ -893,14 +902,15 @@ def append_message( def _do(conn): cursor = conn.execute( - """INSERT INTO messages (session_id, role, content, tool_call_id, + """INSERT INTO messages (session_id, role, content, content_json, tool_call_id, tool_calls, tool_name, timestamp, token_count, finish_reason, reasoning, reasoning_details, codex_reasoning_items) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", ( session_id, role, - content, + content_text, + content_json, tool_call_id, tool_calls_json, tool_name, @@ -941,6 +951,10 @@ def get_messages(self, session_id: str) -> List[Dict[str, Any]]: result = [] for row in rows: msg = dict(row) + msg["content"] = deserialize_message_content( + msg.get("content"), + msg.get("content_json"), + ) if msg.get("tool_calls"): try: msg["tool_calls"] = json.loads(msg["tool_calls"]) @@ -956,7 +970,7 @@ def get_messages_as_conversation(self, session_id: str) -> List[Dict[str, Any]]: """ with self._lock: cursor = self._conn.execute( - "SELECT role, content, tool_call_id, tool_calls, tool_name, " + "SELECT role, content, content_json, tool_call_id, tool_calls, tool_name, " "reasoning, reasoning_details, codex_reasoning_items " "FROM messages WHERE session_id = ? ORDER BY timestamp, id", (session_id,), @@ -964,7 +978,10 @@ def get_messages_as_conversation(self, session_id: str) -> List[Dict[str, Any]]: rows = cursor.fetchall() messages = [] for row in rows: - msg = {"role": row["role"], "content": row["content"]} + msg = { + "role": row["role"], + "content": deserialize_message_content(row["content"], row["content_json"]), + } if row["tool_call_id"]: msg["tool_call_id"] = row["tool_call_id"] if row["tool_name"]: diff --git a/model_tools.py b/model_tools.py index 15b8852bcc582..23e4522d00541 100644 --- a/model_tools.py +++ b/model_tools.py @@ -235,6 +235,7 @@ def get_tool_definitions( enabled_toolsets: List[str] = None, disabled_toolsets: List[str] = None, quiet_mode: bool = False, + omit_vision_analyze: bool = False, ) -> List[Dict[str, Any]]: """ Get tool definitions for model API calls with toolset-based filtering. @@ -245,6 +246,9 @@ def get_tool_definitions( enabled_toolsets: Only include tools from these toolsets. disabled_toolsets: Exclude tools from these toolsets (if enabled_toolsets is None). quiet_mode: Suppress status prints. + omit_vision_analyze: Hide the auxiliary `vision_analyze` tool from + the model-facing tool surface. Use this when the current model + accepts native image input directly. Returns: Filtered list of OpenAI-format tool definitions. @@ -301,6 +305,13 @@ def get_tool_definitions( # Ask the registry for schemas (only returns tools whose check_fn passes) filtered_tools = registry.get_definitions(tools_to_include, quiet=quiet_mode) + if omit_vision_analyze: + filtered_tools = [ + tool + for tool in filtered_tools + if tool.get("function", {}).get("name") != "vision_analyze" + ] + # The set of tool names that actually passed check_fn filtering. # Use this (not tools_to_include) for any downstream schema that references # other tools by name โ€” otherwise the model sees tools mentioned in diff --git a/run_agent.py b/run_agent.py index 13159b7b7eb27..0798d7041aa76 100644 --- a/run_agent.py +++ b/run_agent.py @@ -88,6 +88,11 @@ save_context_length, ) from agent.context_compressor import ContextCompressor +from agent.message_content import ( + content_has_image_parts, + content_to_text, + convert_content_to_responses_input, +) from agent.prompt_caching import apply_anthropic_cache_control from agent.prompt_builder import build_skills_system_prompt, build_context_files_prompt, load_soul_md, TOOL_USE_ENFORCEMENT_GUIDANCE, TOOL_USE_ENFORCEMENT_MODELS, DEVELOPER_ROLE_MODELS from agent.usage_pricing import estimate_usage_cost, normalize_usage @@ -933,11 +938,22 @@ def __init__( print(f"๐Ÿ”„ Fallback chain ({len(self._fallback_chain)} providers): " + " โ†’ ".join(f"{f['model']} ({f['provider']})" for f in self._fallback_chain)) + # Hide the auxiliary image tool when the model already accepts native + # image input. That prevents the tool loop from re-routing images back + # through vision_analyze after multimodal content has already been passed. + from agent.model_metadata import get_model_capabilities + + supports_native_vision = bool( + get_model_capabilities(self.model, provider=self.provider).get("supports_vision") + ) + self._supports_native_vision = supports_native_vision + # Get available tools with filtering self.tools = get_tool_definitions( enabled_toolsets=enabled_toolsets, disabled_toolsets=disabled_toolsets, quiet_mode=self.quiet_mode, + omit_vision_analyze=True, ) # Show tool configuration and store valid tool names for validation @@ -1884,10 +1900,11 @@ def _convert_to_trajectory_format(self, messages: List[Dict[str, Any]], user_que if msg.get("reasoning") and msg["reasoning"].strip(): content = f"\n{msg['reasoning']}\n\n" - if msg.get("content") and msg["content"].strip(): + _assistant_text = content_to_text(msg.get("content"), fallback_json=True) + if _assistant_text.strip(): # Convert any tags to tags # (used when native thinking is disabled and model reasons via XML) - content += convert_scratchpad_to_think(msg["content"]) + "\n" + content += convert_scratchpad_to_think(_assistant_text) + "\n" # Add tool calls wrapped in XML tags for tool_call in msg["tool_calls"]: @@ -1968,7 +1985,7 @@ def _convert_to_trajectory_format(self, messages: List[Dict[str, Any]], user_que # Convert any tags to tags # (used when native thinking is disabled and model reasons via XML) - raw_content = msg["content"] or "" + raw_content = content_to_text(msg.get("content"), fallback_json=True) content += convert_scratchpad_to_think(raw_content) # Ensure every gpt turn has a block (empty if no reasoning) @@ -1983,7 +2000,7 @@ def _convert_to_trajectory_format(self, messages: List[Dict[str, Any]], user_que elif msg["role"] == "user": trajectory.append({ "from": "human", - "value": msg["content"] + "value": content_to_text(msg.get("content"), fallback_json=True) }) i += 1 @@ -2152,9 +2169,6 @@ def _dump_api_request_debug( self._vprint(f"{self.log_prefix}๐Ÿงพ Request debug dump written to: {dump_file}") - if env_var_enabled("HERMES_DUMP_REQUEST_STDOUT"): - print(json.dumps(dump_payload, ensure_ascii=False, indent=2, default=str)) - return dump_file except Exception as dump_error: if self.verbose_logging: @@ -2162,9 +2176,9 @@ def _dump_api_request_debug( return None @staticmethod - def _clean_session_content(content: str) -> str: + def _clean_session_content(content: Any) -> Any: """Convert REASONING_SCRATCHPAD to think tags and clean up whitespace.""" - if not content: + if not isinstance(content, str) or not content: return content content = convert_scratchpad_to_think(content) content = re.sub(r'\n+()', r'\n\1', content) @@ -2393,10 +2407,17 @@ def _activate_honcho( # Rebuild tool surface after Honcho context injection. Tool availability # is check_fn-gated and may change once session context is attached. + from agent.model_metadata import get_model_capabilities + + supports_native_vision = bool( + get_model_capabilities(self.model, provider=self.provider).get("supports_vision") + ) + self._supports_native_vision = supports_native_vision self.tools = get_tool_definitions( enabled_toolsets=enabled_toolsets, disabled_toolsets=disabled_toolsets, quiet_mode=True, + omit_vision_analyze=True, ) self.valid_tool_names = { tool["function"]["name"] for tool in self.tools @@ -2998,7 +3019,10 @@ def _chat_messages_to_responses_input(self, messages: List[Dict[str, Any]]) -> L if role in {"user", "assistant"}: content = msg.get("content", "") - content_text = str(content) if content is not None else "" + content_text = ( + content if isinstance(content, str) + else content_to_text(content, image_placeholder="[image]", fallback_json=True) + ) if role == "assistant": # Replay encrypted reasoning items from previous turns @@ -3064,7 +3088,7 @@ def _chat_messages_to_responses_input(self, messages: List[Dict[str, Any]]) -> L }) continue - items.append({"role": role, "content": content_text}) + items.append({"role": role, "content": convert_content_to_responses_input(content)}) continue if role == "tool": @@ -3075,10 +3099,15 @@ def _chat_messages_to_responses_input(self, messages: List[Dict[str, Any]]) -> L call_id = raw_tool_call_id.strip() if not isinstance(call_id, str) or not call_id.strip(): continue + output = msg.get("content", "") + if output is None: + output = "" + elif not isinstance(output, str): + output = convert_content_to_responses_input(output) items.append({ "type": "function_call_output", "call_id": call_id, - "output": str(msg.get("content", "") or ""), + "output": output, }) return items @@ -3125,8 +3154,11 @@ def _preflight_codex_input_items(self, raw_items: Any) -> List[Dict[str, Any]]: output = item.get("output", "") if output is None: output = "" - if not isinstance(output, str): - output = str(output) + elif not isinstance(output, str): + if isinstance(output, list): + output = convert_content_to_responses_input(output) + else: + output = str(output) normalized.append( { @@ -3157,8 +3189,8 @@ def _preflight_codex_input_items(self, raw_items: Any) -> List[Dict[str, Any]]: content = item.get("content", "") if content is None: content = "" - if not isinstance(content, str): - content = str(content) + elif not isinstance(content, str): + content = convert_content_to_responses_input(content) normalized.append({"role": role, "content": content}) continue @@ -4769,12 +4801,7 @@ def _try_activate_fallback(self) -> bool: @staticmethod def _content_has_image_parts(content: Any) -> bool: - if not isinstance(content, list): - return False - for part in content: - if isinstance(part, dict) and part.get("type") in {"image_url", "input_image"}: - return True - return False + return content_has_image_parts(content) @staticmethod def _materialize_data_url_for_vision(image_url: str) -> tuple[str, Optional[Path]]: @@ -4842,12 +4869,107 @@ def _describe_image_for_anthropic_fallback(self, image_url: str, role: str) -> s note = f"[The {role_label} attached an image. Here's what it contains:\n{description}]" if vision_source and not str(image_url or "").startswith("data:"): note += ( - f"\n[If you need a closer look, use vision_analyze with image_url: {vision_source}]" + f"\n[If you need to inspect the original file again, use read_file with path: {vision_source}]" ) self._anthropic_image_fallback_cache[cache_key] = note return note + def _process_read_file_image_result( + self, + function_name: str, + function_result: str, + ) -> tuple[Any, Optional[Dict[str, Any]]]: + """Normalize read_file(image) results for the active model capability. + + Returns: + (tool_content_text, optional_followup_message) + + - Native-vision models get a synthetic multimodal follow-up message. + - Non-vision models get automatic auxiliary vision analysis appended + into the tool result text. + - In all cases, inline base64 is removed from the textual tool result to + avoid blowing up the context window. + """ + if function_name != "read_file": + return function_result, None + + try: + payload = json.loads(function_result) + except Exception: + return function_result, None + if not isinstance(payload, dict) or not payload.get("is_image"): + return function_result, None + + if payload.get("error") and not payload.get("base64_content"): + payload = dict(payload) + payload["next_step"] = ( + "Do not try terminal, OCR, PIL, tesseract, file, or other fallback inspection methods. " + "Tell the user directly that Hermes could not attach or inspect this image in the current environment." + ) + return json.dumps(payload, ensure_ascii=False), None + + base64_content = payload.get("base64_content") + mime_type = payload.get("mime_type") or "image/jpeg" + if not isinstance(base64_content, str) or not base64_content.strip(): + return function_result, None + + path = str(payload.get("path") or "").strip() + dimensions = str(payload.get("dimensions") or "").strip() + file_size = payload.get("file_size") + data_url = f"data:{mime_type};base64,{base64_content}" + + payload = dict(payload) + payload["base64_content"] = "[omitted from tool text output]" + + if not getattr(self, "_supports_native_vision", False): + analysis_note = self._describe_image_for_anthropic_fallback(data_url, "tool") + if path: + analysis_note += f"\n[Original image path: {path}]" + payload["analysis"] = analysis_note + return json.dumps(payload, ensure_ascii=False), None + + intro_bits = ["read_file loaded an image file for inspection."] + if path: + intro_bits.append(f"Path: {path}.") + if dimensions: + intro_bits.append(f"Dimensions: {dimensions}.") + if isinstance(file_size, int) and file_size > 0: + intro_bits.append(f"Size: {file_size:,} bytes.") + intro_text = " ".join(intro_bits) + + if self.api_mode in {"codex_responses", "anthropic_messages"}: + return ( + [ + {"type": "text", "text": intro_text}, + { + "type": "image_url", + "image_url": { + "url": data_url, + "detail": "auto", + }, + }, + ], + None, + ) + + return ( + json.dumps(payload, ensure_ascii=False), + { + "role": "user", + "content": [ + {"type": "text", "text": f"[{intro_text} Treat the attached image as the contents of that file, not as a new user request.]"}, + { + "type": "image_url", + "image_url": { + "url": data_url, + "detail": "auto", + }, + }, + ], + }, + ) + def _preprocess_anthropic_content(self, content: Any, role: str) -> Any: if not self._content_has_image_parts(content): return content @@ -5203,7 +5325,7 @@ def _build_assistant_message(self, assistant_message, finish_reason: str) -> dic # reasoning fields are present (some models/providers embed thinking # directly in the content rather than returning separate API fields). if not reasoning_text: - content = assistant_message.content or "" + content = content_to_text(getattr(assistant_message, "content", None), fallback_json=True) think_blocks = re.findall(r'(.*?)', content, flags=re.DOTALL) if think_blocks: combined = "\n\n".join(b.strip() for b in think_blocks if b.strip()) @@ -5227,9 +5349,14 @@ def _build_assistant_message(self, assistant_message, finish_reason: str) -> dic except Exception: pass + assistant_content = ( + assistant_message.content + if getattr(assistant_message, "content", None) is not None + else "" + ) msg = { "role": "assistant", - "content": assistant_message.content or "", + "content": assistant_content, "reasoning": reasoning_text, "finish_reason": finish_reason, } @@ -5797,16 +5924,24 @@ def _run_tool(index, tool_call, function_name, function_args): # Shouldn't happen, but safety fallback function_result = f"Error executing tool '{name}': thread did not return a result" tool_duration = 0.0 + function_name = name else: function_name, function_args, function_result, tool_duration, is_error = r - if is_error: - result_preview = function_result[:200] if len(function_result) > 200 else function_result - logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, result_preview) + function_result, read_file_followup = self._process_read_file_image_result(name, function_result) + _result_text = ( + function_result + if isinstance(function_result, str) + else content_to_text(function_result, image_placeholder="[image]", fallback_json=True) + ) - if self.verbose_logging: - logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") - logging.debug(f"Tool result ({len(function_result)} chars): {function_result}") + if r is not None and is_error: + result_preview = _result_text[:200] if len(_result_text) > 200 else _result_text + logger.warning("Tool %s returned error (%.2fs): %s", function_name, tool_duration, result_preview) + + if self.verbose_logging: + logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") + logging.debug(f"Tool result ({len(_result_text)} chars): {_result_text}") # Print cute message per tool if self.quiet_mode: @@ -5815,9 +5950,9 @@ def _run_tool(index, tool_call, function_name, function_args): elif not self.quiet_mode: if self.verbose_logging: print(f" โœ… Tool {i+1} completed in {tool_duration:.2f}s") - print(f" Result: {function_result}") + print(f" Result: {_result_text}") else: - response_preview = function_result[:self.log_prefix_chars] + "..." if len(function_result) > self.log_prefix_chars else function_result + response_preview = _result_text[:self.log_prefix_chars] + "..." if len(_result_text) > self.log_prefix_chars else _result_text print(f" โœ… Tool {i+1} completed in {tool_duration:.2f}s - {response_preview}") if self.tool_complete_callback: @@ -5828,7 +5963,7 @@ def _run_tool(index, tool_call, function_name, function_args): # Truncate oversized results MAX_TOOL_RESULT_CHARS = 100_000 - if len(function_result) > MAX_TOOL_RESULT_CHARS: + if isinstance(function_result, str) and len(function_result) > MAX_TOOL_RESULT_CHARS: original_len = len(function_result) function_result = ( function_result[:MAX_TOOL_RESULT_CHARS] @@ -5843,12 +5978,16 @@ def _run_tool(index, tool_call, function_name, function_args): "tool_call_id": tc.id, } messages.append(tool_msg) + if read_file_followup: + messages.append(read_file_followup) # โ”€โ”€ Budget pressure injection โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ budget_warning = self._get_budget_warning(api_call_count) if budget_warning and messages and messages[-1].get("role") == "tool": last_content = messages[-1]["content"] try: + if not isinstance(last_content, str): + raise TypeError parsed = json.loads(last_content) if isinstance(parsed, dict): parsed["_budget_warning"] = budget_warning @@ -5856,7 +5995,13 @@ def _run_tool(index, tool_call, function_name, function_args): else: messages[-1]["content"] = last_content + f"\n\n{budget_warning}" except (json.JSONDecodeError, TypeError): - messages[-1]["content"] = last_content + f"\n\n{budget_warning}" + if isinstance(last_content, list): + messages[-1]["content"] = [ + {"type": "text", "text": budget_warning}, + *last_content, + ] + else: + messages[-1]["content"] = str(last_content) + f"\n\n{budget_warning}" if not self.quiet_mode: remaining = self.max_iterations - api_call_count tier = "โš ๏ธ WARNING" if remaining <= self.max_iterations * 0.1 else "๐Ÿ’ก CAUTION" @@ -6070,8 +6215,14 @@ def _execute_tool_calls_sequential(self, assistant_message, messages: list, effe logger.error("handle_function_call raised for %s: %s", function_name, tool_error, exc_info=True) tool_duration = time.time() - tool_start_time - result_preview = function_result if self.verbose_logging else ( - function_result[:200] if len(function_result) > 200 else function_result + function_result, read_file_followup = self._process_read_file_image_result(function_name, function_result) + _result_text = ( + function_result + if isinstance(function_result, str) + else content_to_text(function_result, image_placeholder="[image]", fallback_json=True) + ) + result_preview = _result_text if self.verbose_logging else ( + _result_text[:200] if len(_result_text) > 200 else _result_text ) # Log tool errors to the persistent error log so [error] tags @@ -6082,7 +6233,7 @@ def _execute_tool_calls_sequential(self, assistant_message, messages: list, effe if self.verbose_logging: logging.debug(f"Tool {function_name} completed in {tool_duration:.2f}s") - logging.debug(f"Tool result ({len(function_result)} chars): {function_result}") + logging.debug(f"Tool result ({len(_result_text)} chars): {_result_text}") if self.tool_complete_callback: try: @@ -6095,7 +6246,7 @@ def _execute_tool_calls_sequential(self, assistant_message, messages: list, effe # enough for any reasonable tool output but prevents catastrophic # context explosions (e.g. accidental base64 image dumps). MAX_TOOL_RESULT_CHARS = 100_000 - if len(function_result) > MAX_TOOL_RESULT_CHARS: + if isinstance(function_result, str) and len(function_result) > MAX_TOOL_RESULT_CHARS: original_len = len(function_result) function_result = ( function_result[:MAX_TOOL_RESULT_CHARS] @@ -6109,13 +6260,15 @@ def _execute_tool_calls_sequential(self, assistant_message, messages: list, effe "tool_call_id": tool_call.id } messages.append(tool_msg) + if read_file_followup: + messages.append(read_file_followup) if not self.quiet_mode: if self.verbose_logging: print(f" โœ… Tool {i} completed in {tool_duration:.2f}s") print(f" Result: {function_result}") else: - response_preview = function_result[:self.log_prefix_chars] + "..." if len(function_result) > self.log_prefix_chars else function_result + response_preview = _result_text[:self.log_prefix_chars] + "..." if len(_result_text) > self.log_prefix_chars else _result_text print(f" โœ… Tool {i} completed in {tool_duration:.2f}s - {response_preview}") if self._interrupt_requested and i < len(assistant_message.tool_calls): @@ -6270,7 +6423,10 @@ def _handle_max_iterations(self, messages: list, api_call_count: int) -> str: codex_kwargs.pop("tools", None) summary_response = self._run_codex_stream(codex_kwargs) assistant_message, _ = self._normalize_codex_response(summary_response) - final_response = (assistant_message.content or "").strip() if assistant_message else "" + final_response = content_to_text( + getattr(assistant_message, "content", None) if assistant_message else "", + fallback_json=True, + ).strip() else: summary_kwargs = { "model": self.model, @@ -6410,6 +6566,16 @@ def run_conversation( user_message = _sanitize_surrogates(user_message) if isinstance(persist_user_message, str): persist_user_message = _sanitize_surrogates(persist_user_message) + user_message_text = content_to_text( + user_message, + image_placeholder="[image]", + fallback_json=True, + ) + persist_user_message_text = content_to_text( + persist_user_message, + image_placeholder="[image]", + fallback_json=True, + ) # Store stream callback for _interruptible_api_call to pick up self._stream_callback = stream_callback @@ -6475,6 +6641,9 @@ def run_conversation( # Preserve the original user message (no nudge injection). # Honcho should receive the actual user input, not system nudges. original_user_message = persist_user_message if persist_user_message is not None else user_message + original_user_message_text = ( + persist_user_message_text if persist_user_message is not None else user_message_text + ) # Track memory nudge trigger (turn-based, checked here). # Skill trigger is checked AFTER the agent loop completes, based on @@ -6500,7 +6669,7 @@ def run_conversation( _recall_mode = (self._honcho_config.recall_mode if self._honcho_config else "hybrid") if self._honcho and self._honcho_session_key and _recall_mode != "tools": try: - prefetched_context = self._honcho_prefetch(original_user_message) + prefetched_context = self._honcho_prefetch(original_user_message_text) if prefetched_context: if not conversation_history: self._honcho_context = prefetched_context @@ -6516,7 +6685,9 @@ def run_conversation( self._persist_user_message_idx = current_turn_user_idx if not self.quiet_mode: - self._safe_print(f"๐Ÿ’ฌ Starting conversation: '{user_message[:60]}{'...' if len(user_message) > 60 else ''}'") + self._safe_print( + f"๐Ÿ’ฌ Starting conversation: '{user_message_text[:60]}{'...' if len(user_message_text) > 60 else ''}'" + ) # โ”€โ”€ System prompt (cached per session for prefix caching) โ”€โ”€ # Built once on first call, reused for all subsequent calls. @@ -6838,9 +7009,6 @@ def run_conversation( if self.api_mode == "codex_responses": api_kwargs = self._preflight_codex_api_kwargs(api_kwargs, allow_stream=False) - if env_var_enabled("HERMES_DUMP_REQUESTS"): - self._dump_api_request_debug(api_kwargs, reason="preflight") - # Always prefer the streaming path โ€” even without stream # consumers. Streaming gives us fine-grained health # checking (90s stale-stream detection, 60s read timeout) @@ -7102,8 +7270,12 @@ def _stop_spinner(): length_continue_retries += 1 interim_msg = self._build_assistant_message(assistant_message, finish_reason) messages.append(interim_msg) - if assistant_message.content: - truncated_response_prefix += assistant_message.content + _trunc_text = content_to_text( + getattr(assistant_message, "content", None), + fallback_json=True, + ) + if _trunc_text: + truncated_response_prefix += _trunc_text if length_continue_retries < 3: self._vprint( @@ -7803,41 +7975,26 @@ def _stop_spinner(): else: assistant_message = response.choices[0].message - # Normalize content to string โ€” some OpenAI-compatible servers - # (llama-server, etc.) return content as a dict or list instead - # of a plain string, which crashes downstream .strip() calls. - if assistant_message.content is not None and not isinstance(assistant_message.content, str): - raw = assistant_message.content - if isinstance(raw, dict): - assistant_message.content = raw.get("text", "") or raw.get("content", "") or json.dumps(raw) - elif isinstance(raw, list): - # Multimodal content list โ€” extract text parts - parts = [] - for part in raw: - if isinstance(part, str): - parts.append(part) - elif isinstance(part, dict) and part.get("type") == "text": - parts.append(part.get("text", "")) - elif isinstance(part, dict) and "text" in part: - parts.append(str(part["text"])) - assistant_message.content = "\n".join(parts) - else: - assistant_message.content = str(raw) + assistant_content_text = content_to_text( + getattr(assistant_message, "content", None), + image_placeholder="[image]", + fallback_json=True, + ) # Handle assistant response - if assistant_message.content and not self.quiet_mode: + if assistant_content_text and not self.quiet_mode: if self.verbose_logging: - self._vprint(f"{self.log_prefix}๐Ÿค– Assistant: {assistant_message.content}") + self._vprint(f"{self.log_prefix}๐Ÿค– Assistant: {assistant_content_text}") else: - self._vprint(f"{self.log_prefix}๐Ÿค– Assistant: {assistant_message.content[:100]}{'...' if len(assistant_message.content) > 100 else ''}") + self._vprint(f"{self.log_prefix}๐Ÿค– Assistant: {assistant_content_text[:100]}{'...' if len(assistant_content_text) > 100 else ''}") # Notify progress callback of model's thinking (used by subagent # delegation to relay the child's reasoning to the parent display). # Guard: only fire for subagents (_delegate_depth >= 1) to avoid # spamming gateway platforms with the main agent's every thought. - if (assistant_message.content and self.tool_progress_callback + if (assistant_content_text and self.tool_progress_callback and getattr(self, '_delegate_depth', 0) > 0): - _think_text = assistant_message.content.strip() + _think_text = assistant_content_text.strip() # Strip reasoning XML tags that shouldn't leak to parent display _think_text = re.sub( r'', '', _think_text @@ -7851,7 +8008,7 @@ def _stop_spinner(): # Check for incomplete (opened but never closed) # This means the model ran out of output tokens mid-reasoning โ€” retry up to 2 times - if has_incomplete_scratchpad(assistant_message.content or ""): + if has_incomplete_scratchpad(assistant_content_text): if not hasattr(self, '_incomplete_scratchpad_retries'): self._incomplete_scratchpad_retries = 0 self._incomplete_scratchpad_retries += 1 @@ -7890,7 +8047,7 @@ def _stop_spinner(): self._codex_incomplete_retries += 1 interim_msg = self._build_assistant_message(assistant_message, finish_reason) - interim_has_content = bool((interim_msg.get("content") or "").strip()) + interim_has_content = bool(content_to_text(interim_msg.get("content")).strip()) interim_has_reasoning = bool(interim_msg.get("reasoning", "").strip()) if isinstance(interim_msg.get("reasoning"), str) else False interim_has_codex_reasoning = bool(interim_msg.get("codex_reasoning_items")) @@ -7907,7 +8064,7 @@ def _stop_spinner(): isinstance(last_msg, dict) and last_msg.get("role") == "assistant" and last_msg.get("finish_reason") == "incomplete" - and (last_msg.get("content") or "") == (interim_msg.get("content") or "") + and content_to_text(last_msg.get("content")) == content_to_text(interim_msg.get("content")) and (last_msg.get("reasoning") or "") == (interim_msg.get("reasoning") or "") and last_codex_items == interim_codex_items ) @@ -8074,7 +8231,7 @@ def _stop_spinner(): # as a fallback final response. Common pattern: model delivers its # answer and calls memory/skill tools as a side-effect in the same # turn. If the follow-up turn after tools is empty, we use this. - turn_content = assistant_message.content or "" + turn_content = assistant_content_text if turn_content and self._has_content_after_think_block(turn_content): self._last_content_with_tools = turn_content # Only mute subsequent output when EVERY tool call in @@ -8173,7 +8330,7 @@ def _stop_spinner(): else: # No tool calls - this is the final response - final_response = assistant_message.content or "" + final_response = assistant_content_text # Check if response only has think block with no actual content after it if not self._has_content_after_think_block(final_response): @@ -8288,7 +8445,7 @@ def _stop_spinner(): and self.valid_tool_names and codex_ack_continuations < 2 and self._looks_like_codex_intermediate_ack( - user_message=user_message, + user_message=user_message_text, assistant_content=final_response, messages=messages, ) @@ -8391,7 +8548,7 @@ def _stop_spinner(): completed = final_response is not None and api_call_count < self.max_iterations # Save trajectory if enabled - self._save_trajectory(messages, user_message, completed) + self._save_trajectory(messages, user_message_text, completed) # Clean up VM and browser for this task after conversation completes self._cleanup_task_resources(effective_task_id) @@ -8401,8 +8558,8 @@ def _stop_spinner(): # Sync conversation to Honcho for user modeling if final_response and not interrupted and sync_honcho: - self._honcho_sync(original_user_message, final_response) - self._queue_honcho_prefetch(original_user_message) + self._honcho_sync(original_user_message_text, final_response) + self._queue_honcho_prefetch(original_user_message_text) # Plugin hook: post_llm_call # Fired once per turn after the tool-calling loop completes. @@ -8414,7 +8571,7 @@ def _stop_spinner(): _invoke_hook( "post_llm_call", session_id=self.session_id, - user_message=original_user_message, + user_message=original_user_message_text, assistant_response=final_response, conversation_history=list(messages), model=self.model, diff --git a/tests/agent/test_display.py b/tests/agent/test_display.py new file mode 100644 index 0000000000000..32ee0aa569492 --- /dev/null +++ b/tests/agent/test_display.py @@ -0,0 +1,13 @@ +from agent.display import _detect_tool_failure + + +def test_detect_tool_failure_accepts_structured_tool_results(): + result = [ + {"type": "text", "text": "Image loaded from read_file."}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,QUFB", "detail": "auto"}}, + ] + + is_failure, suffix = _detect_tool_failure("read_file", result) + + assert is_failure is False + assert suffix == "" diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index 51a4c887393d9..477200482b855 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -13,6 +13,7 @@ import os import time import tempfile +import base64 import pytest import yaml @@ -58,9 +59,17 @@ def test_proportional(self): assert long > short def test_unicode_multibyte(self): - """Unicode chars are still 1 Python char each โ€” 4 chars/token holds.""" + """Non-ASCII text should be estimated conservatively, not //4.""" text = "ไฝ ๅฅฝไธ–็•Œ" # 4 CJK characters - assert estimate_tokens_rough(text) == 1 + assert estimate_tokens_rough(text) == 4 + + def test_japanese_text_not_severely_underestimated(self): + text = "ใ“ใ‚Œใฏๆ—ฅๆœฌ่ชžใฎๆ–‡็ซ ใงใ™" + assert estimate_tokens_rough(text) == len(text) + + def test_mixed_ascii_and_cjk_uses_conservative_max(self): + text = "helloไฝ ๅฅฝ" + assert estimate_tokens_rough(text) == 3 class TestEstimateMessagesTokensRough: @@ -71,7 +80,7 @@ def test_single_message_concrete_value(self): """Verify against known str(msg) length.""" msg = {"role": "user", "content": "a" * 400} result = estimate_messages_tokens_rough([msg]) - expected = len(str(msg)) // 4 + expected = estimate_tokens_rough(str(msg)) assert result == expected def test_multiple_messages_additive(self): @@ -80,7 +89,7 @@ def test_multiple_messages_additive(self): {"role": "assistant", "content": "Hi there, how can I help?"}, ] result = estimate_messages_tokens_rough(msgs) - expected = sum(len(str(m)) for m in msgs) // 4 + expected = estimate_tokens_rough("".join(str(m) for m in msgs)) assert result == expected def test_tool_call_message(self): @@ -89,7 +98,7 @@ def test_tool_call_message(self): "tool_calls": [{"id": "1", "function": {"name": "terminal", "arguments": "{}"}}]} result = estimate_messages_tokens_rough([msg]) assert result > 0 - assert result == len(str(msg)) // 4 + assert result == estimate_tokens_rough(str(msg)) def test_message_with_list_content(self): """Vision messages with multimodal content arrays.""" @@ -98,7 +107,50 @@ def test_message_with_list_content(self): {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}} ]} result = estimate_messages_tokens_rough([msg]) - assert result == len(str(msg)) // 4 + expected = estimate_tokens_rough(str({"role": "user", "content": "describe\n[image]"})) + 1600 + assert result == expected + + def test_message_with_low_detail_image_uses_gpt54_budget(self): + png_bytes = ( + b"\x89PNG\r\n\x1a\n" + b"\x00\x00\x00\rIHDR" + b"\x00\x00\x04\x00" # width = 1024 + b"\x00\x00\x04\x00" # height = 1024 + b"\x08\x02\x00\x00\x00" + ) + data_url = "data:image/png;base64," + base64.b64encode(png_bytes).decode("ascii") + msg = { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + {"type": "image_url", "image_url": {"url": data_url, "detail": "low"}}, + ], + } + + result = estimate_messages_tokens_rough([msg]) + expected = estimate_tokens_rough(str({"role": "user", "content": "describe\n[image]"})) + 256 + assert result == expected + + def test_message_with_large_high_detail_image_uses_patch_count(self): + png_bytes = ( + b"\x89PNG\r\n\x1a\n" + b"\x00\x00\x00\rIHDR" + b"\x00\x00\x07\x08" # width = 1800 + b"\x00\x00\x09\x60" # height = 2400 + b"\x08\x02\x00\x00\x00" + ) + data_url = "data:image/png;base64," + base64.b64encode(png_bytes).decode("ascii") + msg = { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + {"type": "image_url", "image_url": {"url": data_url, "detail": "high"}}, + ], + } + + result = estimate_messages_tokens_rough([msg]) + expected = estimate_tokens_rough(str({"role": "user", "content": "describe\n[image]"})) + 2451 + assert result == expected # ========================================================================= @@ -633,3 +685,75 @@ def test_special_chars_in_model_name(self, tmp_path): with patch("agent.model_metadata._get_context_cache_path", return_value=cache_file): save_context_length(model, url, 200000) assert get_cached_context_length(model, url) == 200000 + + +# ========================================================================= +# Model capabilities +# ========================================================================= + +class TestModelCapabilities: + def test_extract_vision_capability(self): + """Test that vision capability is extracted from OpenRouter data.""" + from agent.model_metadata import get_model_capabilities + + mock_response = MagicMock() + mock_response.json.return_value = { + "data": [ + { + "id": "test/vision-model", + "context_length": 128000, + "architecture": { + "input_modalities": ["text", "image"] + } + } + ] + } + + with patch("agent.model_metadata.requests.get", return_value=mock_response): + caps = get_model_capabilities("test/vision-model") + assert caps["supports_vision"] is True + assert caps["supports_audio"] is False + + def test_extract_audio_capability(self): + """Test that audio capability is extracted from OpenRouter data.""" + from agent.model_metadata import get_model_capabilities + + mock_response = MagicMock() + mock_response.json.return_value = { + "data": [ + { + "id": "test/audio-model", + "context_length": 128000, + "architecture": { + "input_modalities": ["text", "audio"] + } + } + ] + } + + with patch("agent.model_metadata.requests.get", return_value=mock_response): + caps = get_model_capabilities("test/audio-model") + assert caps["supports_vision"] is False + assert caps["supports_audio"] is True + + def test_no_capabilities(self): + """Test model with no multimodal capabilities.""" + from agent.model_metadata import get_model_capabilities + + mock_response = MagicMock() + mock_response.json.return_value = { + "data": [ + { + "id": "test/text-only", + "context_length": 128000, + "architecture": { + "input_modalities": ["text"] + } + } + ] + } + + with patch("agent.model_metadata.requests.get", return_value=mock_response): + caps = get_model_capabilities("test/text-only") + assert caps["supports_vision"] is False + assert caps["supports_audio"] is False diff --git a/tests/gateway/test_api_server.py b/tests/gateway/test_api_server.py index 5bde076a68fa9..3ecf25b5c3ffe 100644 --- a/tests/gateway/test_api_server.py +++ b/tests/gateway/test_api_server.py @@ -547,6 +547,27 @@ async def test_successful_completion(self, adapter): assert data["choices"][0]["finish_reason"] == "stop" assert "usage" in data + @pytest.mark.asyncio + async def test_chat_completions_preserve_multimodal_content_arrays(self, adapter): + """Structured chat content should reach the agent unchanged.""" + mock_result = {"final_response": "Done", "messages": [], "api_calls": 1} + multimodal = [ + {"type": "text", "text": "Describe this image"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA", "detail": "auto"}}, + ] + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + mock_run.return_value = (mock_result, {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}) + resp = await cli.post( + "/v1/chat/completions", + json={"model": "hermes-agent", "messages": [{"role": "user", "content": multimodal}]}, + ) + + assert resp.status == 200 + assert mock_run.call_args.kwargs["user_message"] == multimodal + @pytest.mark.asyncio async def test_system_prompt_extracted(self, adapter): """System messages from the client are passed as ephemeral_system_prompt.""" @@ -708,6 +729,27 @@ async def test_successful_response_with_array_input(self, adapter): assert call_kwargs["user_message"] == "What is 2+2?" assert len(call_kwargs["conversation_history"]) == 1 + @pytest.mark.asyncio + async def test_responses_preserve_multimodal_content_arrays(self, adapter): + """Structured Responses input should reach the agent unchanged.""" + mock_result = {"final_response": "Done", "messages": [], "api_calls": 1} + multimodal = [ + {"type": "input_text", "text": "Describe this image"}, + {"type": "input_image", "image_url": "data:image/png;base64,AAAA", "detail": "auto"}, + ] + + app = _create_app(adapter) + async with TestClient(TestServer(app)) as cli: + with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run: + mock_run.return_value = (mock_result, {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}) + resp = await cli.post( + "/v1/responses", + json={"model": "hermes-agent", "input": [{"role": "user", "content": multimodal}]}, + ) + + assert resp.status == 200 + assert mock_run.call_args.kwargs["user_message"] == multimodal + @pytest.mark.asyncio async def test_instructions_as_ephemeral_prompt(self, adapter): """The instructions field maps to ephemeral_system_prompt.""" diff --git a/tests/gateway/test_native_multimodal_gateway.py b/tests/gateway/test_native_multimodal_gateway.py new file mode 100644 index 0000000000000..455c5c1b8391b --- /dev/null +++ b/tests/gateway/test_native_multimodal_gateway.py @@ -0,0 +1,208 @@ +from datetime import datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from gateway.config import GatewayConfig, Platform, PlatformConfig +from gateway.platforms.base import MessageEvent, MessageType, SendResult +from gateway.run import GatewayRunner +from gateway.session import SessionEntry, SessionSource + + +class _Adapter: + def __init__(self): + self.send = AsyncMock(return_value=SendResult(success=True, message_id="m1")) + + +def _build_runner(): + runner = object.__new__(GatewayRunner) + runner.config = GatewayConfig( + platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="fake-token")} + ) + runner.adapters = {Platform.TELEGRAM: _Adapter()} + runner._voice_mode = {} + runner.hooks = SimpleNamespace(emit=AsyncMock(), loaded_hooks=False) + runner.session_store = MagicMock() + runner.session_store.get_or_create_session.return_value = SessionEntry( + session_key="agent:main:telegram:dm:1", + session_id="sess-1", + created_at=datetime.now(), + updated_at=datetime.now(), + platform=Platform.TELEGRAM, + chat_type="dm", + ) + runner.session_store.load_transcript.return_value = [] + runner.session_store.has_any_sessions.return_value = True + runner.session_store.append_to_transcript = MagicMock() + runner.session_store.update_session = MagicMock() + runner.session_store._save = MagicMock() + runner._running_agents = {} + runner._pending_messages = {} + runner._pending_approvals = {} + runner._session_db = None + runner._voice_reply_config = {} + runner._smart_model_routing = {} + runner._tool_progress_cfg = {} + runner._response_semaphores = {} + runner._response_semaphore_default = 1 + runner._is_user_authorized = lambda _source: True + runner._set_session_env = lambda _context: None + runner._format_session_info = lambda: "" + runner._get_guild_id = lambda _event: None + runner._has_setup_skill = lambda: False + runner._run_agent = AsyncMock( + return_value={ + "final_response": "ok", + "messages": [], + "tools": [], + "history_offset": 0, + "last_prompt_tokens": 0, + } + ) + runner.delivery_router = MagicMock() + runner._model = "test-model" + runner._base_url = "" + return runner + + +def _source(): + return SessionSource(platform=Platform.TELEGRAM, chat_id="1", chat_type="dm", user_id="u1") + + +@pytest.mark.asyncio +async def test_gateway_photo_uses_native_multimodal_when_model_supports_vision(tmp_path, monkeypatch): + runner = _build_runner() + image_path = tmp_path / "img.png" + image_path.write_bytes(b"\x89PNG\r\n\x1a\nfake") + + monkeypatch.setattr("gateway.run._resolve_runtime_agent_kwargs", lambda: {"api_key": "fake"}) + monkeypatch.setattr("gateway.run._resolve_gateway_model", lambda *_args, **_kwargs: "test-model") + monkeypatch.setattr("agent.model_metadata.get_model_capabilities", lambda *_args, **_kwargs: {"supports_vision": True}) + + enrich_mock = AsyncMock(return_value="should-not-be-used") + runner._enrich_message_with_vision = enrich_mock + + event = MessageEvent( + text="describe", + message_type=MessageType.PHOTO, + source=_source(), + media_urls=[str(image_path)], + media_types=["image/png"], + message_id="1", + ) + + result = await runner._handle_message_with_agent(event, event.source, "q1") + + assert result == "ok" + enrich_mock.assert_not_awaited() + sent_message = runner._run_agent.call_args.kwargs["message"] + assert isinstance(sent_message, list) + assert sent_message[0] == {"type": "text", "text": "describe"} + assert sent_message[1]["type"] == "image_url" + assert sent_message[1]["image_url"]["url"].startswith("data:image/png;base64,") + + +@pytest.mark.asyncio +async def test_gateway_photo_falls_back_to_vision_tool_when_model_lacks_vision(tmp_path, monkeypatch): + runner = _build_runner() + image_path = tmp_path / "img.png" + image_path.write_bytes(b"\x89PNG\r\n\x1a\nfake") + + monkeypatch.setattr("gateway.run._resolve_runtime_agent_kwargs", lambda: {"api_key": "fake"}) + monkeypatch.setattr("gateway.run._resolve_gateway_model", lambda *_args, **_kwargs: "test-model") + monkeypatch.setattr("agent.model_metadata.get_model_capabilities", lambda *_args, **_kwargs: {"supports_vision": False}) + + enrich_mock = AsyncMock(return_value="[vision fallback]") + runner._enrich_message_with_vision = enrich_mock + + event = MessageEvent( + text="describe", + message_type=MessageType.PHOTO, + source=_source(), + media_urls=[str(image_path)], + media_types=["image/png"], + message_id="1", + ) + + result = await runner._handle_message_with_agent(event, event.source, "q1") + + assert result == "ok" + enrich_mock.assert_awaited_once() + assert runner._run_agent.call_args.kwargs["message"] == "[vision fallback]" + + +@pytest.mark.asyncio +async def test_gateway_photo_uses_resolved_turn_model_for_native_multimodal(tmp_path, monkeypatch): + runner = _build_runner() + runner._model = "non-vision-shell-model" + runner._resolve_turn_agent_config = lambda *_args, **_kwargs: { + "model": "vision-turn-model", + "runtime": {"api_key": "fake"}, + } + image_path = tmp_path / "img.png" + image_path.write_bytes(b"\x89PNG\r\n\x1a\nfake") + + monkeypatch.setattr("gateway.run._resolve_runtime_agent_kwargs", lambda: {"api_key": "fake"}) + monkeypatch.setattr("gateway.run._resolve_gateway_model", lambda *_args, **_kwargs: "base-model") + monkeypatch.setattr( + "agent.model_metadata.get_model_capabilities", + lambda model, *_args, **_kwargs: {"supports_vision": model == "vision-turn-model"}, + ) + + enrich_mock = AsyncMock(return_value="should-not-be-used") + runner._enrich_message_with_vision = enrich_mock + + event = MessageEvent( + text="describe", + message_type=MessageType.PHOTO, + source=_source(), + media_urls=[str(image_path)], + media_types=["image/png"], + message_id="1", + ) + + result = await runner._handle_message_with_agent(event, event.source, "q1") + + assert result == "ok" + enrich_mock.assert_not_awaited() + sent_message = runner._run_agent.call_args.kwargs["message"] + assert isinstance(sent_message, list) + assert sent_message[1]["type"] == "image_url" + + +@pytest.mark.asyncio +async def test_gateway_photo_falls_back_when_resolved_turn_model_lacks_vision(tmp_path, monkeypatch): + runner = _build_runner() + runner._model = "vision-shell-model" + runner._resolve_turn_agent_config = lambda *_args, **_kwargs: { + "model": "text-only-turn-model", + "runtime": {"api_key": "fake"}, + } + image_path = tmp_path / "img.png" + image_path.write_bytes(b"\x89PNG\r\n\x1a\nfake") + + monkeypatch.setattr("gateway.run._resolve_runtime_agent_kwargs", lambda: {"api_key": "fake"}) + monkeypatch.setattr("gateway.run._resolve_gateway_model", lambda *_args, **_kwargs: "base-model") + monkeypatch.setattr( + "agent.model_metadata.get_model_capabilities", + lambda model, *_args, **_kwargs: {"supports_vision": model == "vision-shell-model"}, + ) + + enrich_mock = AsyncMock(return_value="[vision fallback]") + runner._enrich_message_with_vision = enrich_mock + + event = MessageEvent( + text="describe", + message_type=MessageType.PHOTO, + source=_source(), + media_urls=[str(image_path)], + media_types=["image/png"], + message_id="1", + ) + + result = await runner._handle_message_with_agent(event, event.source, "q1") + + assert result == "ok" + enrich_mock.assert_awaited_once() + assert runner._run_agent.call_args.kwargs["message"] == "[vision fallback]" diff --git a/tests/gateway/test_session.py b/tests/gateway/test_session.py index 82281acc2eba4..edd3cdafcd6cd 100644 --- a/tests/gateway/test_session.py +++ b/tests/gateway/test_session.py @@ -903,3 +903,21 @@ def test_reasoning_survives_rewrite(self, tmp_path): assert after[0].get("reasoning") == "I need to think step by step." assert after[0].get("reasoning_details") == [{"type": "summary", "text": "step by step"}] assert after[0].get("codex_reasoning_items") == [{"id": "r1", "type": "reasoning"}] + + +class TestStructuredContentRoundTrip: + def test_structured_content_survives_db_round_trip(self, tmp_path): + from hermes_state import SessionDB + + db = SessionDB(db_path=tmp_path / "test.db") + session_id = "multimodal-roundtrip" + db.create_session(session_id=session_id, source="cli") + + content = [ + {"type": "text", "text": "Describe this image"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA", "detail": "auto"}}, + ] + db.append_message(session_id=session_id, role="user", content=content) + + loaded = db.get_messages_as_conversation(session_id) + assert loaded[0]["content"] == content diff --git a/tests/test_anthropic_adapter.py b/tests/test_anthropic_adapter.py index 4b4669eabc9d5..7faa1c67ef4c0 100644 --- a/tests/test_anthropic_adapter.py +++ b/tests/test_anthropic_adapter.py @@ -612,6 +612,35 @@ def test_converts_tool_results(self): assert user_msg["content"][0]["type"] == "tool_result" assert user_msg["content"][0]["tool_use_id"] == "tc_1" + def test_converts_multimodal_tool_results(self): + messages = [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "tc_1", "function": {"name": "read_file", "arguments": "{}"}}, + ], + }, + { + "role": "tool", + "tool_call_id": "tc_1", + "content": [ + {"type": "text", "text": "Image loaded from read_file."}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,QUFBQQ==", "detail": "auto"}, + }, + ], + }, + ] + _, result = convert_messages_to_anthropic(messages) + user_msg = [m for m in result if m["role"] == "user"][0] + tool_block = user_msg["content"][0] + assert tool_block["type"] == "tool_result" + assert isinstance(tool_block["content"], list) + assert tool_block["content"][0]["type"] == "text" + assert tool_block["content"][1]["type"] == "image" + def test_merges_consecutive_tool_results(self): messages = [ { diff --git a/tests/test_run_agent.py b/tests/test_run_agent.py index 617ae092882da..650277486c7ca 100644 --- a/tests/test_run_agent.py +++ b/tests/test_run_agent.py @@ -2681,7 +2681,7 @@ def test_build_api_kwargs_converts_multimodal_user_image_to_text(self, agent): assert isinstance(transformed[0]["content"], str) assert "A cat sitting on a chair." in transformed[0]["content"] assert "Can you see this now?" in transformed[0]["content"] - assert "vision_analyze with image_url: https://example.com/cat.png" in transformed[0]["content"] + assert "use read_file with path: https://example.com/cat.png" in transformed[0]["content"] def test_build_api_kwargs_reuses_cached_image_analysis_for_duplicate_images(self, agent): agent.api_mode = "anthropic_messages" @@ -2716,6 +2716,185 @@ def test_build_api_kwargs_reuses_cached_image_analysis_for_duplicate_images(self assert mock_vision.await_count == 1 +class TestNativeMultimodalRouting: + def test_native_vision_models_hide_auxiliary_vision_tool(self): + with ( + patch("run_agent.check_toolset_requirements", return_value={}), + patch("run_agent.OpenAI"), + patch( + "run_agent.get_tool_definitions", + side_effect=lambda **kwargs: _make_tool_defs( + "web_search", + *(["vision_analyze"] if not kwargs.get("omit_vision_analyze") else []), + ), + ) as mock_get_tools, + patch( + "agent.model_metadata.get_model_capabilities", + return_value={"supports_vision": True, "supports_audio": False}, + ), + ): + agent = AIAgent( + model="gpt-5.4", + api_key="test-key-1234567890", + quiet_mode=True, + skip_context_files=True, + skip_memory=True, + ) + + assert "vision_analyze" not in agent.valid_tool_names + assert mock_get_tools.call_args.kwargs["omit_vision_analyze"] is True + + def test_read_file_image_result_builds_native_followup_for_vision_models(self, agent): + agent.api_mode = "chat_completions" + agent._supports_native_vision = True + + tool_content, followup = agent._process_read_file_image_result( + "read_file", + json.dumps( + { + "is_image": True, + "path": "/tmp/cat.png", + "mime_type": "image/png", + "base64_content": "QUFBQQ==", + "dimensions": "64x64", + "file_size": 4, + } + ), + ) + + assert "[omitted from tool text output]" in tool_content + assert followup is not None + assert followup["role"] == "user" + assert followup["content"][1]["type"] == "image_url" + assert followup["content"][1]["image_url"]["url"].startswith("data:image/png;base64,") + + def test_read_file_image_result_stays_in_tool_output_for_responses(self, agent): + agent.api_mode = "codex_responses" + agent._supports_native_vision = True + + tool_content, followup = agent._process_read_file_image_result( + "read_file", + json.dumps( + { + "is_image": True, + "path": "/tmp/cat.png", + "mime_type": "image/png", + "base64_content": "QUFBQQ==", + } + ), + ) + + assert isinstance(tool_content, list) + assert tool_content[0]["type"] == "text" + assert tool_content[1]["type"] == "image_url" + assert followup is None + + def test_read_file_image_result_is_processed_before_string_truncation(self, agent): + agent.api_mode = "codex_responses" + agent._supports_native_vision = True + huge_b64 = "A" * 150_000 + + tool_content, followup = agent._process_read_file_image_result( + "read_file", + json.dumps( + { + "is_image": True, + "path": "/tmp/cat.png", + "mime_type": "image/png", + "base64_content": huge_b64, + } + ), + ) + + assert isinstance(tool_content, list) + assert tool_content[1]["type"] == "image_url" + assert followup is None + + def test_read_file_image_result_auto_analyzes_without_native_vision(self, agent): + agent.api_mode = "chat_completions" + agent._supports_native_vision = False + + with patch.object(agent, "_describe_image_for_anthropic_fallback", return_value="[auto analysis]"): + tool_content, followup = agent._process_read_file_image_result( + "read_file", + json.dumps( + { + "is_image": True, + "path": "/tmp/cat.png", + "mime_type": "image/png", + "base64_content": "QUFBQQ==", + } + ), + ) + + assert "[auto analysis]" in tool_content + assert "[omitted from tool text output]" in tool_content + assert followup is None + + def test_read_file_image_failure_tells_model_not_to_try_other_methods(self, agent): + agent.api_mode = "chat_completions" + agent._supports_native_vision = False + + tool_content, followup = agent._process_read_file_image_result( + "read_file", + json.dumps( + { + "is_image": True, + "file_size": 123, + "error": "Failed to read image data for inline attachment.", + } + ), + ) + + assert "Do not try terminal, OCR, PIL, tesseract, file, or other fallback inspection methods" in tool_content + assert followup is None + + def test_chat_completions_build_api_kwargs_preserves_multimodal_content(self, agent): + agent.api_mode = "chat_completions" + api_messages = [{ + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA", "detail": "auto"}}, + ], + }] + + kwargs = agent._build_api_kwargs(api_messages) + assert kwargs["messages"][0]["content"][0]["type"] == "text" + assert kwargs["messages"][0]["content"][1]["type"] == "image_url" + + def test_responses_build_api_kwargs_preserves_multimodal_content(self, agent): + agent.api_mode = "codex_responses" + api_messages = [{ + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA", "detail": "auto"}}, + ], + }] + + kwargs = agent._build_api_kwargs(api_messages) + assert kwargs["input"][0]["content"][0]["type"] == "input_text" + assert kwargs["input"][0]["content"][1]["type"] == "input_image" + + def test_responses_tool_output_preserves_multimodal_content(self, agent): + agent.api_mode = "codex_responses" + api_messages = [{ + "role": "tool", + "tool_call_id": "call_123", + "content": [ + {"type": "text", "text": "Image loaded from read_file."}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA", "detail": "auto"}}, + ], + }] + + kwargs = agent._build_api_kwargs(api_messages) + output = kwargs["input"][0]["output"] + assert isinstance(output, list) + assert output[0]["type"] == "input_text" + assert output[1]["type"] == "input_image" + + class TestFallbackAnthropicProvider: """Bug fix: _try_activate_fallback had no case for anthropic provider.""" diff --git a/tests/tools/test_file_operations.py b/tests/tools/test_file_operations.py index 0db3fb43b6a3a..58a14461c5998 100644 --- a/tests/tools/test_file_operations.py +++ b/tests/tools/test_file_operations.py @@ -218,6 +218,50 @@ def test_is_likely_binary_by_content(self, file_ops): def test_is_image(self, file_ops): assert file_ops._is_image("photo.png") is True assert file_ops._is_image("pic.jpg") is True + + +class TestShellFileReadImages: + def test_read_file_routes_images_to_image_reader(self, file_ops): + image_result = ReadResult( + is_image=True, + is_binary=True, + file_size=123, + base64_content="QUFBQQ==", + mime_type="image/png", + ) + file_ops._exec = MagicMock(return_value=MagicMock(exit_code=0, stdout="123")) + file_ops._read_image = MagicMock(return_value=image_result) + + result = file_ops.read_file("/tmp/test.png") + + file_ops._read_image.assert_called_once_with("/tmp/test.png") + assert result.is_image is True + assert result.base64_content == "QUFBQQ==" + + def test_read_image_returns_user_facing_hint_when_too_large(self, file_ops): + oversized = file_ops.MAX_IMAGE_BYTES + 1 + file_ops._exec = MagicMock(return_value=MagicMock(exit_code=0, stdout=str(oversized))) + + result = file_ops._read_image("/tmp/huge.png") + + assert result.is_image is True + assert result.base64_content is None + assert "too large to inline" in (result.hint or "") + assert "Do not keep trying read_file or terminal commands" in (result.hint or "") + + def test_read_image_failure_tells_model_to_stop_retrying(self, file_ops): + file_ops._exec = MagicMock( + side_effect=[ + MagicMock(exit_code=0, stdout="123"), + MagicMock(exit_code=1, stdout=""), + ] + ) + + result = file_ops._read_image("/tmp/fail.png") + + assert result.is_image is True + assert "Do not try terminal, OCR, PIL, or other fallback inspection steps" in (result.error or "") + assert "Stop here and inform the user" in (result.hint or "") assert file_ops._is_image("icon.ico") is True assert file_ops._is_image("data.pdf") is False assert file_ops._is_image("code.py") is False diff --git a/tests/tools/test_file_read_guards.py b/tests/tools/test_file_read_guards.py index b4a688aa61c38..9bf6ed6ad78f8 100644 --- a/tests/tools/test_file_read_guards.py +++ b/tests/tools/test_file_read_guards.py @@ -30,23 +30,32 @@ class _FakeReadResult: """Minimal stand-in for FileOperations.read_file return value.""" - def __init__(self, content="line1\nline2\n", total_lines=2, file_size=100): + def __init__( + self, + content="line1\nline2\n", + total_lines=2, + file_size=100, + **extra, + ): self.content = content self._total_lines = total_lines self._file_size = file_size + self._extra = extra def to_dict(self): - return { + result = { "content": self.content, "total_lines": self._total_lines, "file_size": self._file_size, } + result.update(self._extra) + return result -def _make_fake_ops(content="hello\n", total_lines=1, file_size=6): +def _make_fake_ops(content="hello\n", total_lines=1, file_size=6, **extra): fake = MagicMock() fake.read_file = lambda path, offset=1, limit=500: _FakeReadResult( - content=content, total_lines=total_lines, file_size=file_size, + content=content, total_lines=total_lines, file_size=file_size, **extra, ) return fake @@ -214,6 +223,31 @@ def test_different_task_not_deduped(self, mock_ops): r2 = json.loads(read_file_tool(self._tmpfile, task_id="task_b")) self.assertNotEqual(r2.get("dedup"), True) + @patch("tools.file_tools._get_file_ops") + def test_image_reads_are_not_deduped(self, mock_ops): + """Image reads must return a fresh payload, not a dedup stub.""" + image_path = os.path.join(self._tmpdir, "dedup_test.png") + with open(image_path, "wb") as f: + f.write(b"\x89PNG\r\n\x1a\n") + + mock_ops.return_value = _make_fake_ops( + content="", + file_size=8, + is_binary=True, + is_image=True, + base64_content="QUFBQQ==", + mime_type="image/png", + ) + + r1 = json.loads(read_file_tool(image_path, task_id="img")) + self.assertTrue(r1.get("is_image")) + self.assertEqual(r1.get("base64_content"), "QUFBQQ==") + + r2 = json.loads(read_file_tool(image_path, task_id="img")) + self.assertNotEqual(r2.get("dedup"), True) + self.assertTrue(r2.get("is_image")) + self.assertEqual(r2.get("base64_content"), "QUFBQQ==") + # --------------------------------------------------------------------------- # Dedup reset on compression diff --git a/tests/tools/test_file_tools_live.py b/tests/tools/test_file_tools_live.py index 90fdfac089161..6913b9002fe42 100644 --- a/tests/tools/test_file_tools_live.py +++ b/tests/tools/test_file_tools_live.py @@ -359,6 +359,22 @@ def test_no_noise_in_content(self, ops, tmp_path): assert result.error is None _assert_clean(result.content) + def test_image_returns_base64_payload(self, ops, tmp_path): + image_path = tmp_path / "pixel.png" + image_path.write_bytes( + b"\x89PNG\r\n\x1a\n" + b"\x00\x00\x00\rIHDR" + b"\x00\x00\x00\x01" + b"\x00\x00\x00\x01" + b"\x08\x02\x00\x00\x00" + b"\x90wS\xde" + ) + result = ops.read_file(str(image_path)) + assert result.error is None + assert result.is_image is True + assert result.mime_type == "image/png" + assert result.base64_content + # โ”€โ”€ write_file โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ diff --git a/tools/file_operations.py b/tools/file_operations.py index d0e3ad3c8ba73..4b57767634e1d 100644 --- a/tools/file_operations.py +++ b/tools/file_operations.py @@ -33,6 +33,7 @@ from typing import Optional, List, Dict, Any from pathlib import Path from hermes_constants import get_hermes_home +from agent.message_content import image_path_to_data_url # --------------------------------------------------------------------------- @@ -502,17 +503,9 @@ def read_file(self, path: str, offset: int = 1, limit: int = 500) -> ReadResult: # Still try to read, but warn pass - # Images are never inlined โ€” redirect to the vision tool + # Images can be returned as base64 payloads for native multimodal models. if self._is_image(path): - return ReadResult( - is_image=True, - is_binary=True, - file_size=file_size, - hint=( - "Image file detected. Automatically redirected to vision_analyze tool. " - "Use vision_analyze with this file path to inspect the image contents." - ), - ) + return self._read_image(path) # Read a sample to check for binary content sample_cmd = f"head -c 1000 {self._escape_shell_arg(path)} 2>/dev/null" @@ -555,9 +548,10 @@ def read_file(self, path: str, offset: int = 1, limit: int = 500) -> ReadResult: hint=hint ) - # Images larger than this are too expensive to inline as base64 in the - # conversation context. Return metadata only and suggest vision_analyze. - MAX_IMAGE_BYTES = 512 * 1024 # 512 KB + # Images larger than this are too expensive/risky to inline as base64 in + # the conversation context. Keep this aligned with the practical per-image + # limit expected by multimodal APIs when using data URLs. + MAX_IMAGE_BYTES = 20 * 1024 * 1024 # 20 MB def _read_image(self, path: str) -> ReadResult: """Read an image file, returning base64 content.""" @@ -575,22 +569,31 @@ def _read_image(self, path: str) -> ReadResult: is_binary=True, file_size=file_size, hint=( - f"Image is too large to inline ({file_size:,} bytes). " - "Use vision_analyze to inspect the image, or reference it by path." + f"Image is too large to inline ({file_size:,} bytes; max inline size is {self.MAX_IMAGE_BYTES:,} bytes). " + "Do not keep trying read_file or terminal commands to inspect the image contents. " + "Instead, tell the user directly that the image is too large to attach and ask them to send a smaller image or a cropped version." ), ) - # Get base64 content - b64_cmd = f"base64 -w 0 {self._escape_shell_arg(path)} 2>/dev/null" - b64_result = self._exec(b64_cmd, timeout=30) - - if b64_result.exit_code != 0: + # Use the same Python-side image reading/data-url path as Gateway so + # read_file(image) and inbound gateway images behave consistently. + data_url = image_path_to_data_url(path) + if not data_url: return ReadResult( is_image=True, is_binary=True, file_size=file_size, - error=f"Failed to read image: {b64_result.stdout}" + error=( + "Failed to read image data for inline attachment. " + "Do not try terminal, OCR, PIL, or other fallback inspection steps. " + "Instead, tell the user directly that this image could not be attached in the current environment." + ), + hint=( + "Stop here and inform the user that Hermes could not attach this image. " + "Ask for a smaller image, a re-upload, or a different file if they want further analysis." + ), ) + _, _, base64_content = data_url.partition(",") # Try to get dimensions (requires ImageMagick) dimensions = None @@ -617,7 +620,7 @@ def _read_image(self, path: str) -> ReadResult: is_image=True, is_binary=True, file_size=file_size, - base64_content=b64_result.stdout, + base64_content=base64_content, mime_type=mime_type, dimensions=dimensions ) diff --git a/tools/file_tools.py b/tools/file_tools.py index 79a111cb7961d..e97079c9ce0a1 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -7,7 +7,7 @@ import os import threading from pathlib import Path -from tools.file_operations import ShellFileOperations +from tools.file_operations import IMAGE_EXTENSIONS, ShellFileOperations from agent.redact import redact_sensitive_text logger = logging.getLogger(__name__) @@ -70,6 +70,17 @@ def _get_max_read_chars() -> int: }) +def _is_likely_image_path(filepath: str) -> bool: + """Return True when the path looks like an image file. + + Image reads must not use the generic dedup stub, because the earlier + `read_file` result may have been transformed into a synthetic multimodal + follow-up rather than remaining available as reusable tool text. + """ + ext = Path(os.path.expanduser(filepath)).suffix.lower() + return ext in IMAGE_EXTENSIONS + + def _is_blocked_device(filepath: str) -> bool: """Return True if the path would hang the process (infinite output or blocking input). @@ -326,7 +337,7 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str = }) cached_mtime = task_data.get("dedup", {}).get(dedup_key) - if cached_mtime is not None: + if cached_mtime is not None and not _is_likely_image_path(path): try: current_mtime = os.path.getmtime(resolved_str) if current_mtime == cached_mtime: @@ -403,7 +414,8 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 500, task_id: str = # the agent last read it (external edit, concurrent agent, etc.). try: _mtime_now = os.path.getmtime(resolved_str) - task_data["dedup"][dedup_key] = _mtime_now + if not (result_dict.get("is_image") or result_dict.get("is_binary")): + task_data["dedup"][dedup_key] = _mtime_now task_data.setdefault("read_timestamps", {})[resolved_str] = _mtime_now except OSError: pass # Can't stat โ€” skip tracking for this entry @@ -727,7 +739,7 @@ def _check_file_reqs(): READ_FILE_SCHEMA = { "name": "read_file", - "description": "Read a text file with line numbers and pagination. Use this instead of cat/head/tail in terminal. Output format: 'LINE_NUM|CONTENT'. Suggests similar filenames if not found. Use offset and limit for large files. Reads exceeding ~100K characters are rejected; use offset and limit to read specific sections of large files. NOTE: Cannot read images or binary files โ€” use vision_analyze for images.", + "description": "Read a file from the workspace. For text files, returns line-numbered content in 'LINE_NUM|CONTENT' format with pagination support. For image files, returns image metadata and image data that Hermes can attach to native multimodal models. Use this instead of cat/head/tail in terminal, including for image inspection. Suggests similar filenames if not found. Use offset and limit for large text files. Reads exceeding ~100K characters are rejected; use offset and limit to read specific text ranges.", "parameters": { "type": "object", "properties": { diff --git a/tools/session_search_tool.py b/tools/session_search_tool.py index 3ff36f940b842..aaed83d1b6fb6 100644 --- a/tools/session_search_tool.py +++ b/tools/session_search_tool.py @@ -22,6 +22,7 @@ from typing import Dict, Any, List, Optional, Union from agent.auxiliary_client import async_call_llm, extract_content_or_reasoning +from agent.message_content import content_to_text MAX_SESSION_CHARS = 100_000 MAX_SUMMARY_TOKENS = 10000 @@ -57,7 +58,11 @@ def _format_conversation(messages: List[Dict[str, Any]]) -> str: parts = [] for msg in messages: role = msg.get("role", "unknown").upper() - content = msg.get("content") or "" + content = content_to_text( + msg.get("content"), + image_placeholder="[image]", + fallback_json=True, + ) tool_name = msg.get("tool_name") if role == "TOOL" and tool_name: