From 15a7337d755eb5fa2df17ea7bbaad96104cba659 Mon Sep 17 00:00:00 2001 From: kagura-agent Date: Sun, 17 May 2026 16:18:37 +0800 Subject: [PATCH 1/2] fix(run_agent): guard multimodal tool content by provider capability (#27344) Vision-capable models whose providers do not accept list-type content in tool messages (e.g. Xiaomi MiMo) get HTTP 400 when computer_use returns screenshot parts. The OpenAI spec defines tool message content as a string; only OpenAI, Anthropic, and Gemini extend it to accept arrays. Add _provider_supports_multimodal_tool_content() with a conservative allowlist (anthropic_messages, codex_responses, gemini_native api modes, plus openai/azure/anthropic/google/gemini providers on chat_completions). Vision-capable models on unsupported providers now gracefully fall back to a text summary instead of passing raw content parts. Fixes #27344 --- run_agent.py | 77 +++++++-- .../test_multimodal_tool_content_guard.py | 146 ++++++++++++++++++ 2 files changed, 206 insertions(+), 17 deletions(-) create mode 100644 tests/run_agent/test_multimodal_tool_content_guard.py diff --git a/run_agent.py b/run_agent.py index 6e39ccfbb564..aed7d491a66c 100644 --- a/run_agent.py +++ b/run_agent.py @@ -3202,6 +3202,34 @@ def _model_supports_vision(self) -> bool: except Exception: return False + def _provider_supports_multimodal_tool_content(self) -> bool: + """Return True if the active provider accepts list-type content in tool messages. + + The OpenAI chat-completions spec defines tool message ``content`` as a + string. A handful of first-party providers extend this to accept an + array of content parts (text + image_url). For all other providers we + fall back to a plain text summary so the request is not rejected. + + See: https://github.com/NousResearch/hermes-agent/issues/27344 + """ + api_mode = getattr(self, "api_mode", "") or "" + if api_mode in { + "anthropic_messages", # Anthropic natively converts parts → blocks + "codex_responses", # OpenAI Responses API accepts content parts + "gemini_native", # Gemini native accepts inline_data parts + }: + return True + # chat_completions: only first-party OpenAI endpoints reliably accept + # list-type tool content. + provider_lower = (getattr(self, "provider", "") or "").lower() + if provider_lower in { + "openai", "openai-codex", "azure", + "anthropic", # Anthropic-via-OpenAI compat layer + "google", "gemini", + }: + return True + return False + def _preprocess_anthropic_content(self, content: Any, role: str) -> Any: if not self._content_has_image_parts(content): return content @@ -3340,29 +3368,44 @@ def _tool_result_content_for_active_model(self, tool_name: str, result: Any) -> if not self._content_has_image_parts(content): return content - if self._model_supports_vision(): - return content + if not self._model_supports_vision(): + summary = _multimodal_text_summary(result) + if tool_name == "computer_use": + return json.dumps({ + "error": ( + "computer_use returned screenshot/image content, but the active " + "model/provider does not support image input. Switch to a " + "vision-capable model for desktop computer use, or use browser " + "tools for browser tasks." + ), + "text_summary": summary, + }) - summary = _multimodal_text_summary(result) - if tool_name == "computer_use": - return json.dumps({ - "error": ( - "computer_use returned screenshot/image content, but the active " - "model/provider does not support image input. Switch to a " - "vision-capable model for desktop computer use, or use browser " - "tools for browser tasks." - ), - "text_summary": summary, - }) + logger.warning( + "Tool %s returned image content for non-vision model %s/%s; " + "falling back to text summary", + tool_name, + self.provider, + self.model, + ) + return summary + + # Vision-capable model, but check if the provider supports multimodal + # content in tool messages. The OpenAI spec defines tool message + # ``content`` as a string; only a subset of providers extend it to + # accept a content-parts list. Send the text summary for providers + # that are not known to handle list-type tool content. + if self._provider_supports_multimodal_tool_content(): + return content - logger.warning( - "Tool %s returned image content for non-vision model %s/%s; " - "falling back to text summary", + logger.debug( + "Tool %s returned multimodal content but provider %s/%s may not " + "support list-type tool message content; falling back to text summary", tool_name, self.provider, self.model, ) - return summary + return _multimodal_text_summary(result) def _try_shrink_image_parts_in_messages(self, api_messages: list) -> bool: """Forwarder — see ``agent.conversation_compression.try_shrink_image_parts_in_messages``.""" diff --git a/tests/run_agent/test_multimodal_tool_content_guard.py b/tests/run_agent/test_multimodal_tool_content_guard.py new file mode 100644 index 000000000000..3125ac0b66ff --- /dev/null +++ b/tests/run_agent/test_multimodal_tool_content_guard.py @@ -0,0 +1,146 @@ +"""Tests for multimodal tool content provider guard. + +Covers the fix for #27344: vision-capable models whose providers do not +accept list-type ``content`` in tool messages should receive a text +summary instead of the raw multimodal content parts. +""" + +from __future__ import annotations + +from unittest.mock import patch + +from run_agent import AIAgent + + +def _make_agent(provider: str = "openai", model: str = "gpt-4o", + api_mode: str = "chat_completions") -> AIAgent: + """Build a bare-bones AIAgent for pure-method tests.""" + agent = object.__new__(AIAgent) + agent.provider = provider + agent.model = model + agent.api_mode = api_mode + agent._anthropic_image_fallback_cache = {} + return agent + + +MULTIMODAL_RESULT = { + "_multimodal": True, + "text_summary": "Screenshot captured", + "content": [ + {"type": "text", "text": "Screenshot captured"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ], +} + + +# ─── _provider_supports_multimodal_tool_content ────────────────────────────── + + +class TestProviderSupportsMultimodalToolContent: + """Verify the allowlist correctly identifies supported providers.""" + + def test_anthropic_messages_api_mode(self): + agent = _make_agent(provider="anthropic", api_mode="anthropic_messages") + assert agent._provider_supports_multimodal_tool_content() is True + + def test_codex_responses_api_mode(self): + agent = _make_agent(provider="openai-codex", api_mode="codex_responses") + assert agent._provider_supports_multimodal_tool_content() is True + + def test_gemini_native_api_mode(self): + agent = _make_agent(provider="google", api_mode="gemini_native") + assert agent._provider_supports_multimodal_tool_content() is True + + def test_openai_chat_completions(self): + agent = _make_agent(provider="openai", api_mode="chat_completions") + assert agent._provider_supports_multimodal_tool_content() is True + + def test_azure_chat_completions(self): + agent = _make_agent(provider="azure", api_mode="chat_completions") + assert agent._provider_supports_multimodal_tool_content() is True + + def test_xiaomi_chat_completions_not_supported(self): + agent = _make_agent(provider="xiaomi", api_mode="chat_completions") + assert agent._provider_supports_multimodal_tool_content() is False + + def test_deepseek_chat_completions_not_supported(self): + agent = _make_agent(provider="deepseek", api_mode="chat_completions") + assert agent._provider_supports_multimodal_tool_content() is False + + def test_openrouter_chat_completions_not_supported(self): + agent = _make_agent(provider="openrouter", api_mode="chat_completions") + assert agent._provider_supports_multimodal_tool_content() is False + + def test_custom_provider_not_supported(self): + agent = _make_agent(provider="my-custom-llm", api_mode="chat_completions") + assert agent._provider_supports_multimodal_tool_content() is False + + +# ─── _tool_result_content_for_active_model ─────────────────────────────────── + + +class TestToolResultContentGuard: + """Verify the end-to-end tool result guard for #27344.""" + + def test_vision_supported_provider_returns_content(self): + """OpenAI + vision = multimodal content passed through.""" + agent = _make_agent(provider="openai", api_mode="chat_completions") + with patch.object(agent, "_model_supports_vision", return_value=True): + result = agent._tool_result_content_for_active_model( + "computer_use", MULTIMODAL_RESULT, + ) + # Should return the content list directly + assert isinstance(result, list) + types = [p.get("type") for p in result] + assert "image_url" in types + + def test_vision_unsupported_provider_returns_summary(self): + """Xiaomi MiMo has vision but provider rejects list tool content.""" + agent = _make_agent(provider="xiaomi", model="mimo-v2.5", + api_mode="chat_completions") + with patch.object(agent, "_model_supports_vision", return_value=True): + result = agent._tool_result_content_for_active_model( + "computer_use", MULTIMODAL_RESULT, + ) + # Should fall back to text summary, not the list + assert isinstance(result, str) + assert "Screenshot captured" in result + + def test_non_vision_model_returns_error_for_computer_use(self): + """Non-vision model gets a JSON error for computer_use.""" + agent = _make_agent(provider="deepseek", model="deepseek-chat", + api_mode="chat_completions") + with patch.object(agent, "_model_supports_vision", return_value=False): + result = agent._tool_result_content_for_active_model( + "computer_use", MULTIMODAL_RESULT, + ) + assert isinstance(result, str) + assert "does not support image input" in result + + def test_non_vision_model_returns_summary_for_other_tools(self): + """Non-vision model gets text summary for non-computer_use tools.""" + agent = _make_agent(provider="deepseek", model="deepseek-chat", + api_mode="chat_completions") + with patch.object(agent, "_model_supports_vision", return_value=False): + result = agent._tool_result_content_for_active_model( + "vision_analyze", MULTIMODAL_RESULT, + ) + assert isinstance(result, str) + assert "Screenshot captured" in result + + def test_non_multimodal_result_passes_through(self): + """Plain string results are not affected by the guard.""" + agent = _make_agent(provider="xiaomi", api_mode="chat_completions") + result = agent._tool_result_content_for_active_model( + "terminal", "command output here", + ) + assert result == "command output here" + + def test_anthropic_api_mode_bypasses_provider_check(self): + """anthropic_messages api_mode always supports multimodal tool content.""" + agent = _make_agent(provider="some-proxy", api_mode="anthropic_messages") + with patch.object(agent, "_model_supports_vision", return_value=True): + result = agent._tool_result_content_for_active_model( + "computer_use", MULTIMODAL_RESULT, + ) + assert isinstance(result, list) From 754e5db5ea158f4f3db807cf66130531c101c832 Mon Sep 17 00:00:00 2001 From: kagura-agent Date: Mon, 18 May 2026 21:17:16 +0800 Subject: [PATCH 2/2] fix(ci): add kagura-agent email to AUTHOR_MAP for check-attribution --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index 3d37c199b410..ea6949366b15 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -684,6 +684,7 @@ "juan.ovalle@mistral.ai": "jjovalle99", "julien.talbot@ergonomia.re": "Julientalbot", "kagura.chen28@gmail.com": "kagura-agent", + "kagura.agent.ai@gmail.com": "kagura-agent", "1342088860@qq.com": "youngDoo", "kamil@gwozdz.me": "kamil-gwozdz", "skmishra1991@gmail.com": "bugkill3r",