feat(vision): vision_analyze returns pixels to vision-capable models

agent/auxiliary_client.py

@@ -1463,7 +1463,16 @@ def _read_main_model() -> str:
 
     config.yaml model.default is the single source of truth for the active
     model. Environment variables are no longer consulted.
+
+    Runtime override: when an AIAgent is active with a CLI/gateway-provided
+    model that differs from config.yaml, ``set_runtime_main()`` records the
+    override in a process-local global. This is consulted FIRST so tools
+    that gate on "the active main model" (e.g. ``vision_analyze``'s native
+    fast path) see the live runtime, not the persisted config default.
     """
+    override = _RUNTIME_MAIN_MODEL
+    if isinstance(override, str) and override.strip():
+        return override.strip()
     try:
         from hermes_cli.config import load_config
         cfg = load_config()
@@ -1484,7 +1493,13 @@ def _read_main_provider() -> str:
 
     Returns the lowercase provider id (e.g. "alibaba", "openrouter") or ""
     if not configured.
+
+    Runtime override: see ``_read_main_model`` — same mechanism for the
+    provider half of the runtime tuple.
     """
+    override = _RUNTIME_MAIN_PROVIDER
+    if isinstance(override, str) and override.strip():
+        return override.strip().lower()
     try:
         from hermes_cli.config import load_config
         cfg = load_config()
@@ -1498,6 +1513,32 @@ def _read_main_provider() -> str:
     return ""
 
 
+# Process-local override set by AIAgent at session/turn start. Single-threaded
+# per turn — no lock needed. Cleared by ``clear_runtime_main()``.
+_RUNTIME_MAIN_PROVIDER: str = ""
+_RUNTIME_MAIN_MODEL: str = ""
+
+
+def set_runtime_main(provider: str, model: str) -> None:
+    """Record the live runtime provider/model for the current AIAgent.
+
+    Called by ``run_agent.AIAgent._sync_runtime_main_for_aux_routing`` (or
+    equivalent setter) at the top of each turn so that
+    ``_read_main_provider`` / ``_read_main_model`` reflect CLI/gateway
+    overrides instead of the stale config.yaml default.
+    """
+    global _RUNTIME_MAIN_PROVIDER, _RUNTIME_MAIN_MODEL
+    _RUNTIME_MAIN_PROVIDER = (provider or "").strip().lower()
+    _RUNTIME_MAIN_MODEL = (model or "").strip()
+
+
+def clear_runtime_main() -> None:
+    """Clear the runtime override (e.g. on session end)."""
+    global _RUNTIME_MAIN_PROVIDER, _RUNTIME_MAIN_MODEL
+    _RUNTIME_MAIN_PROVIDER = ""
+    _RUNTIME_MAIN_MODEL = ""
+
+
 def _resolve_custom_runtime() -> Tuple[Optional[str], Optional[str], Optional[str]]:
     """Resolve the active custom/main endpoint the same way the main CLI does.
 

agent/codex_responses_adapter.py

@@ -410,10 +410,29 @@ def _chat_messages_to_responses_input(messages: List[Dict[str, Any]]) -> List[Di
                     call_id = raw_tool_call_id.strip()
             if not isinstance(call_id, str) or not call_id.strip():
                 continue
+
+            # Multimodal tool result: convert OpenAI-style content list into
+            # Responses ``function_call_output.output`` array. The Responses
+            # API accepts ``output`` as either a string or an array of
+            # ``input_text``/``input_image`` items. See
+            # https://developers.openai.com/api/reference/python/resources/responses/.
+            tool_content = msg.get("content")
+            output_value: Any
+            if isinstance(tool_content, list):
+                converted = _chat_content_to_responses_parts(
+                    tool_content, role="user",
+                )
+                if converted:
+                    output_value = converted
+                else:
+                    output_value = ""
+            else:
+                output_value = str(tool_content or "")
+
             items.append({
                 "type": "function_call_output",
                 "call_id": call_id,
-                "output": str(msg.get("content", "") or ""),
+                "output": output_value,
             })
 
     return items
@@ -466,6 +485,38 @@ def _preflight_codex_input_items(raw_items: Any) -> List[Dict[str, Any]]:
             output = item.get("output", "")
             if output is None:
                 output = ""
+            # Output may be a string OR an array of structured content
+            # items (input_text / input_image) for multimodal tool results.
+            # Both shapes are accepted by the Responses API. We preserve
+            # the array form when present.
+            if isinstance(output, list):
+                # Validate each item is a recognised content shape; drop
+                # anything else to avoid 4xx from the API.
+                cleaned: List[Dict[str, Any]] = []
+                for part in output:
+                    if not isinstance(part, dict):
+                        continue
+                    ptype = part.get("type")
+                    if ptype == "input_text":
+                        text = part.get("text")
+                        if isinstance(text, str) and text:
+                            cleaned.append({"type": "input_text", "text": text})
+                    elif ptype == "input_image":
+                        url = part.get("image_url")
+                        if isinstance(url, str) and url:
+                            entry: Dict[str, Any] = {"type": "input_image", "image_url": url}
+                            detail = part.get("detail")
+                            if isinstance(detail, str) and detail.strip():
+                                entry["detail"] = detail.strip()
+                            cleaned.append(entry)
+                normalized.append(
+                    {
+                        "type": "function_call_output",
+                        "call_id": call_id.strip(),
+                        "output": cleaned if cleaned else "",
+                    }
+                )
+                continue
             if not isinstance(output, str):
                 output = str(output)
 

run_agent.py

@@ -11119,6 +11119,20 @@ def run_conversation(
 
         self._ensure_db_session()
 
+        # Tell auxiliary_client what the live main provider/model are for
+        # this turn. Used by tools whose behaviour depends on the active
+        # main model (e.g. vision_analyze's native fast path) so they see
+        # the CLI/gateway override instead of the stale config.yaml
+        # default. Idempotent — fine to call every turn.
+        try:
+            from agent.auxiliary_client import set_runtime_main
+            set_runtime_main(
+                getattr(self, "provider", "") or "",
+                getattr(self, "model", "") or "",
+            )
+        except Exception:
+            pass
+
         # Tag all log records on this thread with the session ID so
         # ``hermes logs --session <id>`` can filter a single conversation.
         from hermes_logging import set_session_context

tests/conftest.py

@@ -427,6 +427,15 @@ def _reset_module_state():
     except Exception:
         pass
 
+    # --- agent.auxiliary_client — runtime main provider/model override ---
+    # Set per-turn by AIAgent.run_conversation; tests that import it must
+    # see a clean state so config.yaml fallback works as expected.
+    try:
+        from agent import auxiliary_client as _aux_mod
+        _aux_mod.clear_runtime_main()
+    except Exception:
+        pass
+
     # --- tools.file_tools — per-task read history + file-ops cache ---
     # _read_tracker accumulates per-task_id read history for loop detection,
     # capped by _READ_HISTORY_CAP. If entries from a prior test persist, the

tests/run_agent/test_codex_multimodal_tool_result.py

@@ -0,0 +1,173 @@
+"""Tests for codex_responses_adapter multimodal tool-result handling.
+
+Tool messages can contain a list of OpenAI-style content parts
+(``[{type:"text"...}, {type:"image_url"...}]``) when the
+``vision_analyze`` native fast path returns image bytes for the main model.
+This file verifies the Codex Responses adapter:
+
+  1. Converts that list into ``function_call_output.output`` as an array of
+     ``input_text``/``input_image`` items (not a stringified blob).
+  2. Preserves array-shaped output through the preflight validator.
+"""
+
+from __future__ import annotations
+
+from agent.codex_responses_adapter import (
+    _chat_messages_to_responses_input,
+    _preflight_codex_input_items,
+)
+
+
+def _build_messages_with_multimodal_tool_result():
+    return [
+        {"role": "user", "content": "What's in /tmp/foo.png?"},
+        {
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [{
+                "id": "call_abc",
+                "type": "function",
+                "function": {
+                    "name": "vision_analyze",
+                    "arguments": '{"image_url": "/tmp/foo.png", "question": "describe"}',
+                },
+            }],
+        },
+        {
+            "role": "tool",
+            "name": "vision_analyze",
+            "tool_call_id": "call_abc",
+            "content": [
+                {"type": "text", "text": "Image loaded."},
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,XYZ"}},
+            ],
+        },
+    ]
+
+
+class TestMultimodalToolResultConversion:
+    def test_list_content_becomes_output_array(self):
+        items = _chat_messages_to_responses_input(
+            _build_messages_with_multimodal_tool_result()
+        )
+        # Find the function_call_output item
+        outputs = [it for it in items if it.get("type") == "function_call_output"]
+        assert len(outputs) == 1
+        out = outputs[0]
+        assert out["call_id"] == "call_abc"
+        # Output should be a LIST (array form), not a string
+        assert isinstance(out["output"], list), \
+            f"Expected array output for multimodal tool result, got {type(out['output']).__name__}: {out['output']!r}"
+        types = [p.get("type") for p in out["output"]]
+        assert "input_text" in types
+        assert "input_image" in types
+
+    def test_input_image_preserves_data_url(self):
+        items = _chat_messages_to_responses_input(
+            _build_messages_with_multimodal_tool_result()
+        )
+        out = next(it for it in items if it.get("type") == "function_call_output")
+        image_parts = [p for p in out["output"] if p.get("type") == "input_image"]
+        assert len(image_parts) == 1
+        assert image_parts[0]["image_url"] == "data:image/png;base64,XYZ"
+
+    def test_string_tool_content_still_string_output(self):
+        msgs = [
+            {"role": "user", "content": "hi"},
+            {
+                "role": "assistant", "content": "",
+                "tool_calls": [{
+                    "id": "call_x", "type": "function",
+                    "function": {"name": "terminal", "arguments": "{}"},
+                }],
+            },
+            {
+                "role": "tool", "name": "terminal", "tool_call_id": "call_x",
+                "content": "ls output here",
+            },
+        ]
+        items = _chat_messages_to_responses_input(msgs)
+        out = next(it for it in items if it.get("type") == "function_call_output")
+        assert isinstance(out["output"], str)
+        assert out["output"] == "ls output here"
+
+
+class TestPreflightAcceptsArrayOutput:
+    def test_preflight_passes_array_through(self):
+        raw = [
+            {
+                "type": "function_call",
+                "call_id": "call_abc",
+                "name": "vision_analyze",
+                "arguments": "{}",
+            },
+            {
+                "type": "function_call_output",
+                "call_id": "call_abc",
+                "output": [
+                    {"type": "input_text", "text": "Image loaded."},
+                    {"type": "input_image", "image_url": "data:image/png;base64,ABC"},
+                ],
+            },
+        ]
+        normalized = _preflight_codex_input_items(raw)
+        out = [it for it in normalized if it.get("type") == "function_call_output"][0]
+        assert isinstance(out["output"], list)
+        assert len(out["output"]) == 2
+        assert out["output"][1]["type"] == "input_image"
+        assert out["output"][1]["image_url"] == "data:image/png;base64,ABC"
+
+    def test_preflight_drops_unknown_part_types(self):
+        raw = [
+            {
+                "type": "function_call",
+                "call_id": "call_abc", "name": "vision_analyze", "arguments": "{}",
+            },
+            {
+                "type": "function_call_output",
+                "call_id": "call_abc",
+                "output": [
+                    {"type": "input_text", "text": "ok"},
+                    {"type": "garbage", "data": "nope"},  # unknown — should be dropped
+                    {"type": "input_image", "image_url": "data:image/png;base64,ZZ"},
+                ],
+            },
+        ]
+        normalized = _preflight_codex_input_items(raw)
+        out = [it for it in normalized if it.get("type") == "function_call_output"][0]
+        # The "garbage" part is dropped; valid parts remain
+        types = [p.get("type") for p in out["output"]]
+        assert types == ["input_text", "input_image"]
+
+    def test_preflight_empty_array_becomes_empty_string(self):
+        # Defensive: an array with no valid parts shouldn't break the API call
+        raw = [
+            {
+                "type": "function_call",
+                "call_id": "call_x", "name": "vision_analyze", "arguments": "{}",
+            },
+            {
+                "type": "function_call_output",
+                "call_id": "call_x",
+                "output": [{"type": "garbage"}],  # all dropped
+            },
+        ]
+        normalized = _preflight_codex_input_items(raw)
+        out = [it for it in normalized if it.get("type") == "function_call_output"][0]
+        assert out["output"] == ""
+
+    def test_preflight_string_output_unchanged(self):
+        raw = [
+            {
+                "type": "function_call",
+                "call_id": "call_x", "name": "terminal", "arguments": "{}",
+            },
+            {
+                "type": "function_call_output",
+                "call_id": "call_x",
+                "output": "plain text output",
+            },
+        ]
+        normalized = _preflight_codex_input_items(raw)
+        out = [it for it in normalized if it.get("type") == "function_call_output"][0]
+        assert out["output"] == "plain text output"