fix(vision): make vision tools honor model.supports_vision

tests/tools/test_browser_console.py

@@ -251,6 +251,131 @@ def test_browser_vision_defaults_temperature_when_config_omits_it(self, tmp_path
         assert mock_llm.call_args.kwargs["temperature"] == 0.1
         assert mock_llm.call_args.kwargs["timeout"] == 120.0
 
+    def test_browser_vision_native_fast_path_returns_multimodal(self, tmp_path):
+        from agent.auxiliary_client import clear_runtime_main, set_runtime_main
+        from tools.browser_tool import browser_vision
+
+        shots_dir, screenshot = self._setup_screenshot(tmp_path)
+        annotations = [{"id": 1, "label": "Search box"}]
+        set_runtime_main("brand-new-provider", "llava-v1.6")
+        try:
+            with (
+                patch("hermes_constants.get_hermes_dir", return_value=shots_dir),
+                patch("tools.browser_tool._cleanup_old_screenshots"),
+                patch(
+                    "tools.browser_tool._run_browser_command",
+                    return_value={
+                        "success": True,
+                        "data": {
+                            "path": str(screenshot),
+                            "annotations": annotations,
+                        },
+                    },
+                ),
+                patch(
+                    "hermes_cli.config.load_config",
+                    return_value={"model": {"supports_vision": True}},
+                ),
+                patch("tools.browser_tool._get_vision_model") as mock_get_vision_model,
+                patch("tools.browser_tool.call_llm") as mock_llm,
+            ):
+                result = browser_vision(
+                    "what is on the page?", annotate=True, task_id="test"
+                )
+        finally:
+            clear_runtime_main()
+
+        assert isinstance(result, dict)
+        assert result["_multimodal"] is True
+        assert result["meta"]["screenshot_path"] == str(screenshot)
+        assert result["meta"]["annotations"] == annotations
+        assert any(p.get("type") == "image_url" for p in result["content"])
+        assert "what is on the page?" in result["content"][0]["text"]
+        assert str(screenshot) in result["content"][0]["text"]
+        assert "Screenshot path:" in result["text_summary"]
+        mock_get_vision_model.assert_not_called()
+        mock_llm.assert_not_called()
+
+    def test_browser_vision_native_mode_without_supports_vision_uses_aux_llm(self, tmp_path):
+        from agent.auxiliary_client import clear_runtime_main, set_runtime_main
+        from tools.browser_tool import browser_vision
+
+        shots_dir, screenshot = self._setup_screenshot(tmp_path)
+        mock_response = MagicMock()
+        mock_choice = MagicMock()
+        mock_choice.message.content = "Fallback screenshot analysis"
+        mock_response.choices = [mock_choice]
+
+        set_runtime_main("brand-new-provider", "opaque-model")
+        try:
+            with (
+                patch("hermes_constants.get_hermes_dir", return_value=shots_dir),
+                patch("tools.browser_tool._cleanup_old_screenshots"),
+                patch(
+                    "tools.browser_tool._run_browser_command",
+                    return_value={"success": True, "data": {"path": str(screenshot)}},
+                ),
+                patch(
+                    "hermes_cli.config.load_config",
+                    return_value={"agent": {"image_input_mode": "native"}},
+                ),
+                patch("tools.browser_tool._get_vision_model", return_value="test-model"),
+                patch("tools.browser_tool.call_llm", return_value=mock_response) as mock_llm,
+            ):
+                result = json.loads(browser_vision("what is on the page?", task_id="test"))
+        finally:
+            clear_runtime_main()
+
+        assert result["success"] is True
+        assert result["analysis"] == "Fallback screenshot analysis"
+        assert result["screenshot_path"] == str(screenshot)
+        mock_llm.assert_called_once()
+        kwargs = mock_llm.call_args.kwargs
+        assert kwargs["task"] == "vision"
+        assert kwargs["model"] == "test-model"
+        assert kwargs["messages"][0]["content"][1]["type"] == "image_url"
+        assert kwargs["messages"][0]["content"][1]["image_url"]["url"].startswith(
+            "data:image/png;base64,"
+        )
+
+    def test_browser_vision_text_mode_blocks_native_fast_path(self, tmp_path):
+        from agent.auxiliary_client import clear_runtime_main, set_runtime_main
+        from tools.browser_tool import browser_vision
+
+        shots_dir, screenshot = self._setup_screenshot(tmp_path)
+        mock_response = MagicMock()
+        mock_choice = MagicMock()
+        mock_choice.message.content = "Text-mode screenshot analysis"
+        mock_response.choices = [mock_choice]
+
+        set_runtime_main("brand-new-provider", "llava-v1.6")
+        try:
+            with (
+                patch("hermes_constants.get_hermes_dir", return_value=shots_dir),
+                patch("tools.browser_tool._cleanup_old_screenshots"),
+                patch(
+                    "tools.browser_tool._run_browser_command",
+                    return_value={"success": True, "data": {"path": str(screenshot)}},
+                ),
+                patch(
+                    "hermes_cli.config.load_config",
+                    return_value={
+                        "agent": {"image_input_mode": "text"},
+                        "model": {"supports_vision": True},
+                    },
+                ),
+                patch("tools.browser_tool._get_vision_model", return_value="test-model"),
+                patch("tools.browser_tool.call_llm", return_value=mock_response) as mock_llm,
+            ):
+                result = json.loads(browser_vision("what is on the page?", task_id="test"))
+        finally:
+            clear_runtime_main()
+
+        assert result["success"] is True
+        assert result["analysis"] == "Text-mode screenshot analysis"
+        assert result["screenshot_path"] == str(screenshot)
+        mock_llm.assert_called_once()
+
 
 # ── auto-recording config ────────────────────────────────────────────
 

tests/tools/test_vision_native_fast_path.py

@@ -148,66 +148,115 @@ def test_file_url_scheme_resolves(self, tmp_path):
 class TestHandleVisionAnalyzeFastPath:
     """Verify the dispatcher chooses fast-path vs aux-LLM correctly."""
 
-    def test_vision_capable_main_model_uses_fast_path(self, tmp_path, monkeypatch):
-        """Main model supports native vision → fast path returns multimodal."""
+    def test_native_mode_with_supported_transport_uses_fast_path(self, tmp_path):
+        """Explicit native mode + known transport returns multimodal."""
         img = tmp_path / "x.png"
         img.write_bytes(_TINY_PNG)
 
-        # Set runtime override so the handler thinks we're on opus@openrouter
+        async def _aux_sentinel(*args, **kwargs):
+            return '{"sentinel": "aux-path"}'
+
         from agent.auxiliary_client import set_runtime_main, clear_runtime_main
         set_runtime_main("openrouter", "anthropic/claude-opus-4.6")
         try:
-            # Mock decide_image_input_mode to always return "native" so the
-            # fast path fires regardless of model-catalog state in CI.
             with patch(
-                "agent.image_routing.decide_image_input_mode",
-                return_value="native",
-            ):
-                coro = _handle_vision_analyze({"image_url": str(img), "question": "?"})
-                result = asyncio.get_event_loop().run_until_complete(coro)
+                "hermes_cli.config.load_config",
+                return_value={"agent": {"image_input_mode": "native"}},
+            ), patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel) as mock_aux:
+                result = asyncio.get_event_loop().run_until_complete(
+                    _handle_vision_analyze({"image_url": str(img), "question": "?"})
+                )
         finally:
             clear_runtime_main()
 
-        assert isinstance(result, dict), \
+        assert isinstance(result, dict), (
             f"Expected multimodal envelope, got {type(result).__name__}: {str(result)[:200]}"
+        )
         assert result.get("_multimodal") is True
+        mock_aux.assert_not_called()
 
-    def test_non_vision_main_model_falls_through_to_aux(self, tmp_path, monkeypatch):
-        """Non-vision main model → fast path skipped, aux LLM path attempted."""
+    def test_native_mode_with_unsupported_transport_falls_through(self, tmp_path):
+        """Explicit native mode still respects the transport gate."""
         img = tmp_path / "x.png"
         img.write_bytes(_TINY_PNG)
 
         async def _aux_sentinel(*args, **kwargs):
             return '{"sentinel": "aux-path"}'
 
         from agent.auxiliary_client import set_runtime_main, clear_runtime_main
-        set_runtime_main("openrouter", "qwen/qwen3-coder")
+        set_runtime_main("brand-new-provider", "opaque-model")
         try:
-            with patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel):
-                coro = _handle_vision_analyze({"image_url": str(img), "question": "?"})
-                result = asyncio.get_event_loop().run_until_complete(coro)
+            with (
+                patch(
+                    "hermes_cli.config.load_config",
+                    return_value={"agent": {"image_input_mode": "native"}},
+                ),
+                patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel) as mock_aux,
+            ):
+                result = asyncio.get_event_loop().run_until_complete(
+                    _handle_vision_analyze({"image_url": str(img), "question": "?"})
+                )
         finally:
             clear_runtime_main()
 
-        assert not (isinstance(result, dict) and result.get("_multimodal") is True), \
-            "Fast path fired for non-vision model; should have fallen through to aux LLM"
+        assert isinstance(result, str)
+        assert json.loads(result) == {"sentinel": "aux-path"}
+        mock_aux.assert_called_once()
 
-    def test_fast_path_disabled_for_unsupported_provider(self, tmp_path, monkeypatch):
-        """Even with vision-capable model, unknown provider → fall through."""
+    def test_supports_vision_bypasses_transport_gate(self, tmp_path):
+        """supports_vision=True enables fast path even on unknown providers."""
         img = tmp_path / "x.png"
         img.write_bytes(_TINY_PNG)
 
         async def _aux_sentinel(*args, **kwargs):
             return '{"sentinel": "aux-path"}'
 
         from agent.auxiliary_client import set_runtime_main, clear_runtime_main
-        set_runtime_main("brand-new-provider", "anthropic/claude-opus-4.6")
+        set_runtime_main("brand-new-provider", "llava-v1.6")
         try:
-            with patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel):
-                coro = _handle_vision_analyze({"image_url": str(img), "question": "?"})
-                result = asyncio.get_event_loop().run_until_complete(coro)
+            with patch(
+                "hermes_cli.config.load_config",
+                return_value={"model": {"supports_vision": True}},
+            ), patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel) as mock_aux:
+                result = asyncio.get_event_loop().run_until_complete(
+                    _handle_vision_analyze({"image_url": str(img), "question": "?"})
+                )
         finally:
             clear_runtime_main()
 
-        assert not (isinstance(result, dict) and result.get("_multimodal") is True), \
-            "Fast path fired for unknown provider; should have fallen through"
+        assert isinstance(result, dict), (
+            f"Expected multimodal envelope, got {type(result).__name__}: {str(result)[:200]}"
+        )
+        assert result.get("_multimodal") is True
+        mock_aux.assert_not_called()
+
+    def test_text_mode_still_blocks_fast_path_when_supports_vision_true(self, tmp_path):
+        """Routing mode wins over supports_vision when text mode was chosen."""
+        img = tmp_path / "x.png"
+        img.write_bytes(_TINY_PNG)
+
+        async def _aux_sentinel(*args, **kwargs):
+            return '{"sentinel": "aux-path"}'
+
+        from agent.auxiliary_client import set_runtime_main, clear_runtime_main
+        set_runtime_main("brand-new-provider", "llava-v1.6")
+        try:
+            with (
+                patch(
+                    "hermes_cli.config.load_config",
+                    return_value={
+                        "agent": {"image_input_mode": "text"},
+                        "model": {"supports_vision": True},
+                    },
+                ),
+                patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel) as mock_aux,
+            ):
+                result = asyncio.get_event_loop().run_until_complete(
+                    _handle_vision_analyze({"image_url": str(img), "question": "?"})
+                )
+        finally:
+            clear_runtime_main()
+
+        assert isinstance(result, str)
+        assert json.loads(result) == {"sentinel": "aux-path"}
+        mock_aux.assert_called_once()

tools/browser_tool.py

@@ -1577,7 +1577,7 @@ def _update_session_activity(task_id: str):
     },
     {
         "name": "browser_vision",
-        "description": "Take a screenshot of the current page and analyze it with vision AI. Use this when you need to visually understand what's on the page - especially useful for CAPTCHAs, visual verification challenges, complex layouts, or when the text snapshot doesn't capture important visual information. Returns both the AI analysis and a screenshot_path that you can share with the user by including MEDIA:<screenshot_path> in your response. Requires browser_navigate to be called first.",
+        "description": "Take a screenshot of the current page so you can inspect it visually. Use this when you need to understand what the page looks like - especially for CAPTCHAs, visual verification challenges, complex layouts, or cases where the text snapshot misses important visual information. When your active model has native vision, the screenshot is attached to your context directly and you inspect it on the next turn; otherwise Hermes falls back to an auxiliary vision model and returns a text analysis. Includes a screenshot_path that you can share with the user by including MEDIA:<screenshot_path> in your response. Requires browser_navigate to be called first.",
         "parameters": {
             "type": "object",
             "properties": {
@@ -3045,23 +3045,26 @@ def browser_get_images(task_id: Optional[str] = None) -> str:
 
 def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] = None) -> str:
     """
-    Take a screenshot of the current page and analyze it with vision AI.
+    Take a screenshot of the current page for visual inspection.
 
-    This tool captures what's visually displayed in the browser and sends it
-    to Gemini for analysis. Useful for understanding visual content that the
-    text-based snapshot may not capture (CAPTCHAs, verification challenges,
-    images, complex layouts, etc.).
+    This tool captures what's visually displayed in the browser. When the
+    active model supports native vision, the screenshot is attached directly
+    to the conversation so the model can inspect it on the next turn.
+    Otherwise Hermes falls back to the auxiliary vision model. Useful for
+    understanding visual content that the text-based snapshot may not capture
+    (CAPTCHAs, verification challenges, images, complex layouts, etc.).
 
-    The screenshot is saved persistently and its file path is returned alongside
-    the analysis, so it can be shared with users via MEDIA:<path> in the response.
+    The screenshot is saved persistently and its file path is returned so it
+    can be shared with users via MEDIA:<path> in the response.
 
     Args:
         question: What you want to know about the page visually
         annotate: If True, overlay numbered [N] labels on interactive elements
         task_id: Task identifier for session isolation
 
     Returns:
-        JSON string with vision analysis results and screenshot_path
+        Either a JSON string with vision analysis results and screenshot_path,
+        or a multimodal tool-result envelope with the screenshot and metadata.
     """
     if _is_camofox_mode():
         from tools.browser_camofox import camofox_vision
@@ -3186,6 +3189,56 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str]
         _screenshot_b64 = base64.b64encode(_screenshot_bytes).decode("ascii")
         data_url = f"data:image/png;base64,{_screenshot_b64}"
 
+        # Fast path: when the active main model supports native vision AND the
+        # provider supports image content inside tool results, short-circuit
+        # the auxiliary LLM and return the image bytes as a multimodal
+        # tool-result envelope. The user can force native vision with the 
+        # supports_vision override. The main model sees the pixels directly on its
+        # next turn — no aux call, no information loss, no extra latency.
+        try:
+            from agent.auxiliary_client import _read_main_model, _read_main_provider
+            from agent.image_routing import decide_image_input_mode, _lookup_supports_vision
+            from hermes_cli.config import load_config
+            from tools.vision_tools import (
+                _build_native_vision_tool_result,
+                _supports_media_in_tool_results,
+            )
+
+            _provider = _read_main_provider()
+            _model = _read_main_model()
+            _cfg = load_config()
+            _mode = decide_image_input_mode(_provider, _model, _cfg)
+            _supports_vision = _lookup_supports_vision(_provider, _model, _cfg) is True
+            if _mode == "native" and (
+                _supports_media_in_tool_results(_provider, _model)
+                or _supports_vision
+            ):
+                native_result = _build_native_vision_tool_result(
+                    image_url=str(screenshot_path),
+                    question=question,
+                    image_data_url=data_url,
+                    image_size_bytes=len(_screenshot_bytes),
+                )
+                native_result.setdefault("meta", {})
+                native_result["meta"]["screenshot_path"] = str(screenshot_path)
+                if _lp_fallback_warning:
+                    native_result["meta"]["fallback_warning"] = _lp_fallback_warning
+                if annotate and result.get("data", {}).get("annotations"):
+                    native_result["meta"]["annotations"] = result["data"]["annotations"]
+                text_parts = native_result.get("content") or []
+                if text_parts and isinstance(text_parts[0], dict) and text_parts[0].get("type") == "text":
+                    text_parts[0]["text"] = (
+                        str(text_parts[0].get("text", ""))
+                        + f"\n\nScreenshot path: {screenshot_path}"
+                    )
+                native_result["text_summary"] = (
+                    str(native_result.get("text_summary") or "")
+                    + f" Screenshot path: {screenshot_path}"
+                ).strip()
+                return native_result
+        except Exception:
+            pass
+
         vision_prompt = (
             f"You are analyzing a screenshot of a web browser.\n\n"
             f"User's question: {question}\n\n"