From e9692c247f4ba346be78670c5e1db1d874111dc6 Mon Sep 17 00:00:00 2001 From: tillfalko Date: Thu, 21 May 2026 17:30:51 +0200 Subject: [PATCH 1/4] fix(vision): respect supports_vision in vision_analyze --- tools/vision_tools.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/vision_tools.py b/tools/vision_tools.py index 912777e2e255d..36c91416630e6 100644 --- a/tools/vision_tools.py +++ b/tools/vision_tools.py @@ -1018,18 +1018,23 @@ def _handle_vision_analyze(args: Dict[str, Any], **kw: Any) -> Awaitable[str]: # Fast path: when the active main model supports native vision AND the # provider supports image content inside tool results, short-circuit # the auxiliary LLM and return the image bytes as a multimodal - # tool-result envelope. The main model sees the pixels directly on its + # tool-result envelope. The user can force native vision with the + # supports_vision override. The main model sees the pixels directly on its # next turn — no aux call, no information loss, no extra latency. try: from agent.auxiliary_client import _read_main_provider, _read_main_model - from agent.image_routing import decide_image_input_mode + from agent.image_routing import decide_image_input_mode, _lookup_supports_vision from hermes_cli.config import load_config _provider = _read_main_provider() _model = _read_main_model() _cfg = load_config() _mode = decide_image_input_mode(_provider, _model, _cfg) - if _mode == "native" and _supports_media_in_tool_results(_provider, _model): + _supports_vision = _lookup_supports_vision(_provider, _model, _cfg) is True + if _mode == "native" and ( + _supports_media_in_tool_results(_provider, _model) + or _supports_vision + ): logger.info( "vision_analyze: native fast path (provider=%s, model=%s)", _provider, _model, From ec16d435f67316846f29a334fa94b5fa95fb4e75 Mon Sep 17 00:00:00 2001 From: tillfalko Date: Thu, 21 May 2026 17:31:35 +0200 Subject: [PATCH 2/4] fix(browser): add native image support to browser_vision and respect supports_vision --- tools/browser_tool.py | 50 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tools/browser_tool.py b/tools/browser_tool.py index 447f65007140f..d89a59b8a2aca 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -3186,6 +3186,56 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] _screenshot_b64 = base64.b64encode(_screenshot_bytes).decode("ascii") data_url = f"data:image/png;base64,{_screenshot_b64}" + # Fast path: when the active main model supports native vision AND the + # provider supports image content inside tool results, short-circuit + # the auxiliary LLM and return the image bytes as a multimodal + # tool-result envelope. The user can force native vision with the + # supports_vision override. The main model sees the pixels directly on its + # next turn — no aux call, no information loss, no extra latency. + try: + from agent.auxiliary_client import _read_main_model, _read_main_provider + from agent.image_routing import decide_image_input_mode, _lookup_supports_vision + from hermes_cli.config import load_config + from tools.vision_tools import ( + _build_native_vision_tool_result, + _supports_media_in_tool_results, + ) + + _provider = _read_main_provider() + _model = _read_main_model() + _cfg = load_config() + _mode = decide_image_input_mode(_provider, _model, _cfg) + _supports_vision = _lookup_supports_vision(_provider, _model, _cfg) is True + if _mode == "native" and ( + _supports_media_in_tool_results(_provider, _model) + or _supports_vision + ): + native_result = _build_native_vision_tool_result( + image_url=str(screenshot_path), + question=question, + image_data_url=data_url, + image_size_bytes=len(_screenshot_bytes), + ) + native_result.setdefault("meta", {}) + native_result["meta"]["screenshot_path"] = str(screenshot_path) + if _lp_fallback_warning: + native_result["meta"]["fallback_warning"] = _lp_fallback_warning + if annotate and result.get("data", {}).get("annotations"): + native_result["meta"]["annotations"] = result["data"]["annotations"] + text_parts = native_result.get("content") or [] + if text_parts and isinstance(text_parts[0], dict) and text_parts[0].get("type") == "text": + text_parts[0]["text"] = ( + str(text_parts[0].get("text", "")) + + f"\n\nScreenshot path: {screenshot_path}" + ) + native_result["text_summary"] = ( + str(native_result.get("text_summary") or "") + + f" Screenshot path: {screenshot_path}" + ).strip() + return native_result + except Exception: + pass + vision_prompt = ( f"You are analyzing a screenshot of a web browser.\n\n" f"User's question: {question}\n\n" From ef7b43651d50dacf76695949d4434439b7652326 Mon Sep 17 00:00:00 2001 From: tillfalko Date: Thu, 21 May 2026 17:32:40 +0200 Subject: [PATCH 3/4] test: extend test coverage to native image routing --- tests/tools/test_browser_console.py | 125 ++++++++++++++++++++ tests/tools/test_vision_native_fast_path.py | 103 +++++++++++----- 2 files changed, 201 insertions(+), 27 deletions(-) diff --git a/tests/tools/test_browser_console.py b/tests/tools/test_browser_console.py index b058fb3f36537..29a92c53baffc 100644 --- a/tests/tools/test_browser_console.py +++ b/tests/tools/test_browser_console.py @@ -251,6 +251,131 @@ def test_browser_vision_defaults_temperature_when_config_omits_it(self, tmp_path assert mock_llm.call_args.kwargs["temperature"] == 0.1 assert mock_llm.call_args.kwargs["timeout"] == 120.0 + def test_browser_vision_native_fast_path_returns_multimodal(self, tmp_path): + from agent.auxiliary_client import clear_runtime_main, set_runtime_main + from tools.browser_tool import browser_vision + + shots_dir, screenshot = self._setup_screenshot(tmp_path) + annotations = [{"id": 1, "label": "Search box"}] + set_runtime_main("brand-new-provider", "llava-v1.6") + try: + with ( + patch("hermes_constants.get_hermes_dir", return_value=shots_dir), + patch("tools.browser_tool._cleanup_old_screenshots"), + patch( + "tools.browser_tool._run_browser_command", + return_value={ + "success": True, + "data": { + "path": str(screenshot), + "annotations": annotations, + }, + }, + ), + patch( + "hermes_cli.config.load_config", + return_value={"model": {"supports_vision": True}}, + ), + patch("tools.browser_tool._get_vision_model") as mock_get_vision_model, + patch("tools.browser_tool.call_llm") as mock_llm, + ): + result = browser_vision( + "what is on the page?", annotate=True, task_id="test" + ) + finally: + clear_runtime_main() + + assert isinstance(result, dict) + assert result["_multimodal"] is True + assert result["meta"]["screenshot_path"] == str(screenshot) + assert result["meta"]["annotations"] == annotations + assert any(p.get("type") == "image_url" for p in result["content"]) + assert "what is on the page?" in result["content"][0]["text"] + assert str(screenshot) in result["content"][0]["text"] + assert "Screenshot path:" in result["text_summary"] + mock_get_vision_model.assert_not_called() + mock_llm.assert_not_called() + + def test_browser_vision_native_mode_without_supports_vision_uses_aux_llm(self, tmp_path): + from agent.auxiliary_client import clear_runtime_main, set_runtime_main + from tools.browser_tool import browser_vision + + shots_dir, screenshot = self._setup_screenshot(tmp_path) + mock_response = MagicMock() + mock_choice = MagicMock() + mock_choice.message.content = "Fallback screenshot analysis" + mock_response.choices = [mock_choice] + + set_runtime_main("brand-new-provider", "opaque-model") + try: + with ( + patch("hermes_constants.get_hermes_dir", return_value=shots_dir), + patch("tools.browser_tool._cleanup_old_screenshots"), + patch( + "tools.browser_tool._run_browser_command", + return_value={"success": True, "data": {"path": str(screenshot)}}, + ), + patch( + "hermes_cli.config.load_config", + return_value={"agent": {"image_input_mode": "native"}}, + ), + patch("tools.browser_tool._get_vision_model", return_value="test-model"), + patch("tools.browser_tool.call_llm", return_value=mock_response) as mock_llm, + ): + result = json.loads(browser_vision("what is on the page?", task_id="test")) + finally: + clear_runtime_main() + + assert result["success"] is True + assert result["analysis"] == "Fallback screenshot analysis" + assert result["screenshot_path"] == str(screenshot) + mock_llm.assert_called_once() + kwargs = mock_llm.call_args.kwargs + assert kwargs["task"] == "vision" + assert kwargs["model"] == "test-model" + assert kwargs["messages"][0]["content"][1]["type"] == "image_url" + assert kwargs["messages"][0]["content"][1]["image_url"]["url"].startswith( + "data:image/png;base64," + ) + + def test_browser_vision_text_mode_blocks_native_fast_path(self, tmp_path): + from agent.auxiliary_client import clear_runtime_main, set_runtime_main + from tools.browser_tool import browser_vision + + shots_dir, screenshot = self._setup_screenshot(tmp_path) + mock_response = MagicMock() + mock_choice = MagicMock() + mock_choice.message.content = "Text-mode screenshot analysis" + mock_response.choices = [mock_choice] + + set_runtime_main("brand-new-provider", "llava-v1.6") + try: + with ( + patch("hermes_constants.get_hermes_dir", return_value=shots_dir), + patch("tools.browser_tool._cleanup_old_screenshots"), + patch( + "tools.browser_tool._run_browser_command", + return_value={"success": True, "data": {"path": str(screenshot)}}, + ), + patch( + "hermes_cli.config.load_config", + return_value={ + "agent": {"image_input_mode": "text"}, + "model": {"supports_vision": True}, + }, + ), + patch("tools.browser_tool._get_vision_model", return_value="test-model"), + patch("tools.browser_tool.call_llm", return_value=mock_response) as mock_llm, + ): + result = json.loads(browser_vision("what is on the page?", task_id="test")) + finally: + clear_runtime_main() + + assert result["success"] is True + assert result["analysis"] == "Text-mode screenshot analysis" + assert result["screenshot_path"] == str(screenshot) + mock_llm.assert_called_once() + # ── auto-recording config ──────────────────────────────────────────── diff --git a/tests/tools/test_vision_native_fast_path.py b/tests/tools/test_vision_native_fast_path.py index 1df3003e5cdc4..65155006cb725 100644 --- a/tests/tools/test_vision_native_fast_path.py +++ b/tests/tools/test_vision_native_fast_path.py @@ -148,32 +148,35 @@ def test_file_url_scheme_resolves(self, tmp_path): class TestHandleVisionAnalyzeFastPath: """Verify the dispatcher chooses fast-path vs aux-LLM correctly.""" - def test_vision_capable_main_model_uses_fast_path(self, tmp_path, monkeypatch): - """Main model supports native vision → fast path returns multimodal.""" + def test_native_mode_with_supported_transport_uses_fast_path(self, tmp_path): + """Explicit native mode + known transport returns multimodal.""" img = tmp_path / "x.png" img.write_bytes(_TINY_PNG) - # Set runtime override so the handler thinks we're on opus@openrouter + async def _aux_sentinel(*args, **kwargs): + return '{"sentinel": "aux-path"}' + from agent.auxiliary_client import set_runtime_main, clear_runtime_main set_runtime_main("openrouter", "anthropic/claude-opus-4.6") try: - # Mock decide_image_input_mode to always return "native" so the - # fast path fires regardless of model-catalog state in CI. with patch( - "agent.image_routing.decide_image_input_mode", - return_value="native", - ): - coro = _handle_vision_analyze({"image_url": str(img), "question": "?"}) - result = asyncio.get_event_loop().run_until_complete(coro) + "hermes_cli.config.load_config", + return_value={"agent": {"image_input_mode": "native"}}, + ), patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel) as mock_aux: + result = asyncio.get_event_loop().run_until_complete( + _handle_vision_analyze({"image_url": str(img), "question": "?"}) + ) finally: clear_runtime_main() - assert isinstance(result, dict), \ + assert isinstance(result, dict), ( f"Expected multimodal envelope, got {type(result).__name__}: {str(result)[:200]}" + ) assert result.get("_multimodal") is True + mock_aux.assert_not_called() - def test_non_vision_main_model_falls_through_to_aux(self, tmp_path, monkeypatch): - """Non-vision main model → fast path skipped, aux LLM path attempted.""" + def test_native_mode_with_unsupported_transport_falls_through(self, tmp_path): + """Explicit native mode still respects the transport gate.""" img = tmp_path / "x.png" img.write_bytes(_TINY_PNG) @@ -181,19 +184,27 @@ async def _aux_sentinel(*args, **kwargs): return '{"sentinel": "aux-path"}' from agent.auxiliary_client import set_runtime_main, clear_runtime_main - set_runtime_main("openrouter", "qwen/qwen3-coder") + set_runtime_main("brand-new-provider", "opaque-model") try: - with patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel): - coro = _handle_vision_analyze({"image_url": str(img), "question": "?"}) - result = asyncio.get_event_loop().run_until_complete(coro) + with ( + patch( + "hermes_cli.config.load_config", + return_value={"agent": {"image_input_mode": "native"}}, + ), + patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel) as mock_aux, + ): + result = asyncio.get_event_loop().run_until_complete( + _handle_vision_analyze({"image_url": str(img), "question": "?"}) + ) finally: clear_runtime_main() - assert not (isinstance(result, dict) and result.get("_multimodal") is True), \ - "Fast path fired for non-vision model; should have fallen through to aux LLM" + assert isinstance(result, str) + assert json.loads(result) == {"sentinel": "aux-path"} + mock_aux.assert_called_once() - def test_fast_path_disabled_for_unsupported_provider(self, tmp_path, monkeypatch): - """Even with vision-capable model, unknown provider → fall through.""" + def test_supports_vision_bypasses_transport_gate(self, tmp_path): + """supports_vision=True enables fast path even on unknown providers.""" img = tmp_path / "x.png" img.write_bytes(_TINY_PNG) @@ -201,13 +212,51 @@ async def _aux_sentinel(*args, **kwargs): return '{"sentinel": "aux-path"}' from agent.auxiliary_client import set_runtime_main, clear_runtime_main - set_runtime_main("brand-new-provider", "anthropic/claude-opus-4.6") + set_runtime_main("brand-new-provider", "llava-v1.6") try: - with patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel): - coro = _handle_vision_analyze({"image_url": str(img), "question": "?"}) - result = asyncio.get_event_loop().run_until_complete(coro) + with patch( + "hermes_cli.config.load_config", + return_value={"model": {"supports_vision": True}}, + ), patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel) as mock_aux: + result = asyncio.get_event_loop().run_until_complete( + _handle_vision_analyze({"image_url": str(img), "question": "?"}) + ) finally: clear_runtime_main() - assert not (isinstance(result, dict) and result.get("_multimodal") is True), \ - "Fast path fired for unknown provider; should have fallen through" + assert isinstance(result, dict), ( + f"Expected multimodal envelope, got {type(result).__name__}: {str(result)[:200]}" + ) + assert result.get("_multimodal") is True + mock_aux.assert_not_called() + + def test_text_mode_still_blocks_fast_path_when_supports_vision_true(self, tmp_path): + """Routing mode wins over supports_vision when text mode was chosen.""" + img = tmp_path / "x.png" + img.write_bytes(_TINY_PNG) + + async def _aux_sentinel(*args, **kwargs): + return '{"sentinel": "aux-path"}' + + from agent.auxiliary_client import set_runtime_main, clear_runtime_main + set_runtime_main("brand-new-provider", "llava-v1.6") + try: + with ( + patch( + "hermes_cli.config.load_config", + return_value={ + "agent": {"image_input_mode": "text"}, + "model": {"supports_vision": True}, + }, + ), + patch("tools.vision_tools.vision_analyze_tool", side_effect=_aux_sentinel) as mock_aux, + ): + result = asyncio.get_event_loop().run_until_complete( + _handle_vision_analyze({"image_url": str(img), "question": "?"}) + ) + finally: + clear_runtime_main() + + assert isinstance(result, str) + assert json.loads(result) == {"sentinel": "aux-path"} + mock_aux.assert_called_once() From c961b528f0dbdee0e2363ee6b053ac29bd430cb0 Mon Sep 17 00:00:00 2001 From: tillfalko Date: Thu, 21 May 2026 21:07:32 +0200 Subject: [PATCH 4/4] docs(browser): update browser_vision tool description for native vision routing --- tools/browser_tool.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tools/browser_tool.py b/tools/browser_tool.py index d89a59b8a2aca..5e9b8ef361eec 100644 --- a/tools/browser_tool.py +++ b/tools/browser_tool.py @@ -1577,7 +1577,7 @@ def _update_session_activity(task_id: str): }, { "name": "browser_vision", - "description": "Take a screenshot of the current page and analyze it with vision AI. Use this when you need to visually understand what's on the page - especially useful for CAPTCHAs, visual verification challenges, complex layouts, or when the text snapshot doesn't capture important visual information. Returns both the AI analysis and a screenshot_path that you can share with the user by including MEDIA: in your response. Requires browser_navigate to be called first.", + "description": "Take a screenshot of the current page so you can inspect it visually. Use this when you need to understand what the page looks like - especially for CAPTCHAs, visual verification challenges, complex layouts, or cases where the text snapshot misses important visual information. When your active model has native vision, the screenshot is attached to your context directly and you inspect it on the next turn; otherwise Hermes falls back to an auxiliary vision model and returns a text analysis. Includes a screenshot_path that you can share with the user by including MEDIA: in your response. Requires browser_navigate to be called first.", "parameters": { "type": "object", "properties": { @@ -3045,15 +3045,17 @@ def browser_get_images(task_id: Optional[str] = None) -> str: def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] = None) -> str: """ - Take a screenshot of the current page and analyze it with vision AI. + Take a screenshot of the current page for visual inspection. - This tool captures what's visually displayed in the browser and sends it - to Gemini for analysis. Useful for understanding visual content that the - text-based snapshot may not capture (CAPTCHAs, verification challenges, - images, complex layouts, etc.). + This tool captures what's visually displayed in the browser. When the + active model supports native vision, the screenshot is attached directly + to the conversation so the model can inspect it on the next turn. + Otherwise Hermes falls back to the auxiliary vision model. Useful for + understanding visual content that the text-based snapshot may not capture + (CAPTCHAs, verification challenges, images, complex layouts, etc.). - The screenshot is saved persistently and its file path is returned alongside - the analysis, so it can be shared with users via MEDIA: in the response. + The screenshot is saved persistently and its file path is returned so it + can be shared with users via MEDIA: in the response. Args: question: What you want to know about the page visually @@ -3061,7 +3063,8 @@ def browser_vision(question: str, annotate: bool = False, task_id: Optional[str] task_id: Task identifier for session isolation Returns: - JSON string with vision analysis results and screenshot_path + Either a JSON string with vision analysis results and screenshot_path, + or a multimodal tool-result envelope with the screenshot and metadata. """ if _is_camofox_mode(): from tools.browser_camofox import camofox_vision