diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 96ad615bf6f44..c76be86b01eb2 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -39,7 +39,6 @@ auto-detection chain. This handles the common case where a user depletes their OpenRouter balance but has Codex OAuth or another provider available. """ - import json import logging import os @@ -49,6 +48,7 @@ from types import SimpleNamespace from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING from urllib.parse import urlparse, parse_qs, urlunparse +import contextvars # NOTE: `from openai import OpenAI` is deliberately NOT at module top — the # openai SDK pulls a large type tree (~240 ms cold, including responses/*, diff --git a/model_tools.py b/model_tools.py index db19bb67e539a..0e246a292ac23 100644 --- a/model_tools.py +++ b/model_tools.py @@ -694,6 +694,8 @@ def handle_function_call( user_task: Optional[str] = None, enabled_tools: Optional[List[str]] = None, skip_pre_tool_call_hook: bool = False, + provider: Optional[str] = None, + model: Optional[str] = None, ) -> str: """ Main function call dispatcher that routes calls to the tool registry. @@ -770,12 +772,16 @@ def handle_function_call( function_name, function_args, task_id=task_id, enabled_tools=sandbox_enabled, + provider=provider, + model=model, ) else: result = registry.dispatch( function_name, function_args, task_id=task_id, user_task=user_task, + provider=provider, + model=model, ) duration_ms = int((time.monotonic() - _dispatch_start) * 1000) diff --git a/run_agent.py b/run_agent.py index a4df874977723..598f7f860a413 100644 --- a/run_agent.py +++ b/run_agent.py @@ -10648,6 +10648,10 @@ def _dispatch_delegate_task(self, function_args: dict) -> str: parent_agent=self, ) + def _tool_routing_kwargs(self) -> dict: + """Return provider/model kwargs so tool dispatch follows the active thread backend.""" + return {"provider": self.provider or None, "model": self.model or None} + def _invoke_tool(self, function_name: str, function_args: dict, effective_task_id: str, tool_call_id: Optional[str] = None, messages: list = None, pre_tool_block_checked: bool = False) -> str: @@ -10733,6 +10737,7 @@ def _invoke_tool(self, function_name: str, function_args: dict, effective_task_i session_id=self.session_id or "", enabled_tools=list(self.valid_tool_names) if self.valid_tool_names else None, skip_pre_tool_call_hook=True, + **self._tool_routing_kwargs(), ) @staticmethod @@ -11450,6 +11455,7 @@ def _execute_tool_calls_sequential(self, assistant_message, messages: list, effe session_id=self.session_id or "", enabled_tools=list(self.valid_tool_names) if self.valid_tool_names else None, skip_pre_tool_call_hook=True, + **self._tool_routing_kwargs(), ) _spinner_result = function_result except Exception as tool_error: @@ -11470,6 +11476,7 @@ def _execute_tool_calls_sequential(self, assistant_message, messages: list, effe session_id=self.session_id or "", enabled_tools=list(self.valid_tool_names) if self.valid_tool_names else None, skip_pre_tool_call_hook=True, + **self._tool_routing_kwargs(), ) except Exception as tool_error: function_result = f"Error executing tool '{function_name}': {tool_error}" diff --git a/tests/tools/test_vision_thread_routing.py b/tests/tools/test_vision_thread_routing.py new file mode 100644 index 0000000000000..8a327611e4213 --- /dev/null +++ b/tests/tools/test_vision_thread_routing.py @@ -0,0 +1,131 @@ +"""Regression tests for PR #27015 — vision tool routing via active thread model. + +Verifies that explicit provider/model kwargs propagate through the full +stack (agent → model_tools → registry → vision_tools) and that task-level +per-vision config is still respected. +""" + +import os +from unittest.mock import MagicMock, patch + +import pytest + +from tools.vision_tools import ( + _handle_vision_analyze, + vision_analyze_tool, +) + + +# --------------------------------------------------------------------------- +# 1. handle_function_call → registry.dispatch propagation +# --------------------------------------------------------------------------- + +def test_handle_function_call_passes_provider_model(): + """handle_function_call must forward provider/model kwargs to registry.dispatch.""" + from model_tools import handle_function_call + + with patch("model_tools.registry.dispatch") as mock_dispatch: + mock_dispatch.return_value = '{"success": true}' + handle_function_call( + "browser_get_images", + {}, + provider="openrouter", + model="gpt-4o", + ) + args, kwargs = mock_dispatch.call_args + assert kwargs["provider"] == "openrouter" + assert kwargs["model"] == "gpt-4o" + + +# --------------------------------------------------------------------------- +# 2. _handle_vision_analyze extracts provider/model from kwargs +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_handle_vision_analyze_extracts_kwargs(): + """_handle_vision_analyze must pluck provider/model from **kw and forward them.""" + with patch("tools.vision_tools._read_main_provider", return_value="openrouter"), \ + patch("tools.vision_tools._read_main_model", return_value="invalid-model"), \ + patch("tools.vision_tools._supports_media_in_tool_results", return_value=False), \ + patch("tools.vision_tools.vision_analyze_tool") as mock_tool: + + await _handle_vision_analyze( + {"image_url": "https://example.com/img.png", "question": "What is this?"}, + provider="anthropic", + model="claude-3-opus-20240229", + ) + args, kwargs = mock_tool.call_args + assert kwargs.get("provider") == "anthropic" + assert kwargs.get("model") == "claude-3-opus-20240229" + + +# --------------------------------------------------------------------------- +# 3. vision_analyze_tool includes task="vision" in call to async_call_llm +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_vision_analyze_tool_includes_task_vision(): + """The call_kwargs passed to async_call_llm must contain task='vision'.""" + with patch("tools.vision_tools.async_call_llm") as mock_async, \ + patch("tools.vision_tools._image_to_base64_data_url", return_value="data:image/png;base64,dummy"), \ + patch("tools.vision_tools._detect_image_mime_type", return_value="image/png"), \ + patch("pathlib.Path.exists", return_value=True): + + mock_async.return_value = MagicMock(content="Looks like a cat.", reasoning_content=None) + await vision_analyze_tool( + image_url="https://example.com/img.png", + user_prompt="Describe this", + model="gpt-4o", + provider="openrouter", + ) + args, kwargs = mock_async.call_args + assert kwargs["task"] == "vision" + + +# --------------------------------------------------------------------------- +# 4. Fallback to AUXILIARY_VISION_MODEL env var +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_vision_fallback_to_aux_vision_model_env(): + """When no explicit model is passed, _handle_vision_analyze must pick up + AUXILIARY_VISION_MODEL from the environment.""" + os.environ["AUXILIARY_VISION_MODEL"] = "google/gemini-pro-vision" + try: + with patch("tools.vision_tools._read_main_provider", return_value="openrouter"), \ + patch("tools.vision_tools._read_main_model", return_value="invalid-model"), \ + patch("tools.vision_tools._supports_media_in_tool_results", return_value=False), \ + patch("tools.vision_tools.vision_analyze_tool") as mock_tool: + + await _handle_vision_analyze( + {"image_url": "https://example.com/img.png", "question": "What?"}, + # No model kwarg, no provider kwarg. + ) + args, kwargs = mock_tool.call_args + assert kwargs.get("model") == "google/gemini-pro-vision" + finally: + os.environ.pop("AUXILIARY_VISION_MODEL", None) + + +# --------------------------------------------------------------------------- +# 5. _resolve_task_provider_model prefers explicit args over task config +# --------------------------------------------------------------------------- + +def test_resolve_task_provider_model_prefers_explicit_over_config(): + """When explicit provider/model are passed alongside a task, the explicit + values must win over any task-level config.""" + from agent.auxiliary_client import _resolve_task_provider_model + + # mock _get_auxiliary_task_config to return a task config + with patch("agent.auxiliary_client._get_auxiliary_task_config") as mock_cfg: + mock_cfg.return_value = { + "provider": "openrouter", + "model": "anthropic/claude-3", + } + provider, model, *_ = _resolve_task_provider_model( + task="vision", + provider="some-explicit", + model="explicit-model", + ) + assert provider == "some-explicit" + assert model == "explicit-model" diff --git a/tools/vision_tools.py b/tools/vision_tools.py index 912777e2e255d..035f99a7206d2 100644 --- a/tools/vision_tools.py +++ b/tools/vision_tools.py @@ -634,6 +634,7 @@ async def vision_analyze_tool( image_url: str, user_prompt: str, model: str = None, + provider: str = None, ) -> str: """ Analyze an image from a URL or local file path using vision AI. @@ -650,7 +651,8 @@ async def vision_analyze_tool( image_url (str): The URL or local file path of the image to analyze. Accepts http://, https:// URLs or absolute/relative file paths. user_prompt (str): The pre-formatted prompt for the vision model - model (str): The vision model to use (default: google/gemini-3-flash-preview) + model (str): The vision model to use (default: auto-resolved from config or env) + provider (str): The provider slug to route the call through. Returns: str: JSON string containing the analysis results with the following structure: @@ -805,6 +807,8 @@ async def vision_analyze_tool( } if model: call_kwargs["model"] = model + if provider: + call_kwargs["provider"] = provider # Try full-size image first; on size-related rejection, downscale and retry. try: response = await async_call_llm(**call_kwargs) @@ -1043,8 +1047,9 @@ def _handle_vision_analyze(args: Dict[str, Any], **kw: Any) -> Awaitable[str]: "Fully describe and explain everything about this image, then answer the " f"following question:\n\n{question}" ) - model = os.getenv("AUXILIARY_VISION_MODEL", "").strip() or None - return vision_analyze_tool(image_url, full_prompt, model) + model = kw.get("model") or os.getenv("AUXILIARY_VISION_MODEL", "").strip() or None + provider = kw.get("provider") or None + return vision_analyze_tool(image_url, full_prompt, model, provider) registry.register(