Skip to content

fix(agent): prefer native vision when main model supports it (#44299) #44450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kyssta-exe wants to merge 1 commit into from
Closed

fix(agent): prefer native vision when main model supports it (#44299) #44450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions agent/image_routing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,12 +337,17 @@ def decide_image_input_mode(
return "text"

# auto
if _explicit_aux_vision_override(cfg):
return "text"

# Check if the main model supports vision first — if it does, use native
# mode even if auxiliary.vision is configured. The user's explicit
# auxiliary.vision config is only used as a fallback when the main model
# lacks vision support (#44299).
supports = _lookup_supports_vision(provider, model, cfg)
if supports is True:
return "native"

if _explicit_aux_vision_override(cfg):
return "text"

return "text"


Expand Down
9 changes: 4 additions & 5 deletions tests/agent/test_image_routing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,10 @@ def test_auto_with_unknown_model(self):
assert decide_image_input_mode("openrouter", "brand-new-slug", {}) == "text"

def test_auto_respects_aux_vision_override_even_for_vision_model(self):
"""If the user configured a dedicated vision backend, don't bypass it."""
"""If the main model supports vision, use native even with aux vision config (#44299)."""
cfg = {"auxiliary": {"vision": {"provider": "openrouter", "model": "google/gemini-2.5-flash"}}}
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
assert decide_image_input_mode("anthropic", "claude-sonnet-4", cfg) == "text"
assert decide_image_input_mode("anthropic", "claude-sonnet-4", cfg) == "native"

def test_none_config_is_auto(self):
with patch("agent.image_routing._lookup_supports_vision", return_value=True):
Expand Down Expand Up @@ -280,14 +280,13 @@ def test_auto_text_for_custom_with_no_override(self):
assert decide_image_input_mode("custom", "unknown", {}) == "text"

def test_explicit_aux_vision_override_still_wins(self):
# If the user has configured a dedicated vision aux backend, respect
# it even when supports_vision: true is also set.
# If the main model supports vision, use native even with aux vision config (#44299).
cfg = {
"model": {"supports_vision": True},
"auxiliary": {"vision": {"provider": "openrouter", "model": "gemini-2.5-pro"}},
}
with patch("agent.models_dev.get_model_capabilities", return_value=None):
assert decide_image_input_mode("custom", "qwen3.6-35b", cfg) == "text"
assert decide_image_input_mode("custom", "qwen3.6-35b", cfg) == "native"


# ─── build_native_content_parts ──────────────────────────────────────────────
Expand Down
Loading