From c3c771dff18f757ad1f96fba1645752421d14d10 Mon Sep 17 00:00:00 2001 From: teyrebaz33 Date: Sun, 8 Mar 2026 12:46:01 +0300 Subject: [PATCH] fix: vision support detection + image error recovery (#638) - Add model_supports_vision() to agent/model_metadata.py with a known-models substring list and a provider deny-list for non-vision endpoints (e.g. api.nous.systems) - cli.py: check model_supports_vision() before attaching images; warn the user and discard the image if unsupported; fall back to vision_analyze tool with a descriptive prompt for graceful degradation - run_agent.py: detect image-related API errors, strip image_url parts from conversation_history, and retry so subsequent text-only messages are not broken Fixes #638 --- agent/model_metadata.py | 49 +++++++++++++++++++++++++++++++++++++++++ cli.py | 35 ++++++++++++++++++++++++----- run_agent.py | 34 ++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 6 deletions(-) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 3b2ab9d0f1a2..2a99e6616f29 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -222,3 +222,52 @@ def estimate_messages_tokens_rough(messages: List[Dict[str, Any]]) -> int: """Rough token estimate for a message list (pre-flight only).""" total_chars = sum(len(str(msg)) for msg in messages) return total_chars // 4 + + +# --------------------------------------------------------------------------- +# Vision capability detection (Issue #638) +# --------------------------------------------------------------------------- + +# Model name substrings that indicate vision/multimodal support. +VISION_CAPABLE_MODEL_SUBSTRINGS = [ + "gpt-4o", + "gpt-4-turbo", + "gpt-4-vision", + "claude-3", + "claude-sonnet", + "claude-opus", + "claude-haiku", + "gemini", + "pixtral", + "llama-3.2-11b-vision", + "llama-3.2-90b-vision", + "qwen-vl", + "qwen2-vl", + "vision", +] + +# Provider base_url substrings that are known to NOT support vision. +NON_VISION_PROVIDERS = [ + "api.nous.systems", +] + + +def model_supports_vision(model: str, base_url: str = "") -> bool: + """Return True if this model/provider combo is expected to accept + image_url content parts. + + Best-effort heuristic based on known model names and provider URLs. + Returns False (safe default) when unknown. + """ + model_lower = model.lower() + base_url_lower = (base_url or "").lower() + + for provider_substr in NON_VISION_PROVIDERS: + if provider_substr in base_url_lower: + return False + + for substr in VISION_CAPABLE_MODEL_SUBSTRINGS: + if substr in model_lower: + return True + + return False diff --git a/cli.py b/cli.py index 6c44ef61b7f7..1e10a48bc9c3 100755 --- a/cli.py +++ b/cli.py @@ -2528,12 +2528,35 @@ def chat(self, message, images: list = None) -> Optional[str]: # Convert attached images to OpenAI vision multimodal content if images: - message = self._build_multimodal_content( - message if isinstance(message, str) else "", images - ) - for img_path in images: - if img_path.exists(): - _cprint(f" {_DIM}📎 attached {img_path.name} ({img_path.stat().st_size // 1024}KB){_RST}") + from agent.model_metadata import model_supports_vision + base_url = getattr(self.agent, "base_url", "") if self.agent else "" + model = getattr(self.agent, "model", "") if self.agent else "" + if not model_supports_vision(model, base_url): + _cprint(f" \033[33m⚠️ This model does not support images. The image was not sent.\033[0m") + _cprint(f" \033[2mSwitch to a vision-capable model (e.g. gpt-4o, claude-3, gemini) to send images.\033[0m") + # Offer to describe the images via the vision_analyze tool instead + image_names = [p.name for p in images if p.exists()] + if image_names: + names_str = ", ".join(image_names) + if isinstance(message, str) and message.strip(): + message = ( + message + + f"\n\n[The user attached image(s) that this model cannot process directly: {names_str}. " + "If a vision tool is available, use it to analyze and describe the image(s) " + "and incorporate the description into your response.]" + ) + else: + message = ( + f"[The user attached image(s) that this model cannot process directly: {names_str}. " + "If a vision tool is available, use it to analyze and describe the image(s).]" + ) + else: + message = self._build_multimodal_content( + message if isinstance(message, str) else "", images + ) + for img_path in images: + if img_path.exists(): + _cprint(f" {_DIM}📎 attached {img_path.name} ({img_path.stat().st_size // 1024}KB){_RST}") # Add user message to history self.conversation_history.append({"role": "user", "content": message}) diff --git a/run_agent.py b/run_agent.py index 75e3dfc95fef..8fedb1360b48 100644 --- a/run_agent.py +++ b/run_agent.py @@ -3550,6 +3550,40 @@ def run_conversation( "partial": True } + # Check for image-related errors (Issue #638). + # If the API rejected our request because of image/vision content, + # strip all image_url parts from messages and retry once so that + # subsequent text-only turns are not affected. + image_error_signals = [ + "image_url", "unsupported content", "invalid content", + "does not support", "image input", "multimodal", + "vision", "base64", + ] + is_image_error = any(sig in error_msg for sig in image_error_signals) + + if is_image_error: + removed = 0 + cleaned = [] + for msg in messages: + c = msg.get("content") + if isinstance(c, list): + new_parts = [p for p in c if not (isinstance(p, dict) and p.get("type") == "image_url")] + removed += len(c) - len(new_parts) + if len(new_parts) == 1 and isinstance(new_parts[0], dict) and new_parts[0].get("type") == "text": + cleaned.append({**msg, "content": new_parts[0]["text"]}) + elif new_parts: + cleaned.append({**msg, "content": new_parts}) + # drop message entirely if it had only images + else: + cleaned.append(msg) + if removed: + messages = cleaned + print(f"{self.log_prefix}⚠️ API rejected image content — stripped {removed} image(s) from history and retrying without images.") + print(f"{self.log_prefix} 💡 This model does not support vision input. Use a vision-capable model for images.") + retry_count -= 1 # don't count this against retry budget + continue + # no images found — fall through to normal error handling + # Check for non-retryable client errors (4xx HTTP status codes). # These indicate a problem with the request itself (bad model ID, # invalid API key, forbidden, etc.) and will never succeed on retry.