fix: vision support detection + image error recovery (#638)

agent/model_metadata.py

@@ -222,3 +222,52 @@ def estimate_messages_tokens_rough(messages: List[Dict[str, Any]]) -> int:
     """Rough token estimate for a message list (pre-flight only)."""
     total_chars = sum(len(str(msg)) for msg in messages)
     return total_chars // 4
+
+
+# ---------------------------------------------------------------------------
+# Vision capability detection  (Issue #638)
+# ---------------------------------------------------------------------------
+
+# Model name substrings that indicate vision/multimodal support.
+VISION_CAPABLE_MODEL_SUBSTRINGS = [
+    "gpt-4o",
+    "gpt-4-turbo",
+    "gpt-4-vision",
+    "claude-3",
+    "claude-sonnet",
+    "claude-opus",
+    "claude-haiku",
+    "gemini",
+    "pixtral",
+    "llama-3.2-11b-vision",
+    "llama-3.2-90b-vision",
+    "qwen-vl",
+    "qwen2-vl",
+    "vision",
+]
+
+# Provider base_url substrings that are known to NOT support vision.
+NON_VISION_PROVIDERS = [
+    "api.nous.systems",
+]
+
+
+def model_supports_vision(model: str, base_url: str = "") -> bool:
+    """Return True if this model/provider combo is expected to accept
+    image_url content parts.
+
+    Best-effort heuristic based on known model names and provider URLs.
+    Returns False (safe default) when unknown.
+    """
+    model_lower = model.lower()
+    base_url_lower = (base_url or "").lower()
+
+    for provider_substr in NON_VISION_PROVIDERS:
+        if provider_substr in base_url_lower:
+            return False
+
+    for substr in VISION_CAPABLE_MODEL_SUBSTRINGS:
+        if substr in model_lower:
+            return True
+
+    return False

cli.py

@@ -2528,12 +2528,35 @@ def chat(self, message, images: list = None) -> Optional[str]:
 
         # Convert attached images to OpenAI vision multimodal content
         if images:
-            message = self._build_multimodal_content(
-                message if isinstance(message, str) else "", images
-            )
-            for img_path in images:
-                if img_path.exists():
-                    _cprint(f"  {_DIM}📎 attached {img_path.name} ({img_path.stat().st_size // 1024}KB){_RST}")
+            from agent.model_metadata import model_supports_vision
+            base_url = getattr(self.agent, "base_url", "") if self.agent else ""
+            model = getattr(self.agent, "model", "") if self.agent else ""
+            if not model_supports_vision(model, base_url):
+                _cprint(f"  \033[33m⚠️  This model does not support images. The image was not sent.\033[0m")
+                _cprint(f"  \033[2mSwitch to a vision-capable model (e.g. gpt-4o, claude-3, gemini) to send images.\033[0m")
+                # Offer to describe the images via the vision_analyze tool instead
+                image_names = [p.name for p in images if p.exists()]
+                if image_names:
+                    names_str = ", ".join(image_names)
+                    if isinstance(message, str) and message.strip():
+                        message = (
+                            message
+                            + f"\n\n[The user attached image(s) that this model cannot process directly: {names_str}. "
+                            "If a vision tool is available, use it to analyze and describe the image(s) "
+                            "and incorporate the description into your response.]"
+                        )
+                    else:
+                        message = (
+                            f"[The user attached image(s) that this model cannot process directly: {names_str}. "
+                            "If a vision tool is available, use it to analyze and describe the image(s).]"
+                        )
+            else:
+                message = self._build_multimodal_content(
+                    message if isinstance(message, str) else "", images
+                )
+                for img_path in images:
+                    if img_path.exists():
+                        _cprint(f"  {_DIM}📎 attached {img_path.name} ({img_path.stat().st_size // 1024}KB){_RST}")
 
         # Add user message to history
         self.conversation_history.append({"role": "user", "content": message})

run_agent.py

@@ -3550,6 +3550,40 @@ def run_conversation(
                                 "partial": True
                             }
 
+                    # Check for image-related errors (Issue #638).
+                    # If the API rejected our request because of image/vision content,
+                    # strip all image_url parts from messages and retry once so that
+                    # subsequent text-only turns are not affected.
+                    image_error_signals = [
+                        "image_url", "unsupported content", "invalid content",
+                        "does not support", "image input", "multimodal",
+                        "vision", "base64",
+                    ]
+                    is_image_error = any(sig in error_msg for sig in image_error_signals)
+
+                    if is_image_error:
+                        removed = 0
+                        cleaned = []
+                        for msg in messages:
+                            c = msg.get("content")
+                            if isinstance(c, list):
+                                new_parts = [p for p in c if not (isinstance(p, dict) and p.get("type") == "image_url")]
+                                removed += len(c) - len(new_parts)
+                                if len(new_parts) == 1 and isinstance(new_parts[0], dict) and new_parts[0].get("type") == "text":
+                                    cleaned.append({**msg, "content": new_parts[0]["text"]})
+                                elif new_parts:
+                                    cleaned.append({**msg, "content": new_parts})
+                                # drop message entirely if it had only images
+                            else:
+                                cleaned.append(msg)
+                        if removed:
+                            messages = cleaned
+                            print(f"{self.log_prefix}⚠️  API rejected image content — stripped {removed} image(s) from history and retrying without images.")
+                            print(f"{self.log_prefix}   💡 This model does not support vision input. Use a vision-capable model for images.")
+                            retry_count -= 1  # don't count this against retry budget
+                            continue
+                        # no images found — fall through to normal error handling
+
                     # Check for non-retryable client errors (4xx HTTP status codes).
                     # These indicate a problem with the request itself (bad model ID,
                     # invalid API key, forbidden, etc.) and will never succeed on retry.