feat(agent): describe images for text-only models

frontend/src-tauri/src/agent.rs

@@ -63,6 +63,25 @@ const MAX_AGENT_SESSION_TITLE_CHARS: usize = 80;
 const MAX_AGENT_ERROR_CHARS: usize = 1_200;
 static NEXT_RUN_ID: AtomicU64 = AtomicU64::new(1);
 
+fn validate_session_model_lock(
+    message_count: usize,
+    persisted_model: Option<&str>,
+    requested_model: &str,
+) -> Result<(), String> {
+    if message_count == 0 {
+        return Ok(());
+    }
+    let Some(persisted_model) = persisted_model else {
+        return Ok(());
+    };
+    if persisted_model == requested_model {
+        return Ok(());
+    }
+    Err(format!(
+        "This Agent session is locked to model {persisted_model}. Start a new session to use {requested_model}."
+    ))
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct AgentConfig {
@@ -126,6 +145,8 @@ pub struct AgentSendMessageRequest {
     pub text: String,
     pub model: Option<String>,
     pub mode: Option<String>,
+    #[serde(default)]
+    pub vision_capable: bool,
 }
 
 #[derive(Debug, Clone, Deserialize)]
@@ -866,7 +887,15 @@ pub async fn agent_create_session(
         .lock()
         .await
         .insert(session.id.clone(), permission_mode);
-    configure_session_agent(&agent_manager, &session_manager, &session, &model, &mode).await?;
+    configure_session_agent(
+        &agent_manager,
+        &session_manager,
+        &session,
+        &model,
+        &mode,
+        false,
+    )
+    .await?;
     let summary = session_summary(&session);
     let _ = save_recent_project_root_inner(&app_handle, &user_id, &root);
     let detail = AgentSessionDetail {
@@ -1171,6 +1200,14 @@ pub async fn agent_send_message(
             .get_session(&request.session_id, true)
             .await
             .map_err(|e| format!("Failed to load Goose session: {e}"))?;
+        validate_session_model_lock(
+            session.message_count,
+            session
+                .model_config
+                .as_ref()
+                .map(|model| model.model_name.as_str()),
+            &model,
+        )?;
         let should_restore_autogenerated_title = should_name_session_from_prompt(&session);
         // Cancellation must be able to reverse Goose compaction or recovery
         // that rewrites history during this turn. Move the loaded conversation
@@ -1210,6 +1247,7 @@ pub async fn agent_send_message(
             &session,
             &model,
             &effective_mode,
+            request.vision_capable,
         )
         .await?;
         Ok((agent, turn_snapshot))
@@ -2236,6 +2274,7 @@ async fn configure_session_agent(
     session: &Session,
     model: &str,
     mode: &str,
+    primary_model_supports_vision: bool,
 ) -> Result<Arc<Agent>, String> {
     let agent = agent_manager
         .get_or_create_agent(session.id.clone())
@@ -2269,8 +2308,13 @@ async fn configure_session_agent(
             .map(|tool| tool.to_string())
             .collect(),
     };
-    let developer_client = MapleDeveloperClient::new(agent.extension_manager.get_context().clone())
-        .map_err(|e| format!("Failed to create Maple developer tools: {e}"))?;
+    let mut developer_context = agent.extension_manager.get_context().clone();
+    if !primary_model_supports_vision {
+        developer_context.extension_manager = Some(Arc::downgrade(&agent.extension_manager));
+    }
+    let developer_client =
+        MapleDeveloperClient::new(developer_context, primary_model_supports_vision)
+            .map_err(|e| format!("Failed to create Maple developer tools: {e}"))?;
     agent
         .extension_manager
         .add_client(
@@ -3367,6 +3411,37 @@ mod tests {
         assert_eq!(config.default_model, DEFAULT_AGENT_MODEL);
     }
 
+    #[test]
+    fn agent_send_vision_capability_is_catalog_driven_and_fails_closed() {
+        let without_capability: AgentSendMessageRequest = serde_json::from_value(json!({
+            "sessionId": "session-1",
+            "text": "Inspect the image",
+            "model": "future-vision-model",
+            "mode": "smart_approve"
+        }))
+        .unwrap();
+        assert!(!without_capability.vision_capable);
+
+        let with_capability: AgentSendMessageRequest = serde_json::from_value(json!({
+            "sessionId": "session-1",
+            "text": "Inspect the image",
+            "model": "future-vision-model",
+            "mode": "smart_approve",
+            "visionCapable": true
+        }))
+        .unwrap();
+        assert!(with_capability.vision_capable);
+    }
+
+    #[test]
+    fn agent_session_model_locks_after_first_message() {
+        assert!(validate_session_model_lock(0, Some("glm-5-2"), "gemma4-31b").is_ok());
+        assert!(validate_session_model_lock(3, Some("glm-5-2"), "glm-5-2").is_ok());
+        let error = validate_session_model_lock(3, Some("glm-5-2"), "gemma4-31b").unwrap_err();
+        assert!(error.contains("locked to model glm-5-2"));
+        assert!(error.contains("Start a new session"));
+    }
+
     #[tokio::test]
     async fn permission_policy_is_session_scoped_and_mutable_mid_run() {
         assert_eq!(