feat(attachments): image attachment support for vision-capable models (#4644)

.gitignore

@@ -57,6 +57,11 @@ trace_*.json
 .ironclaw/
 .review/
 
+# Local-dev workspace artifacts: the WebChat v2 attachment lander writes
+# uploaded files under a project workspace; when `serve` runs from the repo
+# root they land in ./attachments/. These are user uploads, never source.
+/attachments/
+
 # Python cache
 __pycache__/
 *.pyc

crates/ironclaw_llm/src/anthropic_oauth.rs

@@ -19,9 +19,9 @@ use crate::config::RegistryProviderConfig;
 use crate::costs;
 use crate::error::LlmError;
 use crate::provider::{
-    ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, Role, ToolCall,
-    ToolCompletionRequest, ToolCompletionResponse, strip_unsupported_completion_params,
-    strip_unsupported_tool_params,
+    ChatMessage, CompletionRequest, CompletionResponse, ContentPart, FinishReason, LlmProvider,
+    Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse,
+    strip_unsupported_completion_params, strip_unsupported_tool_params,
 };
 
 /// Read a fresh `claude login` OAuth token from the OS credential store.
@@ -483,6 +483,8 @@ enum AnthropicContent {
 enum AnthropicContentBlock {
     #[serde(rename = "text")]
     Text { text: String },
+    #[serde(rename = "image")]
+    Image { source: AnthropicImageSource },
     #[serde(rename = "tool_use")]
     ToolUse {
         id: String,
@@ -496,6 +498,15 @@ enum AnthropicContentBlock {
     },
 }
 
+/// Inline base64 image source for an Anthropic `image` content block.
+#[derive(Debug, Serialize)]
+struct AnthropicImageSource {
+    #[serde(rename = "type")]
+    source_type: &'static str,
+    media_type: String,
+    data: String,
+}
+
 #[derive(Debug, Serialize)]
 struct AnthropicTool {
     name: String,
@@ -557,6 +568,29 @@ struct AnthropicUsage {
     cache_read_input_tokens: u32,
 }
 
+/// Build Anthropic `image` content blocks from a user message's multimodal
+/// parts. Only inline base64 `data:` images are forwarded (the Anthropic
+/// messages API also accepts `url` sources, but the model gateway always emits
+/// `data:` URLs); anything else is skipped so the text still reaches the model.
+fn user_image_blocks(parts: &[ContentPart]) -> Vec<AnthropicContentBlock> {
+    parts
+        .iter()
+        .filter_map(|part| match part {
+            ContentPart::ImageUrl { image_url } => {
+                let (media_type, data) = image_url.decode_data_url()?;
+                Some(AnthropicContentBlock::Image {
+                    source: AnthropicImageSource {
+                        source_type: "base64",
+                        media_type: media_type.to_string(),
+                        data: data.to_string(),
+                    },
+                })
+            }
+            ContentPart::Text { .. } => None,
+        })
+        .collect()
+}
+
 /// Convert ChatMessage list to Anthropic format.
 ///
 /// Extracts system messages to the top-level `system` parameter (Anthropic
@@ -574,9 +608,22 @@ fn convert_messages(messages: Vec<ChatMessage>) -> (Option<String>, Vec<Anthropi
                 }
             }
             Role::User => {
+                let content = match user_image_blocks(&msg.content_parts) {
+                    // Text-only (or no inline images): keep the compact string form.
+                    blocks if blocks.is_empty() => AnthropicContent::Text(msg.content),
+                    // Multimodal: text block first (when present), then images.
+                    image_blocks => {
+                        let mut blocks = Vec::with_capacity(1 + image_blocks.len());
+                        if !msg.content.is_empty() {
+                            blocks.push(AnthropicContentBlock::Text { text: msg.content });
+                        }
+                        blocks.extend(image_blocks);
+                        AnthropicContent::Blocks(blocks)
+                    }
+                };
                 anthropic_msgs.push(AnthropicMessage {
                     role: "user".to_string(),
-                    content: AnthropicContent::Text(msg.content),
+                    content,
                 });
             }
             Role::Assistant => {
@@ -614,12 +661,18 @@ fn convert_messages(messages: Vec<ChatMessage>) -> (Option<String>, Vec<Anthropi
                     tool_use_id: tool_call_id,
                     content: msg.content,
                 };
-                // If the last message is already a user message with blocks,
-                // append to it (Anthropic requires consecutive tool results
-                // in one user message).
+                // If the last message is already a user message of *only*
+                // tool-result blocks, append to it (Anthropic requires
+                // consecutive tool results in one user message). Crucially, do
+                // not merge into a multimodal user prompt (text + image
+                // blocks) — that would fold a tool result into a different
+                // conversational turn.
                 if let Some(last) = anthropic_msgs.last_mut()
                     && last.role == "user"
                     && let AnthropicContent::Blocks(ref mut blocks) = last.content
+                    && blocks
+                        .iter()
+                        .all(|b| matches!(b, AnthropicContentBlock::ToolResult { .. }))
                 {
                     blocks.push(block);
                     continue;
@@ -737,6 +790,40 @@ mod tests {
         assert_eq!(msgs.len(), 1);
     }
 
+    #[test]
+    fn test_convert_messages_user_image_becomes_base64_image_block() {
+        let messages = vec![ChatMessage::user_with_parts(
+            "what is this?",
+            vec![ContentPart::ImageUrl {
+                image_url: crate::provider::ImageUrl {
+                    url: "data:image/png;base64,AQIDBA==".to_string(),
+                    detail: None,
+                },
+            }],
+        )];
+        let (_system, msgs) = convert_messages(messages);
+        assert_eq!(msgs.len(), 1);
+        // Text rides as the first block, the image as a base64 `image` block.
+        let value = serde_json::to_value(&msgs[0]).expect("serialize");
+        let blocks = value["content"].as_array().expect("content blocks");
+        assert_eq!(blocks.len(), 2);
+        assert_eq!(blocks[0]["type"], "text");
+        assert_eq!(blocks[0]["text"], "what is this?");
+        assert_eq!(blocks[1]["type"], "image");
+        assert_eq!(blocks[1]["source"]["type"], "base64");
+        assert_eq!(blocks[1]["source"]["media_type"], "image/png");
+        assert_eq!(blocks[1]["source"]["data"], "AQIDBA==");
+    }
+
+    #[test]
+    fn test_convert_messages_text_only_user_stays_a_string() {
+        let messages = vec![ChatMessage::user("just text")];
+        let (_system, msgs) = convert_messages(messages);
+        let value = serde_json::to_value(&msgs[0]).expect("serialize");
+        // No inline images → compact string content, not a blocks array.
+        assert_eq!(value["content"], "just text");
+    }
+
     #[test]
     fn test_convert_messages_tool_calls() {
         let tool_calls = vec![ToolCall {

crates/ironclaw_llm/src/bedrock.rs

@@ -13,11 +13,12 @@ use aws_config::{BehaviorVersion, Region};
 use aws_sdk_bedrockruntime::Client;
 use aws_sdk_bedrockruntime::operation::converse::ConverseError;
 use aws_sdk_bedrockruntime::types::{
-    AnyToolChoice, AutoToolChoice, ContentBlock, ConversationRole, InferenceConfiguration, Message,
-    StopReason, SystemContentBlock, Tool, ToolChoice, ToolConfiguration, ToolInputSchema,
-    ToolResultBlock, ToolResultContentBlock, ToolResultStatus, ToolSpecification, ToolUseBlock,
+    AnyToolChoice, AutoToolChoice, ContentBlock, ConversationRole, ImageBlock, ImageFormat,
+    ImageSource, InferenceConfiguration, Message, StopReason, SystemContentBlock, Tool, ToolChoice,
+    ToolConfiguration, ToolInputSchema, ToolResultBlock, ToolResultContentBlock, ToolResultStatus,
+    ToolSpecification, ToolUseBlock,
 };
-use aws_smithy_types::Document;
+use aws_smithy_types::{Blob, Document};
 use rust_decimal::Decimal;
 
 use crate::config::BedrockConfig;
@@ -329,6 +330,52 @@ fn strip_tool_blocks(messages: &mut [crate::provider::ChatMessage]) {
 // Message conversion
 // ---------------------------------------------------------------------------
 
+/// Map a MIME type to the Converse API [`ImageFormat`]. Bedrock supports only
+/// these four; an unsupported type yields `None` so the image is skipped (the
+/// text still reaches the model) rather than failing the turn.
+fn bedrock_image_format(mime_type: &str) -> Option<ImageFormat> {
+    // MIME types are case-insensitive; normalize before matching.
+    match mime_type.to_ascii_lowercase().as_str() {
+        "image/png" => Some(ImageFormat::Png),
+        "image/jpeg" | "image/jpg" => Some(ImageFormat::Jpeg),
+        "image/gif" => Some(ImageFormat::Gif),
+        "image/webp" => Some(ImageFormat::Webp),
+        _ => None,
+    }
+}
+
+/// Build a Converse `image` content block from an inline base64 `data:` URL.
+/// Returns `None` for a remote URL, an unsupported format, or undecodable
+/// base64 — the caller drops the image and keeps the text.
+fn bedrock_image_block(image_url: &crate::provider::ImageUrl) -> Option<ContentBlock> {
+    use base64::Engine;
+    let (mime_type, data) = image_url.decode_data_url()?;
+    let format = bedrock_image_format(mime_type)?;
+    let bytes = match base64::engine::general_purpose::STANDARD.decode(data) {
+        Ok(bytes) => bytes,
+        // silent-ok: a malformed inline image payload shouldn't fail the turn —
+        // the text still goes through; log the cause so the drop is diagnosable.
+        Err(error) => {
+            tracing::debug!(%error, "dropping malformed inline image data URL for Bedrock");
+            return None;
+        }
+    };
+    let block = match ImageBlock::builder()
+        .format(format)
+        .source(ImageSource::Bytes(Blob::new(bytes)))
+        .build()
+    {
+        Ok(block) => block,
+        // silent-ok: an image the Bedrock builder rejects shouldn't fail the
+        // turn; log the cause and drop just the image.
+        Err(error) => {
+            tracing::debug!(%error, "dropping inline image rejected by the Bedrock block builder");
+            return None;
+        }
+    };
+    Some(ContentBlock::Image(block))
+}
+
 /// Convert IronClaw `ChatMessage` list into Bedrock system blocks + messages.
 ///
 /// Key differences from OpenAI/Anthropic protocol:
@@ -356,7 +403,16 @@ fn convert_messages(
                 // Flush any pending tool results as a User message first
                 flush_tool_results(&mut pending_tool_results, &mut bedrock_messages)?;
 
-                let content = vec![ContentBlock::Text(msg.content.clone())];
+                let mut content = vec![ContentBlock::Text(msg.content.clone())];
+                // Forward inline base64 images as Converse `image` blocks so a
+                // vision model receives the pixels.
+                for part in &msg.content_parts {
+                    if let crate::provider::ContentPart::ImageUrl { image_url } = part
+                        && let Some(block) = bedrock_image_block(image_url)
+                    {
+                        content.push(block);
+                    }
+                }
                 push_message(&mut bedrock_messages, ConversationRole::User, content)?;
             }
             Role::Assistant => {
@@ -796,6 +852,58 @@ mod tests {
         assert_eq!(*msgs[0].role(), ConversationRole::User);
     }
 
+    #[test]
+    fn test_convert_messages_user_image_becomes_image_block() {
+        let messages = vec![ChatMessage::user_with_parts(
+            "what is this?",
+            vec![crate::provider::ContentPart::ImageUrl {
+                image_url: crate::provider::ImageUrl {
+                    // base64 of bytes [1,2,3,4]
+                    url: "data:image/png;base64,AQIDBA==".to_string(),
+                    detail: None,
+                },
+            }],
+        )];
+
+        let (_system, msgs) = convert_messages(&messages).unwrap();
+        assert_eq!(msgs.len(), 1);
+        let content = msgs[0].content();
+        // Text block first, then the decoded image block.
+        assert_eq!(content.len(), 2);
+        assert!(matches!(content[0], ContentBlock::Text(_)));
+        match &content[1] {
+            ContentBlock::Image(image) => {
+                assert_eq!(image.format(), &ImageFormat::Png);
+                match image.source() {
+                    Some(ImageSource::Bytes(blob)) => {
+                        assert_eq!(blob.as_ref(), &[1, 2, 3, 4]);
+                    }
+                    other => panic!("expected inline image bytes, got {other:?}"),
+                }
+            }
+            other => panic!("expected an image block, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn test_convert_messages_unsupported_image_format_is_skipped() {
+        // An SVG (unsupported by Bedrock) is dropped; the text still survives.
+        let messages = vec![ChatMessage::user_with_parts(
+            "look",
+            vec![crate::provider::ContentPart::ImageUrl {
+                image_url: crate::provider::ImageUrl {
+                    url: "data:image/svg+xml;base64,PHN2Zy8+".to_string(),
+                    detail: None,
+                },
+            }],
+        )];
+
+        let (_system, msgs) = convert_messages(&messages).unwrap();
+        let content = msgs[0].content();
+        assert_eq!(content.len(), 1);
+        assert!(matches!(content[0], ContentBlock::Text(_)));
+    }
+
     #[test]
     fn test_convert_messages_basic_conversation() {
         let messages = vec![

crates/ironclaw_llm/src/gemini_oauth.rs

@@ -14,8 +14,8 @@ use url::Url;
 use crate::config::GeminiOauthConfig;
 use crate::error::LlmError;
 use crate::provider::{
-    ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ModelMetadata,
-    Role, ToolCall, ToolDefinition,
+    ChatMessage, CompletionRequest, CompletionResponse, ContentPart, FinishReason, LlmProvider,
+    ModelMetadata, Role, ToolCall, ToolDefinition,
 };
 
 // Official Gemini CLI OAuth credentials (public, from google/gemini-cli).
@@ -1631,9 +1631,21 @@ impl GeminiOauthProvider {
                     // System messages are handled via systemInstruction top-level field
                 }
                 Role::User => {
+                    // Text part first, then any inline base64 images as
+                    // `inlineData` parts so a vision model receives the pixels.
+                    let mut parts = vec![serde_json::json!({ "text": msg.content })];
+                    for part in &msg.content_parts {
+                        if let ContentPart::ImageUrl { image_url } = part
+                            && let Some((mime_type, data)) = image_url.decode_data_url()
+                        {
+                            parts.push(serde_json::json!({
+                                "inlineData": { "mimeType": mime_type, "data": data }
+                            }));
+                        }
+                    }
                     contents.push(serde_json::json!({
                         "role": "user",
-                        "parts": [{ "text": msg.content }]
+                        "parts": parts
                     }));
                 }
                 Role::Assistant => {
@@ -2274,6 +2286,36 @@ mod tests {
         assert!(result.is_err());
     }
 
+    #[test]
+    fn test_to_gemini_request_forwards_user_image_as_inline_data() {
+        let messages = vec![ChatMessage::user_with_parts(
+            "what is this?",
+            vec![ContentPart::ImageUrl {
+                image_url: crate::provider::ImageUrl {
+                    url: "data:image/png;base64,AQIDBA==".to_string(),
+                    detail: None,
+                },
+            }],
+        )];
+
+        let req = GeminiOauthProvider::to_gemini_request(
+            &messages,
+            None,
+            None,
+            None,
+            None,
+            None,
+            "gemini-2.0-flash",
+            &HashMap::new(),
+        );
+
+        let parts = req["contents"][0]["parts"].as_array().expect("parts");
+        assert_eq!(parts.len(), 2);
+        assert_eq!(parts[0]["text"], "what is this?");
+        assert_eq!(parts[1]["inlineData"]["mimeType"], "image/png");
+        assert_eq!(parts[1]["inlineData"]["data"], "AQIDBA==");
+    }
+
     #[test]
     fn test_to_gemini_request_with_tools() {
         let messages = vec![ChatMessage::user("Hello")];