Pass tool result images to OpenAI and Gemini providers

assistant/src/providers/gemini/client.ts

@@ -213,14 +213,26 @@ export class GeminiProvider implements Provider {
           break;
         case 'tool_result': {
           let outputText = block.content;
-          // Extract additional text from contentBlocks (Gemini function responses only support text)
           if (block.contentBlocks && block.contentBlocks.length > 0) {
             const extraText = block.contentBlocks
               .filter((cb): cb is Extract<ContentBlock, { type: 'text' }> => cb.type === 'text')
               .map((cb) => cb.text);
             if (extraText.length > 0) {
               outputText = outputText + '\n' + extraText.join('\n');
             }
+            // Include images as inline data parts alongside the function response
+            // (Gemini function responses only support text, but images can be
+            // added as sibling parts in the same user message).
+            for (const cb of block.contentBlocks) {
+              if (cb.type === 'image') {
+                parts.push({
+                  inlineData: {
+                    mimeType: cb.source.media_type,
+                    data: cb.source.data,
+                  },
+                });
+              }
+            }
           }
           parts.push({
             functionResponse: {

assistant/src/providers/openai/client.ts

@@ -176,16 +176,21 @@ export class OpenAIProvider implements Provider {
         );
 
         // Emit tool results as separate tool-role messages
+        // OpenAI's API only supports string content in tool messages, so images
+        // from contentBlocks are collected and injected as a user message below.
+        const toolResultImages: ContentBlock[] = [];
         for (const tr of toolResults) {
           let textContent = tr.content;
-          // Extract additional text from contentBlocks (images can't be represented in OpenAI tool results)
           if (tr.contentBlocks && tr.contentBlocks.length > 0) {
             const extraText = tr.contentBlocks
               .filter((cb): cb is Extract<ContentBlock, { type: 'text' }> => cb.type === 'text')
               .map((cb) => cb.text);
             if (extraText.length > 0) {
               textContent = textContent + '\n' + extraText.join('\n');
             }
+            for (const cb of tr.contentBlocks) {
+              if (cb.type === 'image') toolResultImages.push(cb);
+            }
           }
           result.push({
             role: 'tool',
@@ -194,9 +199,12 @@ export class OpenAIProvider implements Provider {
           });
         }
 
-        // Emit remaining content as a user message (if any)
-        if (otherBlocks.length > 0) {
-          result.push(this.toOpenAIUserMessage(otherBlocks));
+        // Emit remaining content + any tool result images as a user message.
+        // Images from tool results (e.g. browser_screenshot) must go in a user
+        // message because OpenAI-compatible APIs don't support images in tool messages.
+        const userContent = [...otherBlocks, ...toolResultImages];
+        if (userContent.length > 0) {
+          result.push(this.toOpenAIUserMessage(userContent));
         }
       }
     }