[.Net] Support image input for Anthropic Models

dotnet/src/AutoGen.Anthropic/AnthropicClient.cs

@@ -23,7 +23,8 @@ public sealed class AnthropicClient : IDisposable
 
     private static readonly JsonSerializerOptions JsonSerializerOptions = new()
     {
-        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
+        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+        Converters = { new ContentBaseConverter() }
     };
 
     private static readonly JsonSerializerOptions JsonDeserializerOptions = new()

dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs

@@ -1,11 +1,10 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 
 using System.Text.Json.Serialization;
+using System.Collections.Generic;
 
 namespace AutoGen.Anthropic.DTO;
 
-using System.Collections.Generic;
-
 public class ChatCompletionRequest
 {
     [JsonPropertyName("model")]
@@ -50,9 +49,15 @@ public class ChatMessage
     public string Role { get; set; }
 
     [JsonPropertyName("content")]
-    public string Content { get; set; }
+    public List<ContentBase> Content { get; set; }
 
     public ChatMessage(string role, string content)
+    {
+        Role = role;
+        Content = new List<ContentBase>() { new TextContent { Text = content } };
+    }
+
+    public ChatMessage(string role, List<ContentBase> content)
     {
         Role = role;
         Content = content;

dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Http;
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
@@ -19,7 +20,7 @@ public class AnthropicMessageConnector : IStreamingMiddleware
     public async Task<IMessage> InvokeAsync(MiddlewareContext context, IAgent agent, CancellationToken cancellationToken = default)
     {
         var messages = context.Messages;
-        var chatMessages = ProcessMessage(messages, agent);
+        var chatMessages = await ProcessMessageAsync(messages, agent);
         var response = await agent.GenerateReplyAsync(chatMessages, context.Options, cancellationToken);
 
         return response is IMessage<ChatCompletionResponse> chatMessage
@@ -31,7 +32,7 @@ public async IAsyncEnumerable<IStreamingMessage> InvokeAsync(MiddlewareContext c
         [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
         var messages = context.Messages;
-        var chatMessages = ProcessMessage(messages, agent);
+        var chatMessages = await ProcessMessageAsync(messages, agent);
 
         await foreach (var reply in agent.GenerateStreamingReplyAsync(chatMessages, context.Options, cancellationToken))
         {
@@ -53,60 +54,78 @@ public async IAsyncEnumerable<IStreamingMessage> InvokeAsync(MiddlewareContext c
     private IStreamingMessage? ProcessChatCompletionResponse(IStreamingMessage<ChatCompletionResponse> chatMessage,
         IStreamingAgent agent)
     {
-        Delta? delta = chatMessage.Content.Delta;
+        var delta = chatMessage.Content.Delta;
         return delta != null && !string.IsNullOrEmpty(delta.Text)
             ? new TextMessageUpdate(role: Role.Assistant, delta.Text, from: agent.Name)
             : null;
     }
 
-    private IEnumerable<IMessage> ProcessMessage(IEnumerable<IMessage> messages, IAgent agent)
+    private async Task<IEnumerable<IMessage>> ProcessMessageAsync(IEnumerable<IMessage> messages, IAgent agent)
     {
-        return messages.SelectMany<IMessage, IMessage>(m =>
+        var processedMessages = new List<IMessage>();
+
+        foreach (var message in messages)
         {
-            return m switch
+            var processedMessage = message switch
             {
                 TextMessage textMessage => ProcessTextMessage(textMessage, agent),
-                _ => [m],
+
+                ImageMessage imageMessage =>
+                    new MessageEnvelope<ChatMessage>(new ChatMessage("user",
+                            new ContentBase[] { new ImageContent { Source = await ProcessImageSourceAsync(imageMessage) } }
+                                .ToList()),
+                        from: agent.Name),
+
+                MultiModalMessage multiModalMessage => await ProcessMultiModalMessageAsync(multiModalMessage, agent),
+                _ => message,
             };
-        });
+
+            processedMessages.Add(processedMessage);
+        }
+
+        return processedMessages;
     }
 
     private IMessage PostProcessMessage(ChatCompletionResponse response, IAgent from)
     {
         if (response.Content is null)
+        {
             throw new ArgumentNullException(nameof(response.Content));
+        }
 
         if (response.Content.Count != 1)
+        {
             throw new NotSupportedException($"{nameof(response.Content)} != 1");
+        }
 
         return new TextMessage(Role.Assistant, ((TextContent)response.Content[0]).Text ?? string.Empty, from: from.Name);
     }
 
-    private IEnumerable<IMessage<ChatMessage>> ProcessTextMessage(TextMessage textMessage, IAgent agent)
+    private IMessage<ChatMessage> ProcessTextMessage(TextMessage textMessage, IAgent agent)
     {
-        IEnumerable<ChatMessage> messages;
+        ChatMessage messages;
 
         if (textMessage.From == agent.Name)
         {
-            messages = [new ChatMessage(
-            "assistant", textMessage.Content)];
+            messages = new ChatMessage(
+                "assistant", textMessage.Content);
         }
         else if (textMessage.From is null)
         {
             if (textMessage.Role == Role.User)
             {
-                messages = [new ChatMessage(
-                "user", textMessage.Content)];
+                messages = new ChatMessage(
+                    "user", textMessage.Content);
             }
             else if (textMessage.Role == Role.Assistant)
             {
-                messages = [new ChatMessage(
-                "assistant", textMessage.Content)];
+                messages = new ChatMessage(
+                    "assistant", textMessage.Content);
             }
             else if (textMessage.Role == Role.System)
             {
-                messages = [new ChatMessage(
-                "system", textMessage.Content)];
+                messages = new ChatMessage(
+                    "system", textMessage.Content);
             }
             else
             {
@@ -116,10 +135,61 @@ private IEnumerable<IMessage<ChatMessage>> ProcessTextMessage(TextMessage textMe
         else
         {
             // if from is not null, then the message is from user
-            messages = [new ChatMessage(
-            "user", textMessage.Content)];
+            messages = new ChatMessage(
+                "user", textMessage.Content);
         }
 
-        return messages.Select(m => new MessageEnvelope<ChatMessage>(m, from: textMessage.From));
+        return new MessageEnvelope<ChatMessage>(messages, from: textMessage.From);
+    }
+
+    private async Task<IMessage> ProcessMultiModalMessageAsync(MultiModalMessage multiModalMessage, IAgent agent)
+    {
+        var content = new List<ContentBase>();
+        foreach (var message in multiModalMessage.Content)
+        {
+            switch (message)
+            {
+                case TextMessage textMessage when textMessage.GetContent() is not null:
+                    content.Add(new TextContent { Text = textMessage.GetContent() });
+                    break;
+                case ImageMessage imageMessage:
+                    content.Add(new ImageContent() { Source = await ProcessImageSourceAsync(imageMessage) });
+                    break;
+            }
+        }
+
+        var chatMessage = new ChatMessage("user", content);
+        return MessageEnvelope.Create(chatMessage, agent.Name);
+    }
+
+    private async Task<ImageSource> ProcessImageSourceAsync(ImageMessage imageMessage)
+    {
+        if (imageMessage.Data != null)
+        {
+            return new ImageSource
+            {
+                MediaType = imageMessage.Data.MediaType,
+                Data = Convert.ToBase64String(imageMessage.Data.ToArray())
+            };
+        }
+
+        if (imageMessage.Url is null)
+        {
+            throw new InvalidOperationException("Invalid ImageMessage, the data or url must be provided");
+        }
+
+        var uri = new Uri(imageMessage.Url);
+        using var client = new HttpClient();
+        var response = client.GetAsync(uri).Result;
+        if (!response.IsSuccessStatusCode)
+        {
+            throw new HttpRequestException($"Failed to download the image from {uri}");
+        }
+
+        return new ImageSource
+        {
+            MediaType = "image/jpeg",
+            Data = Convert.ToBase64String(await response.Content.ReadAsByteArrayAsync())
+        };
     }
 }

dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs

@@ -49,7 +49,6 @@ public Task<IMessage> GenerateReplyAsync(IEnumerable<IMessage> messages, Generat
 
     public IAsyncEnumerable<IStreamingMessage> GenerateStreamingReplyAsync(IEnumerable<IMessage> messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default)
     {
-
         return _agent.GenerateStreamingReplyAsync(messages, options, cancellationToken);
     }
 

dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs

@@ -1,31 +1,108 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // AnthropicClientAgentTest.cs
 
+using AutoGen.Anthropic.DTO;
 using AutoGen.Anthropic.Extensions;
 using AutoGen.Anthropic.Utils;
+using AutoGen.Core;
 using AutoGen.Tests;
-using Xunit.Abstractions;
+using FluentAssertions;
 
-namespace AutoGen.Anthropic;
+namespace AutoGen.Anthropic.Tests;
 
 public class AnthropicClientAgentTest
 {
-    private readonly ITestOutputHelper _output;
-
-    public AnthropicClientAgentTest(ITestOutputHelper output) => _output = output;
-
     [ApiKeyFact("ANTHROPIC_API_KEY")]
     public async Task AnthropicAgentChatCompletionTestAsync()
     {
         var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey);
 
+        var agent = new AnthropicClientAgent(
+            client,
+            name: "AnthropicAgent",
+            AnthropicConstants.Claude3Haiku,
+            systemMessage: "You are a helpful AI assistant that convert user message to upper case")
+            .RegisterMessageConnector();
+
+        var uppCaseMessage = new TextMessage(Role.User, "abcdefg");
+
+        var reply = await agent.SendAsync(chatHistory: new[] { uppCaseMessage });
+
+        reply.GetContent().Should().Contain("ABCDEFG");
+        reply.From.Should().Be(agent.Name);
+    }
+
+    [ApiKeyFact("ANTHROPIC_API_KEY")]
+    public async Task AnthropicAgentTestProcessImageAsync()
+    {
+        var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey);
         var agent = new AnthropicClientAgent(
             client,
             name: "AnthropicAgent",
             AnthropicConstants.Claude3Haiku).RegisterMessageConnector();
 
-        var singleAgentTest = new SingleAgentTest(_output);
-        await singleAgentTest.UpperCaseTestAsync(agent);
-        await singleAgentTest.UpperCaseStreamingTestAsync(agent);
+        var base64Image = await AnthropicTestUtils.Base64FromImageAsync("square.png");
+        var imageMessage = new ChatMessage("user",
+            [new ImageContent { Source = new ImageSource { MediaType = "image/png", Data = base64Image } }]);
+
+        var messages = new IMessage[] { MessageEnvelope.Create(imageMessage) };
+
+        // test streaming
+        foreach (var message in messages)
+        {
+            var reply = agent.GenerateStreamingReplyAsync([message]);
+
+            await foreach (var streamingMessage in reply)
+            {
+                streamingMessage.Should().BeOfType<TextMessageUpdate>();
+                streamingMessage.As<TextMessageUpdate>().From.Should().Be(agent.Name);
+            }
+        }
+    }
+
+    [ApiKeyFact("ANTHROPIC_API_KEY")]
+    public async Task AnthropicAgentTestMultiModalAsync()
+    {
+        var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey);
+        var agent = new AnthropicClientAgent(
+            client,
+            name: "AnthropicAgent",
+            AnthropicConstants.Claude3Haiku)
+            .RegisterMessageConnector();
+
+        var image = Path.Combine("images", "square.png");
+        var binaryData = BinaryData.FromBytes(await File.ReadAllBytesAsync(image), "image/png");
+        var imageMessage = new ImageMessage(Role.User, binaryData);
+        var textMessage = new TextMessage(Role.User, "What's in this image?");
+        var multiModalMessage = new MultiModalMessage(Role.User, [textMessage, imageMessage]);
+
+        var reply = await agent.SendAsync(multiModalMessage);
+        reply.Should().BeOfType<TextMessage>();
+        reply.GetRole().Should().Be(Role.Assistant);
+        reply.GetContent().Should().NotBeNullOrEmpty();
+        reply.From.Should().Be(agent.Name);
+    }
+
+    [ApiKeyFact("ANTHROPIC_API_KEY")]
+    public async Task AnthropicAgentTestImageMessageAsync()
+    {
+        var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey);
+        var agent = new AnthropicClientAgent(
+                client,
+                name: "AnthropicAgent",
+                AnthropicConstants.Claude3Haiku,
+                systemMessage: "You are a helpful AI assistant that is capable of determining what an image is. Tell me a brief description of the image."
+                )
+            .RegisterMessageConnector();
+
+        var image = Path.Combine("images", "square.png");
+        var binaryData = BinaryData.FromBytes(await File.ReadAllBytesAsync(image), "image/png");
+        var imageMessage = new ImageMessage(Role.User, binaryData);
+
+        var reply = await agent.SendAsync(imageMessage);
+        reply.Should().BeOfType<TextMessage>();
+        reply.GetRole().Should().Be(Role.Assistant);
+        reply.GetContent().Should().NotBeNullOrEmpty();
+        reply.From.Should().Be(agent.Name);
     }
 }