From 540997cebce9ec06b783f9df738cbc20af71bdc1 Mon Sep 17 00:00:00 2001 From: David Luong Date: Sun, 30 Jun 2024 19:21:34 -0400 Subject: [PATCH] [.Net] Support tools for AnthropicClient and AnthropicAgent (#2944) * Squash commits : support anthropic tools * Support tool_choice * Remove reference from TypeSafeFunctionCallCodeSnippet.cs and add own function in test proj --- .../AutoGen.Anthropic.Samples.csproj | 1 + ...icSamples.cs => Create_Anthropic_Agent.cs} | 2 +- .../Create_Anthropic_Agent_With_Tool.cs | 100 +++++++++++++++ .../AutoGen.Anthropic.Samples/Program.cs | 2 +- .../Agent/AnthropicClientAgent.cs | 11 +- .../src/AutoGen.Anthropic/AnthropicClient.cs | 107 ++++++++++++++-- .../Converters/ContentBaseConverter.cs | 4 + .../JsonPropertyNameEnumCoverter.cs | 44 +++++++ .../DTO/ChatCompletionRequest.cs | 8 ++ .../DTO/ChatCompletionResponse.cs | 6 +- dotnet/src/AutoGen.Anthropic/DTO/Content.cs | 28 +++++ dotnet/src/AutoGen.Anthropic/DTO/Tool.cs | 40 ++++++ .../src/AutoGen.Anthropic/DTO/ToolChoice.cs | 39 ++++++ .../Middleware/AnthropicMessageConnector.cs | 114 ++++++++++++++++-- .../AnthropicClientAgentTest.cs | 97 +++++++++++++++ .../AnthropicClientTest.cs | 52 ++++++++ .../AnthropicTestFunctionCalls.cs | 40 ++++++ .../AnthropicTestUtils.cs | 50 ++++++++ .../AutoGen.Anthropic.Tests.csproj | 1 + 19 files changed, 715 insertions(+), 31 deletions(-) rename dotnet/sample/AutoGen.Anthropic.Samples/{AnthropicSamples.cs => Create_Anthropic_Agent.cs} (95%) create mode 100644 dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs create mode 100644 dotnet/src/AutoGen.Anthropic/Converters/JsonPropertyNameEnumCoverter.cs create mode 100644 dotnet/src/AutoGen.Anthropic/DTO/Tool.cs create mode 100644 dotnet/src/AutoGen.Anthropic/DTO/ToolChoice.cs create mode 100644 dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj b/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj index 33a5aa7f16b6..2948c9bf283c 100644 --- a/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj +++ b/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj @@ -13,6 +13,7 @@ + diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/AnthropicSamples.cs b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs similarity index 95% rename from dotnet/sample/AutoGen.Anthropic.Samples/AnthropicSamples.cs rename to dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs index 94b5f37511e6..031e50685488 100644 --- a/dotnet/sample/AutoGen.Anthropic.Samples/AnthropicSamples.cs +++ b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs @@ -7,7 +7,7 @@ namespace AutoGen.Anthropic.Samples; -public static class AnthropicSamples +public static class Create_Anthropic_Agent { public static async Task RunAsync() { diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs new file mode 100644 index 000000000000..26bd32dd12d5 --- /dev/null +++ b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Single_Anthropic_Tool.cs + +using AutoGen.Anthropic.DTO; +using AutoGen.Anthropic.Extensions; +using AutoGen.Anthropic.Utils; +using AutoGen.Core; +using FluentAssertions; + +namespace AutoGen.Anthropic.Samples; + +#region WeatherFunction + +public partial class WeatherFunction +{ + /// + /// Gets the weather based on the location and the unit + /// + /// + /// + /// + [Function] + public async Task GetWeather(string location, string unit) + { + // dummy implementation + return $"The weather in {location} is currently sunny with a tempature of {unit} (s)"; + } +} +#endregion +public class Create_Anthropic_Agent_With_Tool +{ + public static async Task RunAsync() + { + #region define_tool + var tool = new Tool + { + Name = "GetWeather", + Description = "Get the current weather in a given location", + InputSchema = new InputSchema + { + Type = "object", + Properties = new Dictionary + { + { "location", new SchemaProperty { Type = "string", Description = "The city and state, e.g. San Francisco, CA" } }, + { "unit", new SchemaProperty { Type = "string", Description = "The unit of temperature, either \"celsius\" or \"fahrenheit\"" } } + }, + Required = new List { "location" } + } + }; + + var weatherFunction = new WeatherFunction(); + var functionMiddleware = new FunctionCallMiddleware( + functions: [ + weatherFunction.GetWeatherFunctionContract, + ], + functionMap: new Dictionary>> + { + { weatherFunction.GetWeatherFunctionContract.Name!, weatherFunction.GetWeatherWrapper }, + }); + + #endregion + + #region create_anthropic_agent + + var apiKey = Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY") ?? + throw new Exception("Missing ANTHROPIC_API_KEY environment variable."); + + var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, apiKey); + var agent = new AnthropicClientAgent(anthropicClient, "assistant", AnthropicConstants.Claude3Haiku, + tools: [tool]); // Define tools for AnthropicClientAgent + #endregion + + #region register_middleware + + var agentWithConnector = agent + .RegisterMessageConnector() + .RegisterPrintMessage() + .RegisterStreamingMiddleware(functionMiddleware); + #endregion register_middleware + + #region single_turn + var question = new TextMessage(Role.Assistant, + "What is the weather like in San Francisco?", + from: "user"); + var functionCallReply = await agentWithConnector.SendAsync(question); + #endregion + + #region Single_turn_verify_reply + functionCallReply.Should().BeOfType(); + #endregion Single_turn_verify_reply + + #region Multi_turn + var finalReply = await agentWithConnector.SendAsync(chatHistory: [question, functionCallReply]); + #endregion Multi_turn + + #region Multi_turn_verify_reply + finalReply.Should().BeOfType(); + #endregion Multi_turn_verify_reply + } +} diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs b/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs index f3c615088610..6d1e4e594b99 100644 --- a/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs +++ b/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs @@ -7,6 +7,6 @@ internal static class Program { public static async Task Main(string[] args) { - await AnthropicSamples.RunAsync(); + await Create_Anthropic_Agent_With_Tool.RunAsync(); } } diff --git a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs index e395bb4a225f..173155a96a64 100644 --- a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs +++ b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -16,6 +17,8 @@ public class AnthropicClientAgent : IStreamingAgent private readonly string _systemMessage; private readonly decimal _temperature; private readonly int _maxTokens; + private readonly Tool[]? _tools; + private readonly ToolChoice? _toolChoice; public AnthropicClientAgent( AnthropicClient anthropicClient, @@ -23,7 +26,9 @@ public AnthropicClientAgent( string modelName, string systemMessage = "You are a helpful AI assistant", decimal temperature = 0.7m, - int maxTokens = 1024) + int maxTokens = 1024, + Tool[]? tools = null, + ToolChoice? toolChoice = null) { Name = name; _anthropicClient = anthropicClient; @@ -31,6 +36,8 @@ public AnthropicClientAgent( _systemMessage = systemMessage; _temperature = temperature; _maxTokens = maxTokens; + _tools = tools; + _toolChoice = toolChoice; } public async Task GenerateReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, @@ -59,6 +66,8 @@ private ChatCompletionRequest CreateParameters(IEnumerable messages, G Model = _modelName, Stream = shouldStream, Temperature = (decimal?)options?.Temperature ?? _temperature, + Tools = _tools?.ToList(), + ToolChoice = _toolChoice ?? ToolChoice.Auto }; chatCompletionRequest.Messages = BuildMessages(messages); diff --git a/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs b/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs index 90bd33683f20..babcd5302aac 100644 --- a/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs +++ b/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs @@ -24,12 +24,12 @@ public sealed class AnthropicClient : IDisposable private static readonly JsonSerializerOptions JsonSerializerOptions = new() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - Converters = { new ContentBaseConverter() } + Converters = { new ContentBaseConverter(), new JsonPropertyNameEnumConverter() } }; private static readonly JsonSerializerOptions JsonDeserializerOptions = new() { - Converters = { new ContentBaseConverter() } + Converters = { new ContentBaseConverter(), new JsonPropertyNameEnumConverter() } }; public AnthropicClient(HttpClient httpClient, string baseUrl, string apiKey) @@ -61,24 +61,64 @@ public async IAsyncEnumerable StreamingChatCompletionsAs using var reader = new StreamReader(await httpResponseMessage.Content.ReadAsStreamAsync()); var currentEvent = new SseEvent(); + while (await reader.ReadLineAsync() is { } line) { if (!string.IsNullOrEmpty(line)) { - currentEvent.Data = line.Substring("data:".Length).Trim(); + if (line.StartsWith("event:")) + { + currentEvent.EventType = line.Substring("event:".Length).Trim(); + } + else if (line.StartsWith("data:")) + { + currentEvent.Data = line.Substring("data:".Length).Trim(); + } } - else + else // an empty line indicates the end of an event { - if (currentEvent.Data == "[DONE]") - continue; + if (currentEvent.EventType == "content_block_start" && !string.IsNullOrEmpty(currentEvent.Data)) + { + var dataBlock = JsonSerializer.Deserialize(currentEvent.Data!); + if (dataBlock != null && dataBlock.ContentBlock?.Type == "tool_use") + { + currentEvent.ContentBlock = dataBlock.ContentBlock; + } + } - if (currentEvent.Data != null) + if (currentEvent.EventType is "message_start" or "content_block_delta" or "message_delta" && currentEvent.Data != null) { - yield return await JsonSerializer.DeserializeAsync( + var res = await JsonSerializer.DeserializeAsync( new MemoryStream(Encoding.UTF8.GetBytes(currentEvent.Data)), - cancellationToken: cancellationToken) ?? throw new Exception("Failed to deserialize response"); + cancellationToken: cancellationToken); + + if (res == null) + { + throw new Exception("Failed to deserialize response"); + } + + if (res.Delta?.Type == "input_json_delta" && !string.IsNullOrEmpty(res.Delta.PartialJson) && + currentEvent.ContentBlock != null) + { + currentEvent.ContentBlock.AppendDeltaParameters(res.Delta.PartialJson!); + } + else if (res.Delta is { StopReason: "tool_use" } && currentEvent.ContentBlock != null) + { + if (res.Content == null) + { + res.Content = [currentEvent.ContentBlock.CreateToolUseContent()]; + } + else + { + res.Content.Add(currentEvent.ContentBlock.CreateToolUseContent()); + } + + currentEvent = new SseEvent(); + } + + yield return res; } - else if (currentEvent.Data != null) + else if (currentEvent.EventType == "error" && currentEvent.Data != null) { var res = await JsonSerializer.DeserializeAsync( new MemoryStream(Encoding.UTF8.GetBytes(currentEvent.Data)), cancellationToken: cancellationToken); @@ -86,8 +126,10 @@ public async IAsyncEnumerable StreamingChatCompletionsAs throw new Exception(res?.Error?.Message); } - // Reset the current event for the next one - currentEvent = new SseEvent(); + if (currentEvent.ContentBlock == null) + { + currentEvent = new SseEvent(); + } } } } @@ -113,11 +155,50 @@ public void Dispose() private struct SseEvent { + public string EventType { get; set; } public string? Data { get; set; } + public ContentBlock? ContentBlock { get; set; } - public SseEvent(string? data = null) + public SseEvent(string eventType, string? data = null, ContentBlock? contentBlock = null) { + EventType = eventType; Data = data; + ContentBlock = contentBlock; } } + + private class ContentBlock + { + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("input")] + public object? Input { get; set; } + + public string? parameters { get; set; } + + public void AppendDeltaParameters(string deltaParams) + { + StringBuilder sb = new StringBuilder(parameters); + sb.Append(deltaParams); + parameters = sb.ToString(); + } + + public ToolUseContent CreateToolUseContent() + { + return new ToolUseContent { Id = Id, Name = Name, Input = parameters }; + } + } + + private class DataBlock + { + [JsonPropertyName("content_block")] + public ContentBlock? ContentBlock { get; set; } + } } diff --git a/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs b/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs index 4cb8fdbb34e0..b41a761dc4d3 100644 --- a/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs +++ b/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs @@ -24,6 +24,10 @@ public override ContentBase Read(ref Utf8JsonReader reader, Type typeToConvert, return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); case "image": return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); + case "tool_use": + return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); + case "tool_result": + return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); } } diff --git a/dotnet/src/AutoGen.Anthropic/Converters/JsonPropertyNameEnumCoverter.cs b/dotnet/src/AutoGen.Anthropic/Converters/JsonPropertyNameEnumCoverter.cs new file mode 100644 index 000000000000..cd95d837cffd --- /dev/null +++ b/dotnet/src/AutoGen.Anthropic/Converters/JsonPropertyNameEnumCoverter.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// JsonPropertyNameEnumCoverter.cs + +using System; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AutoGen.Anthropic.Converters; + +internal class JsonPropertyNameEnumConverter : JsonConverter where T : struct, Enum +{ + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + string value = reader.GetString() ?? throw new JsonException("Value was null."); + + foreach (var field in typeToConvert.GetFields()) + { + var attribute = field.GetCustomAttribute(); + if (attribute?.Name == value) + { + return (T)Enum.Parse(typeToConvert, field.Name); + } + } + + throw new JsonException($"Unable to convert \"{value}\" to enum {typeToConvert}."); + } + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + var field = value.GetType().GetField(value.ToString()); + var attribute = field.GetCustomAttribute(); + + if (attribute != null) + { + writer.WriteStringValue(attribute.Name); + } + else + { + writer.WriteStringValue(value.ToString()); + } + } +} + diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs index 0c1749eaa989..b18461e697bc 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs @@ -37,6 +37,12 @@ public class ChatCompletionRequest [JsonPropertyName("top_p")] public decimal? TopP { get; set; } + [JsonPropertyName("tools")] + public List? Tools { get; set; } + + [JsonPropertyName("tool_choice")] + public ToolChoice? ToolChoice { get; set; } + public ChatCompletionRequest() { Messages = new List(); @@ -62,4 +68,6 @@ public ChatMessage(string role, List content) Role = role; Content = content; } + + public void AddContent(ContentBase content) => Content.Add(content); } diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs index c6861f9c3150..2c6fa100fd63 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs @@ -49,9 +49,6 @@ public class StreamingMessage [JsonPropertyName("role")] public string? Role { get; set; } - [JsonPropertyName("content")] - public List? Content { get; set; } - [JsonPropertyName("model")] public string? Model { get; set; } @@ -85,6 +82,9 @@ public class Delta [JsonPropertyName("text")] public string? Text { get; set; } + [JsonPropertyName("partial_json")] + public string? PartialJson { get; set; } + [JsonPropertyName("usage")] public Usage? Usage { get; set; } } diff --git a/dotnet/src/AutoGen.Anthropic/DTO/Content.cs b/dotnet/src/AutoGen.Anthropic/DTO/Content.cs index dd2481bd58f3..ee7a745a1416 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/Content.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/Content.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Content.cs +using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace AutoGen.Anthropic.DTO; @@ -40,3 +41,30 @@ public class ImageSource [JsonPropertyName("data")] public string? Data { get; set; } } + +public class ToolUseContent : ContentBase +{ + [JsonPropertyName("type")] + public override string Type => "tool_use"; + + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("input")] + public JsonNode? Input { get; set; } +} + +public class ToolResultContent : ContentBase +{ + [JsonPropertyName("type")] + public override string Type => "tool_result"; + + [JsonPropertyName("tool_use_id")] + public string? Id { get; set; } + + [JsonPropertyName("content")] + public string? Content { get; set; } +} diff --git a/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs b/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs new file mode 100644 index 000000000000..41c20dc2a42d --- /dev/null +++ b/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Tool.cs + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace AutoGen.Anthropic.DTO; + +public class Tool +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("description")] + public string? Description { get; set; } + + [JsonPropertyName("input_schema")] + public InputSchema? InputSchema { get; set; } +} + +public class InputSchema +{ + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("properties")] + public Dictionary? Properties { get; set; } + + [JsonPropertyName("required")] + public List? Required { get; set; } +} + +public class SchemaProperty +{ + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("description")] + public string? Description { get; set; } +} diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ToolChoice.cs b/dotnet/src/AutoGen.Anthropic/DTO/ToolChoice.cs new file mode 100644 index 000000000000..0a5c3790e1de --- /dev/null +++ b/dotnet/src/AutoGen.Anthropic/DTO/ToolChoice.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ToolChoice.cs + +using System.Text.Json.Serialization; +using AutoGen.Anthropic.Converters; + +namespace AutoGen.Anthropic.DTO; + +[JsonConverter(typeof(JsonPropertyNameEnumConverter))] +public enum ToolChoiceType +{ + [JsonPropertyName("auto")] + Auto, // Default behavior + + [JsonPropertyName("any")] + Any, // Use any provided tool + + [JsonPropertyName("tool")] + Tool // Force a specific tool +} + +public class ToolChoice +{ + [JsonPropertyName("type")] + public ToolChoiceType Type { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + private ToolChoice(ToolChoiceType type, string? name = null) + { + Type = type; + Name = name; + } + + public static ToolChoice Auto => new(ToolChoiceType.Auto); + public static ToolChoice Any => new(ToolChoiceType.Any); + public static ToolChoice ToolUse(string name) => new(ToolChoiceType.Tool, name); +} diff --git a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs index bb2f5820f74c..f78ccd19deae 100644 --- a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs +++ b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Net.Http; using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using AutoGen.Anthropic.DTO; @@ -54,6 +55,17 @@ public async IAsyncEnumerable InvokeAsync(MiddlewareContext c private IStreamingMessage? ProcessChatCompletionResponse(IStreamingMessage chatMessage, IStreamingAgent agent) { + if (chatMessage.Content.Content is { Count: 1 } && + chatMessage.Content.Content[0] is ToolUseContent toolUseContent) + { + return new ToolCallMessage( + toolUseContent.Name ?? + throw new InvalidOperationException($"Expected {nameof(toolUseContent.Name)} to be specified"), + toolUseContent.Input?.ToString() ?? + throw new InvalidOperationException($"Expected {nameof(toolUseContent.Input)} to be specified"), + from: agent.Name); + } + var delta = chatMessage.Content.Delta; return delta != null && !string.IsNullOrEmpty(delta.Text) ? new TextMessageUpdate(role: Role.Assistant, delta.Text, from: agent.Name) @@ -71,16 +83,20 @@ private async Task> ProcessMessageAsync(IEnumerable ProcessTextMessage(textMessage, agent), ImageMessage imageMessage => - new MessageEnvelope(new ChatMessage("user", + (MessageEnvelope[])[new MessageEnvelope(new ChatMessage("user", new ContentBase[] { new ImageContent { Source = await ProcessImageSourceAsync(imageMessage) } } .ToList()), - from: agent.Name), + from: agent.Name)], MultiModalMessage multiModalMessage => await ProcessMultiModalMessageAsync(multiModalMessage, agent), - _ => message, + + ToolCallMessage toolCallMessage => ProcessToolCallMessage(toolCallMessage, agent), + ToolCallResultMessage toolCallResultMessage => ProcessToolCallResultMessage(toolCallResultMessage), + AggregateMessage toolCallAggregateMessage => ProcessToolCallAggregateMessage(toolCallAggregateMessage, agent), + _ => [message], }; - processedMessages.Add(processedMessage); + processedMessages.AddRange(processedMessage); } return processedMessages; @@ -93,15 +109,42 @@ private IMessage PostProcessMessage(ChatCompletionResponse response, IAgent from throw new ArgumentNullException(nameof(response.Content)); } - if (response.Content.Count != 1) + // When expecting a tool call, sometimes the response will contain two messages, one chat and one tool. + // The first message is typically a TextContent, of the LLM explaining what it is trying to do. + // The second message contains the tool call. + if (response.Content.Count > 1) { - throw new NotSupportedException($"{nameof(response.Content)} != 1"); + if (response.Content.Count == 2 && response.Content[0] is TextContent && + response.Content[1] is ToolUseContent toolUseContent) + { + return new ToolCallMessage(toolUseContent.Name ?? string.Empty, + toolUseContent.Input?.ToJsonString() ?? string.Empty, + from: from.Name); + } + + throw new NotSupportedException($"Expected {nameof(response.Content)} to have one output"); } - return new TextMessage(Role.Assistant, ((TextContent)response.Content[0]).Text ?? string.Empty, from: from.Name); + var content = response.Content[0]; + switch (content) + { + case TextContent textContent: + return new TextMessage(Role.Assistant, textContent.Text ?? string.Empty, from: from.Name); + + case ToolUseContent toolUseContent: + return new ToolCallMessage(toolUseContent.Name ?? string.Empty, + toolUseContent.Input?.ToJsonString() ?? string.Empty, + from: from.Name); + + case ImageContent: + throw new InvalidOperationException( + "Claude is an image understanding model only. It can interpret and analyze images, but it cannot generate, produce, edit, manipulate or create images"); + default: + throw new ArgumentOutOfRangeException(nameof(content)); + } } - private IMessage ProcessTextMessage(TextMessage textMessage, IAgent agent) + private IEnumerable> ProcessTextMessage(TextMessage textMessage, IAgent agent) { ChatMessage messages; @@ -139,10 +182,10 @@ private IMessage ProcessTextMessage(TextMessage textMessage, IAgent "user", textMessage.Content); } - return new MessageEnvelope(messages, from: textMessage.From); + return [new MessageEnvelope(messages, from: textMessage.From)]; } - private async Task ProcessMultiModalMessageAsync(MultiModalMessage multiModalMessage, IAgent agent) + private async Task> ProcessMultiModalMessageAsync(MultiModalMessage multiModalMessage, IAgent agent) { var content = new List(); foreach (var message in multiModalMessage.Content) @@ -158,8 +201,7 @@ private async Task ProcessMultiModalMessageAsync(MultiModalMessage mul } } - var chatMessage = new ChatMessage("user", content); - return MessageEnvelope.Create(chatMessage, agent.Name); + return [MessageEnvelope.Create(new ChatMessage("user", content), agent.Name)]; } private async Task ProcessImageSourceAsync(ImageMessage imageMessage) @@ -192,4 +234,52 @@ private async Task ProcessImageSourceAsync(ImageMessage imageMessag Data = Convert.ToBase64String(await response.Content.ReadAsByteArrayAsync()) }; } + + private IEnumerable ProcessToolCallMessage(ToolCallMessage toolCallMessage, IAgent agent) + { + var chatMessage = new ChatMessage("assistant", new List()); + foreach (var toolCall in toolCallMessage.ToolCalls) + { + chatMessage.AddContent(new ToolUseContent + { + Id = toolCall.ToolCallId, + Name = toolCall.FunctionName, + Input = JsonNode.Parse(toolCall.FunctionArguments) + }); + } + + return [MessageEnvelope.Create(chatMessage, toolCallMessage.From)]; + } + + private IEnumerable ProcessToolCallResultMessage(ToolCallResultMessage toolCallResultMessage) + { + var chatMessage = new ChatMessage("user", new List()); + foreach (var toolCall in toolCallResultMessage.ToolCalls) + { + chatMessage.AddContent(new ToolResultContent + { + Id = toolCall.ToolCallId ?? string.Empty, + Content = toolCall.Result, + }); + } + + return [MessageEnvelope.Create(chatMessage, toolCallResultMessage.From)]; + } + + private IEnumerable ProcessToolCallAggregateMessage(AggregateMessage aggregateMessage, IAgent agent) + { + if (aggregateMessage.From is { } from && from != agent.Name) + { + var contents = aggregateMessage.Message2.ToolCalls.Select(t => t.Result); + var messages = contents.Select(c => + new ChatMessage("assistant", c ?? throw new ArgumentNullException(nameof(c)))); + + return messages.Select(m => new MessageEnvelope(m, from: from)); + } + + var toolCallMessage = ProcessToolCallMessage(aggregateMessage.Message1, agent); + var toolCallResult = ProcessToolCallResultMessage(aggregateMessage.Message2); + + return toolCallMessage.Concat(toolCallResult); + } } diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs index d29025b44aff..49cbb54af318 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs @@ -105,4 +105,101 @@ public async Task AnthropicAgentTestImageMessageAsync() reply.GetContent().Should().NotBeNullOrEmpty(); reply.From.Should().Be(agent.Name); } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicAgentTestToolAsync() + { + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + + var function = new TypeSafeFunctionCall(); + var functionCallMiddleware = new FunctionCallMiddleware( + functions: new[] { function.WeatherReportFunctionContract }, + functionMap: new Dictionary>> + { + { function.WeatherReportFunctionContract.Name ?? string.Empty, function.WeatherReportWrapper }, + }); + + var agent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are an LLM that is specialized in finding the weather !", + tools: [AnthropicTestUtils.WeatherTool] + ) + .RegisterMessageConnector() + .RegisterStreamingMiddleware(functionCallMiddleware); + + var reply = await agent.SendAsync("What is the weather in Philadelphia?"); + reply.GetContent().Should().Be("Weather report for Philadelphia on today is sunny"); + } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicAgentFunctionCallMessageTest() + { + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + var agent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are a helpful AI assistant.", + tools: [AnthropicTestUtils.WeatherTool] + ) + .RegisterMessageConnector(); + + var weatherFunctionArgumets = """ + { + "city": "Philadelphia", + "date": "6/14/2024" + } + """; + + var function = new AnthropicTestFunctionCalls(); + var functionCallResult = await function.GetWeatherReportWrapper(weatherFunctionArgumets); + var toolCall = new ToolCall(function.WeatherReportFunctionContract.Name!, weatherFunctionArgumets) + { + ToolCallId = "get_weather", + Result = functionCallResult, + }; + + IMessage[] chatHistory = [ + new TextMessage(Role.User, "what's the weather in Philadelphia?"), + new ToolCallMessage([toolCall], from: "assistant"), + new ToolCallResultMessage([toolCall], from: "user" ), + ]; + + var reply = await agent.SendAsync(chatHistory: chatHistory); + + reply.Should().BeOfType(); + reply.GetContent().Should().Be("The weather report for Philadelphia on 6/14/2024 is sunny."); + } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicAgentFunctionCallMiddlewareMessageTest() + { + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + var function = new AnthropicTestFunctionCalls(); + var functionCallMiddleware = new FunctionCallMiddleware( + functions: [function.WeatherReportFunctionContract], + functionMap: new Dictionary>> + { + { function.WeatherReportFunctionContract.Name!, function.GetWeatherReportWrapper } + }); + + var functionCallAgent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are a helpful AI assistant.", + tools: [AnthropicTestUtils.WeatherTool] + ) + .RegisterMessageConnector() + .RegisterStreamingMiddleware(functionCallMiddleware); + + var question = new TextMessage(Role.User, "what's the weather in Philadelphia?"); + var reply = await functionCallAgent.SendAsync(question); + + var finalReply = await functionCallAgent.SendAsync(chatHistory: [question, reply]); + finalReply.Should().BeOfType(); + finalReply.GetContent()!.ToLower().Should().Contain("sunny"); + } } diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs index a0b1f60cfb95..66b7d007758a 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs @@ -1,5 +1,6 @@ using System.Text; using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; using AutoGen.Anthropic.DTO; using AutoGen.Anthropic.Utils; @@ -108,6 +109,57 @@ public async Task AnthropicClientImageChatCompletionTestAsync() response.Usage.OutputTokens.Should().BeGreaterThan(0); } + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicClientTestToolsAsync() + { + var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + + var request = new ChatCompletionRequest(); + request.Model = AnthropicConstants.Claude3Haiku; + request.Stream = false; + request.MaxTokens = 100; + request.Messages = new List() { new("user", "Use the stock price tool to look for MSFT. Your response should only be the tool.") }; + request.Tools = new List() { AnthropicTestUtils.StockTool }; + + ChatCompletionResponse response = + await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); + + Assert.NotNull(response.Content); + Assert.True(response.Content.First() is ToolUseContent); + ToolUseContent toolUseContent = ((ToolUseContent)response.Content.First()); + Assert.Equal("get_stock_price", toolUseContent.Name); + Assert.NotNull(toolUseContent.Input); + Assert.True(toolUseContent.Input is JsonNode); + JsonNode jsonNode = toolUseContent.Input; + Assert.Equal("{\"ticker\":\"MSFT\"}", jsonNode.ToJsonString()); + } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicClientTestToolChoiceAsync() + { + var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + + var request = new ChatCompletionRequest(); + request.Model = AnthropicConstants.Claude3Haiku; + request.Stream = false; + request.MaxTokens = 100; + request.Messages = new List() { new("user", "What is the weather today? Your response should only be the tool.") }; + request.Tools = new List() { AnthropicTestUtils.StockTool, AnthropicTestUtils.WeatherTool }; + + // Force to use get_stock_price even though the prompt is about weather + request.ToolChoice = ToolChoice.ToolUse("get_stock_price"); + + ChatCompletionResponse response = + await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); + + Assert.NotNull(response.Content); + Assert.True(response.Content.First() is ToolUseContent); + ToolUseContent toolUseContent = ((ToolUseContent)response.Content.First()); + Assert.Equal("get_stock_price", toolUseContent.Name); + Assert.NotNull(toolUseContent.Input); + Assert.True(toolUseContent.Input is JsonNode); + } + private sealed class Person { [JsonPropertyName("name")] diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs new file mode 100644 index 000000000000..5f1c0971bf71 --- /dev/null +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// AnthropicTestFunctions.cs + +using System.Text.Json; +using System.Text.Json.Serialization; +using AutoGen.Core; + +namespace AutoGen.Anthropic.Tests; + +public partial class AnthropicTestFunctionCalls +{ + private class GetWeatherSchema + { + [JsonPropertyName("city")] + public string? City { get; set; } + + [JsonPropertyName("date")] + public string? Date { get; set; } + } + + /// + /// Get weather report + /// + /// city + /// date + [Function] + public async Task WeatherReport(string city, string date) + { + return $"Weather report for {city} on {date} is sunny"; + } + + public Task GetWeatherReportWrapper(string arguments) + { + var schema = JsonSerializer.Deserialize( + arguments, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + + return WeatherReport(schema?.City ?? string.Empty, schema?.Date ?? string.Empty); + } +} diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs index de630da6d87c..a1faffec5344 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // AnthropicTestUtils.cs +using AutoGen.Anthropic.DTO; + namespace AutoGen.Anthropic.Tests; public static class AnthropicTestUtils @@ -13,4 +15,52 @@ public static async Task Base64FromImageAsync(string imageName) return Convert.ToBase64String( await File.ReadAllBytesAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "images", imageName))); } + + public static Tool WeatherTool + { + get + { + return new Tool + { + Name = "WeatherReport", + Description = "Get the current weather", + InputSchema = new InputSchema + { + Type = "object", + Properties = new Dictionary + { + { "city", new SchemaProperty {Type = "string", Description = "The name of the city"} }, + { "date", new SchemaProperty {Type = "string", Description = "date of the day"} } + } + } + }; + } + } + + public static Tool StockTool + { + get + { + return new Tool + { + Name = "get_stock_price", + Description = "Get the current stock price for a given ticker symbol.", + InputSchema = new InputSchema + { + Type = "object", + Properties = new Dictionary + { + { + "ticker", new SchemaProperty + { + Type = "string", + Description = "The stock ticker symbol, e.g. AAPL for Apple Inc." + } + } + }, + Required = new List { "ticker" } + } + }; + } + } } diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj b/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj index 0f22d9fe6764..ac479ed2e722 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj +++ b/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj @@ -12,6 +12,7 @@ +