diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs index bbe684bb7bd..4bc1ec54982 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs @@ -7,6 +7,7 @@ using System.Diagnostics.Metrics; using System.Linq; using System.Runtime.CompilerServices; +using System.Text; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -311,6 +312,99 @@ internal static string SerializeChatMessages( m.Parts.Add(new OtelGenericPart { Type = "error", Content = ec.Message }); break; + // Server tool call content types as specified in the OpenTelemetry semantic conventions: + + case CodeInterpreterToolCallContent citcc: + m.Parts.Add(new OtelServerToolCallPart + { + Id = citcc.CallId, + Name = "code_interpreter", + ServerToolCall = new OtelCodeInterpreterToolCall + { + Code = ExtractCodeFromInputs(citcc.Inputs), + }, + }); + break; + + case CodeInterpreterToolResultContent citrc: + m.Parts.Add(new OtelServerToolCallResponsePart + { + Id = citrc.CallId, + ServerToolCallResponse = new OtelCodeInterpreterToolCallResponse + { + Output = citrc.Outputs, + }, + }); + break; + + case ImageGenerationToolCallContent igtcc: + m.Parts.Add(new OtelServerToolCallPart + { + Id = igtcc.ImageId, + Name = "image_generation", + ServerToolCall = new OtelImageGenerationToolCall(), + }); + break; + + case ImageGenerationToolResultContent igtrc: + m.Parts.Add(new OtelServerToolCallResponsePart + { + Id = igtrc.ImageId, + ServerToolCallResponse = new OtelImageGenerationToolCallResponse + { + Output = igtrc.Outputs, + }, + }); + break; + + case McpServerToolCallContent mstcc: + m.Parts.Add(new OtelServerToolCallPart + { + Id = mstcc.CallId, + Name = mstcc.ToolName, + ServerToolCall = new OtelMcpToolCall + { + Arguments = mstcc.Arguments, + ServerName = mstcc.ServerName, + }, + }); + break; + + case McpServerToolResultContent mstrc: + m.Parts.Add(new OtelServerToolCallResponsePart + { + Id = mstrc.CallId, + ServerToolCallResponse = new OtelMcpToolCallResponse + { + Output = mstrc.Output, + }, + }); + break; + + case McpServerToolApprovalRequestContent mstarc: + m.Parts.Add(new OtelServerToolCallPart + { + Id = mstarc.Id, + Name = mstarc.ToolCall.ToolName, + ServerToolCall = new OtelMcpApprovalRequest + { + Arguments = mstarc.ToolCall.Arguments, + ServerName = mstarc.ToolCall.ServerName, + }, + }); + break; + + case McpServerToolApprovalResponseContent mstaresp: + m.Parts.Add(new OtelServerToolCallResponsePart + { + Id = mstaresp.Id, + ServerToolCallResponse = new OtelMcpApprovalResponse + { + Approved = mstaresp.Approved, + }, + }); + break; + default: JsonElement element = _emptyObject; try @@ -364,6 +458,35 @@ internal static string SerializeChatMessages( return null; } + /// Extracts code text from code interpreter inputs. + /// + /// Code interpreter inputs typically contain a DataContent with a "text/x-python" or similar + /// media type representing the code to execute. + /// + private static string? ExtractCodeFromInputs(IList? inputs) + { + if (inputs is not null) + { + foreach (var input in inputs) + { + // Check for DataContent with text MIME types + if (input is DataContent dc && dc.HasTopLevelMediaType("text")) + { + // Return the data as a string (decode bytes as UTF8) + return Encoding.UTF8.GetString(dc.Data.ToArray()); + } + + // Check for TextContent + if (input is TextContent tc && !string.IsNullOrEmpty(tc.Text)) + { + return tc.Text; + } + } + } + + return null; + } + /// Creates an activity for a chat request, or returns if not enabled. private Activity? CreateAndConfigureActivity(ChatOptions? options) { @@ -691,6 +814,72 @@ private sealed class OtelToolCallResponsePart public object? Response { get; set; } } + private sealed class OtelServerToolCallPart + where T : class + { + public string Type { get; set; } = "server_tool_call"; + public string? Id { get; set; } + public string? Name { get; set; } + public T? ServerToolCall { get; set; } + } + + private sealed class OtelServerToolCallResponsePart + where T : class + { + public string Type { get; set; } = "server_tool_call_response"; + public string? Id { get; set; } + public T? ServerToolCallResponse { get; set; } + } + + private sealed class OtelCodeInterpreterToolCall + { + public string Type { get; set; } = "code_interpreter"; + public string? Code { get; set; } + } + + private sealed class OtelCodeInterpreterToolCallResponse + { + public string Type { get; set; } = "code_interpreter"; + public object? Output { get; set; } + } + + private sealed class OtelImageGenerationToolCall + { + public string Type { get; set; } = "image_generation"; + } + + private sealed class OtelImageGenerationToolCallResponse + { + public string Type { get; set; } = "image_generation"; + public object? Output { get; set; } + } + + private sealed class OtelMcpToolCall + { + public string Type { get; set; } = "mcp"; + public string? ServerName { get; set; } + public IReadOnlyDictionary? Arguments { get; set; } + } + + private sealed class OtelMcpToolCallResponse + { + public string Type { get; set; } = "mcp"; + public object? Output { get; set; } + } + + private sealed class OtelMcpApprovalRequest + { + public string Type { get; set; } = "mcp_approval_request"; + public string? ServerName { get; set; } + public IReadOnlyDictionary? Arguments { get; set; } + } + + private sealed class OtelMcpApprovalResponse + { + public string Type { get; set; } = "mcp_approval_response"; + public bool Approved { get; set; } + } + private sealed class OtelFunction { public string Type { get; set; } = "function"; @@ -727,6 +916,14 @@ private static JsonSerializerOptions CreateDefaultOptions() [JsonSerializable(typeof(OtelFilePart))] [JsonSerializable(typeof(OtelToolCallRequestPart))] [JsonSerializable(typeof(OtelToolCallResponsePart))] + [JsonSerializable(typeof(OtelServerToolCallPart))] + [JsonSerializable(typeof(OtelServerToolCallResponsePart))] + [JsonSerializable(typeof(OtelServerToolCallPart))] + [JsonSerializable(typeof(OtelServerToolCallResponsePart))] + [JsonSerializable(typeof(OtelServerToolCallPart))] + [JsonSerializable(typeof(OtelServerToolCallResponsePart))] + [JsonSerializable(typeof(OtelServerToolCallPart))] + [JsonSerializable(typeof(OtelServerToolCallResponsePart))] [JsonSerializable(typeof(IEnumerable))] private sealed partial class OtelContext : JsonSerializerContext; } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs index f67c62f0ed4..3f1c9f59bce 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs @@ -590,6 +590,251 @@ public async Task UnknownContentTypes_Ignored() """), ReplaceWhitespace(inputMessages)); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ServerToolCallContentTypes_SerializedCorrectly(bool streaming) + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = async (messages, options, cancellationToken) => + { + await Task.Yield(); + return new ChatResponse(new ChatMessage(ChatRole.Assistant, + [ + new TextContent("Processing with tools..."), + new CodeInterpreterToolCallContent { CallId = "ci-call-1", Inputs = [new TextContent("print('hello')")] }, + new CodeInterpreterToolResultContent { CallId = "ci-call-1", Outputs = [new TextContent("hello")] }, + new ImageGenerationToolCallContent { ImageId = "img-123" }, + new ImageGenerationToolResultContent { ImageId = "img-123", Outputs = [new UriContent(new Uri("https://example.com/image.png"), "image/png")] }, + new McpServerToolCallContent("mcp-call-1", "myTool", "myServer") { Arguments = new Dictionary { ["param1"] = "value1" } }, + new McpServerToolResultContent("mcp-call-1") { Output = [new TextContent("Tool result")] }, + ])); + }, + GetStreamingResponseAsyncCallback = CallbackAsync, + }; + + async static IAsyncEnumerable CallbackAsync( + IEnumerable messages, ChatOptions? options, [EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.Yield(); + yield return new(ChatRole.Assistant, "Processing with tools..."); + yield return new() { Contents = [new CodeInterpreterToolCallContent { CallId = "ci-call-1", Inputs = [new TextContent("print('hello')")] }] }; + yield return new() { Contents = [new CodeInterpreterToolResultContent { CallId = "ci-call-1", Outputs = [new TextContent("hello")] }] }; + yield return new() { Contents = [new ImageGenerationToolCallContent { ImageId = "img-123" }] }; + yield return new() { Contents = [new ImageGenerationToolResultContent { ImageId = "img-123", Outputs = [new UriContent(new Uri("https://example.com/image.png"), "image/png")] }] }; + yield return new() { Contents = [new McpServerToolCallContent("mcp-call-1", "myTool", "myServer") { Arguments = new Dictionary { ["param1"] = "value1" } }] }; + yield return new() { Contents = [new McpServerToolResultContent("mcp-call-1") { Output = [new TextContent("Tool result")] }] }; + } + + using var chatClient = innerClient + .AsBuilder() + .UseOpenTelemetry(null, sourceName, configure: instance => + { + instance.EnableSensitiveData = true; + instance.JsonSerializerOptions = TestJsonSerializerContext.Default.Options; + }) + .Build(); + + List messages = + [ + new(ChatRole.User, "Execute code and generate an image"), + ]; + + if (streaming) + { + await foreach (var update in chatClient.GetStreamingResponseAsync(messages)) + { + await Task.Yield(); + } + } + else + { + await chatClient.GetResponseAsync(messages); + } + + var activity = Assert.Single(activities); + Assert.NotNull(activity); + + var outputMessages = activity.Tags.First(kvp => kvp.Key == "gen_ai.output.messages").Value; + Assert.Equal(ReplaceWhitespace(""" + [ + { + "role": "assistant", + "parts": [ + { + "type": "text", + "content": "Processing with tools..." + }, + { + "type": "server_tool_call", + "id": "ci-call-1", + "name": "code_interpreter", + "server_tool_call": { + "type": "code_interpreter", + "code": "print('hello')" + } + }, + { + "type": "server_tool_call_response", + "id": "ci-call-1", + "server_tool_call_response": { + "type": "code_interpreter", + "output": [ + { + "$type": "text", + "text": "hello" + } + ] + } + }, + { + "type": "server_tool_call", + "id": "img-123", + "name": "image_generation", + "server_tool_call": { + "type": "image_generation" + } + }, + { + "type": "server_tool_call_response", + "id": "img-123", + "server_tool_call_response": { + "type": "image_generation", + "output": [ + { + "$type": "uri", + "uri": "https://example.com/image.png", + "media_type": "image/png" + } + ] + } + }, + { + "type": "server_tool_call", + "id": "mcp-call-1", + "name": "myTool", + "server_tool_call": { + "type": "mcp", + "server_name": "myServer", + "arguments": { + "param1": "value1" + } + } + }, + { + "type": "server_tool_call_response", + "id": "mcp-call-1", + "server_tool_call_response": { + "type": "mcp", + "output": [ + { + "$type": "text", + "text": "Tool result" + } + ] + } + } + ] + } + ] + """), ReplaceWhitespace(outputMessages)); + } + + [Fact] + public async Task McpServerToolApprovalContentTypes_SerializedCorrectly() + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = async (messages, options, cancellationToken) => + { + await Task.Yield(); + return new ChatResponse(new ChatMessage(ChatRole.Assistant, "Done")); + }, + }; + + using var chatClient = innerClient + .AsBuilder() + .UseOpenTelemetry(null, sourceName, configure: instance => + { + instance.EnableSensitiveData = true; + instance.JsonSerializerOptions = TestJsonSerializerContext.Default.Options; + }) + .Build(); + + var toolCall = new McpServerToolCallContent("mcp-call-2", "dangerousTool", "secureServer") + { + Arguments = new Dictionary { ["action"] = "delete" } + }; + + List messages = + [ + new(ChatRole.Assistant, + [ + new McpServerToolApprovalRequestContent("approval-1", toolCall), + ]), + new(ChatRole.User, + [ + new McpServerToolApprovalResponseContent("approval-1", true), + ]), + ]; + + await chatClient.GetResponseAsync(messages); + + var activity = Assert.Single(activities); + Assert.NotNull(activity); + + var inputMessages = activity.Tags.First(kvp => kvp.Key == "gen_ai.input.messages").Value; + Assert.Equal(ReplaceWhitespace(""" + [ + { + "role": "assistant", + "parts": [ + { + "type": "server_tool_call", + "id": "approval-1", + "name": "dangerousTool", + "server_tool_call": { + "type": "mcp_approval_request", + "server_name": "secureServer", + "arguments": { + "action": "delete" + } + } + } + ] + }, + { + "role": "user", + "parts": [ + { + "type": "server_tool_call_response", + "id": "approval-1", + "server_tool_call_response": { + "type": "mcp_approval_response", + "approved": true + } + } + ] + } + ] + """), ReplaceWhitespace(inputMessages)); + } + private sealed class NonSerializableAIContent : AIContent; private static string ReplaceWhitespace(string? input) => Regex.Replace(input ?? "", @"\s+", " ").Trim();