Add CodeInterpreterToolCall/ResultContent content types

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs

@@ -5,7 +5,11 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using System.Linq;
+#if !NET
+using System.Runtime.InteropServices;
+#endif
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -181,7 +185,7 @@ static async Task<ChatResponse> ToChatResponseAsync(
     }
 
     /// <summary>Coalesces sequential <see cref="AIContent"/> content elements.</summary>
-    internal static void CoalesceTextContent(IList<AIContent> contents)
+    internal static void CoalesceContent(IList<AIContent> contents)
     {
         Coalesce<TextContent>(
             contents,
@@ -215,6 +219,110 @@ internal static void CoalesceTextContent(IList<AIContent> contents)
                 return content;
             });
 
+        Coalesce<DataContent>(
+            contents,
+            mergeSingle: false,
+            canMerge: static (r1, r2) => r1.MediaType == r2.MediaType && r1.HasTopLevelMediaType("text") && r1.Name == r2.Name,
+            static (contents, start, end) =>
+            {
+                Debug.Assert(end - start > 1, "Expected multiple contents to merge");
+
+                MemoryStream ms = new();
+                for (int i = start; i < end; i++)
+                {
+                    var current = (DataContent)contents[i];
+#if NET
+                    ms.Write(current.Data.Span);
+#else
+                    if (!MemoryMarshal.TryGetArray(current.Data, out var segment))
+                    {
+                        segment = new(current.Data.ToArray());
+                    }
+
+                    ms.Write(segment.Array!, segment.Offset, segment.Count);
+#endif
+                }
+
+                var first = (DataContent)contents[start];
+                return new DataContent(new ReadOnlyMemory<byte>(ms.GetBuffer(), 0, (int)ms.Length), first.MediaType) { Name = first.Name };
+            });
+
+        Coalesce<CodeInterpreterToolCallContent>(
+            contents,
+            mergeSingle: true,
+            canMerge: static (r1, r2) => r1.CallId == r2.CallId,
+            static (contents, start, end) =>
+            {
+                var firstContent = (CodeInterpreterToolCallContent)contents[start];
+
+                if (start == end - 1)
+                {
+                    if (firstContent.Inputs is not null)
+                    {
+                        CoalesceContent(firstContent.Inputs);
+                    }
+
+                    return firstContent;
+                }
+
+                List<AIContent>? inputs = null;
+
+                for (int i = start; i < end; i++)
+                {
+                    (inputs ??= []).AddRange(((CodeInterpreterToolCallContent)contents[i]).Inputs ?? []);
+                }
+
+                if (inputs is not null)
+                {
+                    CoalesceContent(inputs);
+                }
+
+                return new()
+                {
+                    CallId = firstContent.CallId,
+                    Inputs = inputs,
+                    AdditionalProperties = firstContent.AdditionalProperties?.Clone(),
+                };
+            });
+
+        Coalesce<CodeInterpreterToolResultContent>(
+            contents,
+            mergeSingle: true,
+            canMerge: static (r1, r2) => r1.CallId is not null && r2.CallId is not null && r1.CallId == r2.CallId,
+            static (contents, start, end) =>
+            {
+                var firstContent = (CodeInterpreterToolResultContent)contents[start];
+
+                if (start == end - 1)
+                {
+                    if (firstContent.Outputs is not null)
+                    {
+                        CoalesceContent(firstContent.Outputs);
+                    }
+
+                    return firstContent;
+                }
+
+                List<AIContent>? output = null;
+
+                for (int i = start; i < end; i++)
+                {
+                    (output ??= []).AddRange(((CodeInterpreterToolResultContent)contents[i]).Outputs ?? []);
+                }
+
+                if (output is not null)
+                {
+                    CoalesceContent(output);
+                }
+
+                return new()
+                {
+                    CallId = firstContent.CallId,
+                    Outputs = output,
+                    AdditionalProperties = firstContent.AdditionalProperties?.Clone(),
+                };
+            });
+
         static string MergeText(IList<AIContent> contents, int start, int end)
         {
             Debug.Assert(end - start > 1, "Expected multiple contents to merge");
@@ -318,7 +426,7 @@ private static void FinalizeResponse(ChatResponse response)
         int count = response.Messages.Count;
         for (int i = 0; i < count; i++)
         {
-            CoalesceTextContent((List<AIContent>)response.Messages[i].Contents);
+            CoalesceContent((List<AIContent>)response.Messages[i].Contents);
         }
     }
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs

@@ -30,6 +30,8 @@ namespace Microsoft.Extensions.AI;
 // [JsonDerivedType(typeof(McpServerToolResultContent), typeDiscriminator: "mcpServerToolResult")]
 // [JsonDerivedType(typeof(McpServerToolApprovalRequestContent), typeDiscriminator: "mcpServerToolApprovalRequest")]
 // [JsonDerivedType(typeof(McpServerToolApprovalResponseContent), typeDiscriminator: "mcpServerToolApprovalResponse")]
+// [JsonDerivedType(typeof(CodeInterpreterToolCallContent), typeDiscriminator: "codeInterpreterToolCall")]
+// [JsonDerivedType(typeof(CodeInterpreterToolResultContent), typeDiscriminator: "codeInterpreterToolResult")]
 
 public class AIContent
 {

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolCallContent.cs

@@ -0,0 +1,41 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents a code interpreter tool call invocation by a hosted service.
+/// </summary>
+/// <remarks>
+/// This content type represents when a hosted AI service invokes a code interpreter tool.
+/// It is informational only and represents the call itself, not the result.
+/// </remarks>
+[Experimental("MEAI001")]
+public sealed class CodeInterpreterToolCallContent : AIContent
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="CodeInterpreterToolCallContent"/> class.
+    /// </summary>
+    public CodeInterpreterToolCallContent()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the tool call ID.
+    /// </summary>
+    public string? CallId { get; set; }
+
+    /// <summary>
+    /// Gets or sets the inputs to the code interpreter tool.
+    /// </summary>
+    /// <remarks>
+    /// Inputs can include various types of content such as <see cref="HostedFileContent"/> for files,
+    /// <see cref="DataContent"/> for binary data, or other <see cref="AIContent"/> types as supported
+    /// by the service. Typically <see cref="Inputs"/> includes a <see cref="DataContent"/> with a "text/x-python"
+    /// media type representing the code for execution by the code interpreter tool.
+    /// </remarks>
+    public IList<AIContent>? Inputs { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CodeInterpreterToolResultContent.cs

@@ -0,0 +1,36 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents the result of a code interpreter tool invocation by a hosted service.
+/// </summary>
+[Experimental("MEAI001")]
+public sealed class CodeInterpreterToolResultContent : AIContent
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="CodeInterpreterToolResultContent"/> class.
+    /// </summary>
+    public CodeInterpreterToolResultContent()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets the tool call ID that this result corresponds to.
+    /// </summary>
+    public string? CallId { get; set; }
+
+    /// <summary>
+    /// Gets or sets the output of code interpreter tool.
+    /// </summary>
+    /// <remarks>
+    /// Outputs can include various types of content such as <see cref="HostedFileContent"/> for files,
+    /// <see cref="DataContent"/> for binary data, <see cref="TextContent"/> for standard output text,
+    /// or other <see cref="AIContent"/> types as supported by the service.
+    /// </remarks>
+    public IList<AIContent>? Outputs { get; set; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs

@@ -9,6 +9,7 @@
 #endif
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.Text;
 #if !NET
 using System.Runtime.InteropServices;
 #endif
@@ -115,6 +116,7 @@ public DataContent([StringSyntax(StringSyntaxAttribute.Uri)] string uri, string?
     /// <param name="mediaType">The media type (also known as MIME type) represented by the content.</param>
     /// <exception cref="ArgumentNullException"><paramref name="mediaType"/> is <see langword="null"/>.</exception>
     /// <exception cref="ArgumentException"><paramref name="mediaType"/> is empty or composed entirely of whitespace.</exception>
+    /// <exception cref="ArgumentException"><paramref name="mediaType"/> represents an invalid media type.</exception>
     public DataContent(ReadOnlyMemory<byte> data, string mediaType)
     {
         MediaType = DataUriParser.ThrowIfInvalidMediaType(mediaType);
@@ -236,6 +238,16 @@ private string DebuggerDisplay
     {
         get
         {
+            if (HasTopLevelMediaType("text"))
+            {
+                return $"MediaType = {MediaType}, Text = \"{Encoding.UTF8.GetString(Data.ToArray())}\"";
+            }
+
+            if ("application/json".Equals(MediaType, StringComparison.OrdinalIgnoreCase))
+            {
+                return $"JSON = {Encoding.UTF8.GetString(Data.ToArray())}";
+            }
+
             const int MaxLength = 80;
 
             string uri = Uri;

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/HostedFileContent.cs

@@ -14,11 +14,9 @@ namespace Microsoft.Extensions.AI;
 /// Unlike <see cref="DataContent"/> which contains the data for a file or blob, this class represents a file that is hosted
 /// by the AI service and referenced by an identifier. Such identifiers are specific to the provider.
 /// </remarks>
-[DebuggerDisplay("FileId = {FileId}")]
+[DebuggerDisplay("{DebuggerDisplay,nq}")]
 public sealed class HostedFileContent : AIContent
 {
-    private string _fileId;
-
     /// <summary>
     /// Initializes a new instance of the <see cref="HostedFileContent"/> class.
     /// </summary>
@@ -27,7 +25,7 @@ public sealed class HostedFileContent : AIContent
     /// <exception cref="ArgumentException"><paramref name="fileId"/> is empty or composed entirely of whitespace.</exception>
     public HostedFileContent(string fileId)
     {
-        _fileId = Throw.IfNullOrWhitespace(fileId);
+        FileId = fileId;
     }
 
     /// <summary>
@@ -37,7 +35,40 @@ public HostedFileContent(string fileId)
     /// <exception cref="ArgumentException"><paramref name="value"/> is empty or composed entirely of whitespace.</exception>
     public string FileId
     {
-        get => _fileId;
-        set => _fileId = Throw.IfNullOrWhitespace(value);
+        get => field;
+        set => field = Throw.IfNullOrWhitespace(value);
+    }
+
+    /// <summary>Gets or sets an optional media type (also known as MIME type) associated with the file.</summary>
+    /// <exception cref="ArgumentException"><paramref name="value"/> represents an invalid media type.</exception>
+    public string? MediaType
+    {
+        get;
+        set => field = value is not null ? DataUriParser.ThrowIfInvalidMediaType(value) : value;
+    }
+
+    /// <summary>Gets or sets an optional name associated with the file.</summary>
+    public string? Name { get; set; }
+
+    /// <summary>Gets a string representing this instance to display in the debugger.</summary>
+    [DebuggerBrowsable(DebuggerBrowsableState.Never)]
+    private string DebuggerDisplay
+    {
+        get
+        {
+            string display = $"FileId = {FileId}";
+
+            if (MediaType is string mediaType)
+            {
+                display += $", MediaType = {mediaType}";
+            }
+
+            if (Name is string name)
+            {
+                display += $", Name = \"{name}\"";
+            }
+
+            return display;
+        }
     }
 }

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/UriContent.cs

@@ -67,6 +67,7 @@ public Uri Uri
     }
 
     /// <summary>Gets or sets the media type (also known as MIME type) for this content.</summary>
+    /// <exception cref="ArgumentException"><paramref name="value"/> represents an invalid media type.</exception>
     public string MediaType
     {
         get => _mediaType;

src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json

@@ -2012,6 +2012,14 @@
         {
           "Member": "string Microsoft.Extensions.AI.HostedFileContent.FileId { get; set; }",
           "Stage": "Stable"
+        },
+        {
+          "Member": "string? Microsoft.Extensions.AI.HostedFileContent.MediaType { get; set; }",
+          "Stage": "Stable"
+        },
+        {
+          "Member": "string? Microsoft.Extensions.AI.HostedFileContent.Name { get; set; }",
+          "Stage": "Stable"
         }
       ]
     },

src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextResponseUpdateExtensions.cs

@@ -30,7 +30,7 @@ public static SpeechToTextResponse ToSpeechToTextResponse(
             ProcessUpdate(update, response);
         }
 
-        ChatResponseExtensions.CoalesceTextContent((List<AIContent>)response.Contents);
+        ChatResponseExtensions.CoalesceContent((List<AIContent>)response.Contents);
 
         return response;
     }
@@ -56,7 +56,7 @@ static async Task<SpeechToTextResponse> ToResponseAsync(
                 ProcessUpdate(update, response);
             }
 
-            ChatResponseExtensions.CoalesceTextContent((List<AIContent>)response.Contents);
+            ChatResponseExtensions.CoalesceContent((List<AIContent>)response.Contents);
 
             return response;
         }

src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs

@@ -57,6 +57,8 @@ private static JsonSerializerOptions CreateDefaultOptions()
         AddAIContentType(options, typeof(McpServerToolResultContent), typeDiscriminatorId: "mcpServerToolResult", checkBuiltIn: false);
         AddAIContentType(options, typeof(McpServerToolApprovalRequestContent), typeDiscriminatorId: "mcpServerToolApprovalRequest", checkBuiltIn: false);
         AddAIContentType(options, typeof(McpServerToolApprovalResponseContent), typeDiscriminatorId: "mcpServerToolApprovalResponse", checkBuiltIn: false);
+        AddAIContentType(options, typeof(CodeInterpreterToolCallContent), typeDiscriminatorId: "codeInterpreterToolCall", checkBuiltIn: false);
+        AddAIContentType(options, typeof(CodeInterpreterToolResultContent), typeDiscriminatorId: "codeInterpreterToolResult", checkBuiltIn: false);
 
         if (JsonSerializer.IsReflectionEnabledByDefault)
         {
@@ -129,6 +131,8 @@ private static JsonSerializerOptions CreateDefaultOptions()
     [JsonSerializable(typeof(McpServerToolResultContent))]
     [JsonSerializable(typeof(McpServerToolApprovalRequestContent))]
     [JsonSerializable(typeof(McpServerToolApprovalResponseContent))]
+    [JsonSerializable(typeof(CodeInterpreterToolCallContent))]
+    [JsonSerializable(typeof(CodeInterpreterToolResultContent))]
     [JsonSerializable(typeof(ResponseContinuationToken))]
 
     // IEmbeddingGenerator