Fix handling of tool calls with some OpenAI endpoints (#6405)

* Fix handling of tool calls with some endpoints

Most assistant messages containing tool calls don't contain text as well (though some can). In such a case, we were still creating the assistant with empty text. While OpenAI's service permits that, some other endpoints are more finicky about it. This avoids doing so.

* Reduce to single iteration through assistant content

Author: stephentoub

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs

@@ -157,30 +157,54 @@ void IDisposable.Dispose()
             }
             else if (input.Role == ChatRole.Assistant)
             {
-                AssistantChatMessage message = new(ToOpenAIChatContent(input.Contents))
-                {
-                    ParticipantName = input.AuthorName
-                };
-
+                List<ChatMessageContentPart>? contentParts = null;
+                List<ChatToolCall>? toolCalls = null;
+                string? refusal = null;
                 foreach (var content in input.Contents)
                 {
                     switch (content)
                     {
-                        case ErrorContent errorContent when errorContent.ErrorCode is nameof(message.Refusal):
-                            message.Refusal = errorContent.Message;
+                        case ErrorContent ec when ec.ErrorCode == nameof(AssistantChatMessage.Refusal):
+                            refusal = ec.Message;
                             break;
 
-                        case FunctionCallContent callRequest:
-                            message.ToolCalls.Add(
-                                ChatToolCall.CreateFunctionToolCall(
-                                    callRequest.CallId,
-                                    callRequest.Name,
-                                    new(JsonSerializer.SerializeToUtf8Bytes(
-                                        callRequest.Arguments,
-                                        options.GetTypeInfo(typeof(IDictionary<string, object?>))))));
+                        case FunctionCallContent fc:
+                            (toolCalls ??= []).Add(
+                                ChatToolCall.CreateFunctionToolCall(fc.CallId, fc.Name, new(JsonSerializer.SerializeToUtf8Bytes(
+                                    fc.Arguments, options.GetTypeInfo(typeof(IDictionary<string, object?>))))));
                             break;
+
+                        default:
+                            if (ToChatMessageContentPart(content) is { } part)
+                            {
+                                (contentParts ??= []).Add(part);
+                            }
+
+                            break;
+                    }
+                }
+
+                AssistantChatMessage message;
+                if (contentParts is not null)
+                {
+                    message = new(contentParts);
+                    if (toolCalls is not null)
+                    {
+                        foreach (var toolCall in toolCalls)
+                        {
+                            message.ToolCalls.Add(toolCall);
+                        }
                     }
                 }
+                else
+                {
+                    message = toolCalls is not null ?
+                        new(toolCalls) :
+                        new(ChatMessageContentPart.CreateTextPart(string.Empty));
+                }
+
+                message.ParticipantName = input.AuthorName;
+                message.Refusal = refusal;
 
                 yield return message;
             }
@@ -191,38 +215,12 @@ void IDisposable.Dispose()
     private static List<ChatMessageContentPart> ToOpenAIChatContent(IList<AIContent> contents)
     {
         List<ChatMessageContentPart> parts = [];
+
         foreach (var content in contents)
         {
-            switch (content)
+            if (ToChatMessageContentPart(content) is { } part)
             {
-                case TextContent textContent:
-                    parts.Add(ChatMessageContentPart.CreateTextPart(textContent.Text));
-                    break;
-
-                case UriContent uriContent when uriContent.HasTopLevelMediaType("image"):
-                    parts.Add(ChatMessageContentPart.CreateImagePart(uriContent.Uri, GetImageDetail(content)));
-                    break;
-
-                case DataContent dataContent when dataContent.HasTopLevelMediaType("image"):
-                    parts.Add(ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, GetImageDetail(content)));
-                    break;
-
-                case DataContent dataContent when dataContent.HasTopLevelMediaType("audio"):
-                    var audioData = BinaryData.FromBytes(dataContent.Data);
-                    if (dataContent.MediaType.Equals("audio/mpeg", StringComparison.OrdinalIgnoreCase))
-                    {
-                        parts.Add(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Mp3));
-                    }
-                    else if (dataContent.MediaType.Equals("audio/wav", StringComparison.OrdinalIgnoreCase))
-                    {
-                        parts.Add(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav));
-                    }
-
-                    break;
-
-                case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase):
-                    parts.Add(ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf"));
-                    break;
+                parts.Add(part);
             }
         }
 
@@ -234,6 +232,39 @@ private static List<ChatMessageContentPart> ToOpenAIChatContent(IList<AIContent>
         return parts;
     }
 
+    private static ChatMessageContentPart? ToChatMessageContentPart(AIContent content)
+    {
+        switch (content)
+        {
+            case TextContent textContent:
+                return ChatMessageContentPart.CreateTextPart(textContent.Text);
+
+            case UriContent uriContent when uriContent.HasTopLevelMediaType("image"):
+                return ChatMessageContentPart.CreateImagePart(uriContent.Uri, GetImageDetail(content));
+
+            case DataContent dataContent when dataContent.HasTopLevelMediaType("image"):
+                return ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, GetImageDetail(content));
+
+            case DataContent dataContent when dataContent.HasTopLevelMediaType("audio"):
+                var audioData = BinaryData.FromBytes(dataContent.Data);
+                if (dataContent.MediaType.Equals("audio/mpeg", StringComparison.OrdinalIgnoreCase))
+                {
+                    return ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Mp3);
+                }
+                else if (dataContent.MediaType.Equals("audio/wav", StringComparison.OrdinalIgnoreCase))
+                {
+                    return ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav);
+                }
+
+                break;
+
+            case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase):
+                return ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf");
+        }
+
+        return null;
+    }
+
     private static ChatImageDetailLevel? GetImageDetail(AIContent content)
     {
         if (content.AdditionalProperties?.TryGetValue("detail", out object? value) is true)