Add PDF support to OpenAI AsIChatClient

Author: stephentoub

eng/packages/TestOnly.props

@@ -18,6 +18,7 @@
     <PackageVersion Include="Moq.AutoMock" Version="3.1.0" />
     <PackageVersion Include="Moq" Version="4.18.4" />
     <PackageVersion Include="OpenTelemetry.Exporter.InMemory" Version="1.9.0" />
+    <PackageVersion Include="PdfPig" Version="0.1.10-alpha-20250203-fdb88" />
     <PackageVersion Include="Polly.Testing" Version="8.4.2" />
     <PackageVersion Include="StrongNamer" Version="0.2.5" />
     <PackageVersion Include="System.Configuration.ConfigurationManager" Version="$(SystemConfigurationConfigurationManagerVersion)" />

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs

@@ -209,6 +209,10 @@ private static List<ChatMessageContentPart> ToOpenAIChatContent(IList<AIContent>
                     }
 
                     break;
+
+                case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase):
+                    parts.Add(ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf"));
+                    break;
             }
         }
 

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs

@@ -567,6 +567,11 @@ private static List<ResponseContentPart> ToOpenAIResponsesContent(IList<AIConten
                 case DataContent dataContent when dataContent.HasTopLevelMediaType("image"):
                     parts.Add(ResponseContentPart.CreateInputImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType));
                     break;
+
+                case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase):
+                    parts.Add(ResponseContentPart.CreateInputFilePart(null, $"{Guid.NewGuid():N}.pdf",
+                        BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(dataContent.Uri, ResponseClientJsonContext.Default.String))));
+                    break;
             }
         }
 

test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj

@@ -24,6 +24,7 @@
     <PackageReference Include="Azure.Identity" />
     <PackageReference Include="Microsoft.Extensions.Hosting" />
     <PackageReference Include="OpenAI" />
+    <PackageReference Include="PdfPig" />
   </ItemGroup>
 
   <ItemGroup>

test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs

@@ -183,6 +183,24 @@ public virtual async Task MultiModal_DescribeImage()
         Assert.True(response.Text.IndexOf("net", StringComparison.OrdinalIgnoreCase) >= 0, response.Text);
     }
 
+    [ConditionalFact]
+    public virtual async Task MultiModal_DescribePdf()
+    {
+        SkipIfNotEnabled();
+
+        var response = await _chatClient.GetResponseAsync(
+            [
+                new(ChatRole.User,
+                [
+                    new TextContent("What text does this document contain?"),
+                    new DataContent(ImageDataUri.GetPdfDataUri(), "application/pdf"),
+                ])
+            ],
+            new() { ModelId = GetModel_MultiModal_DescribeImage() });
+
+        Assert.True(response.Text.IndexOf("hello", StringComparison.OrdinalIgnoreCase) >= 0, response.Text);
+    }
+
     [ConditionalFact]
     public virtual async Task FunctionInvocation_AutomaticallyInvokeFunction_Parameterless()
     {

test/Libraries/Microsoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj

@@ -43,6 +43,7 @@
     <PackageReference Include="Microsoft.ML.Tokenizers" />
     <PackageReference Include="Microsoft.ML.Tokenizers.Data.O200kBase" />
     <PackageReference Include="OpenTelemetry.Exporter.InMemory" />
+    <PackageReference Include="PdfPig" />
     <PackageReference Include="System.Numerics.Tensors" />
   </ItemGroup>
 

test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj

@@ -33,5 +33,6 @@
   <ItemGroup>
     <PackageReference Include="Azure.AI.OpenAI" />
     <PackageReference Include="Azure.Identity" />
+    <PackageReference Include="PdfPig" />
   </ItemGroup>
 </Project>

test/Shared/ImageDataUri/ImageDataUri.cs

@@ -3,6 +3,10 @@
 
 using System;
 using System.IO;
+using UglyToad.PdfPig.Content;
+using UglyToad.PdfPig.Core;
+using UglyToad.PdfPig.Fonts.Standard14Fonts;
+using UglyToad.PdfPig.Writer;
 using Xunit;
 
 namespace Microsoft.Extensions.AI;
@@ -17,4 +21,13 @@ internal static Uri GetImageDataUri()
         s.CopyTo(ms);
         return new Uri($"data:image/png;base64,{Convert.ToBase64String(ms.ToArray())}");
     }
+
+    internal static Uri GetPdfDataUri()
+    {
+        using PdfDocumentBuilder builder = new PdfDocumentBuilder();
+        PdfPageBuilder page = builder.AddPage(PageSize.A4);
+        PdfDocumentBuilder.AddedFont font = builder.AddStandard14Font(Standard14Font.Helvetica);
+        page.AddText("Hello World!", 12, new PdfPoint(25, 700), font);
+        return new Uri($"data:application/pdf;base64,{Convert.ToBase64String(builder.Build())}");
+    }
 }