From 4499aedc77bb140bbdc96419eba2f9b82dd50657 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Mon, 28 Apr 2025 22:36:01 -0400 Subject: [PATCH 1/2] Add PDF support to OpenAI AsIChatClient --- eng/packages/TestOnly.props | 1 + .../OpenAIChatClient.cs | 4 ++++ .../OpenAIResponseChatClient.cs | 5 +++++ ...ions.AI.Evaluation.Integration.Tests.csproj | 1 + .../ChatClientIntegrationTests.cs | 18 ++++++++++++++++++ ...soft.Extensions.AI.Integration.Tests.csproj | 1 + ...Microsoft.Extensions.AI.OpenAI.Tests.csproj | 1 + test/Shared/ImageDataUri/ImageDataUri.cs | 13 +++++++++++++ 8 files changed, 44 insertions(+) diff --git a/eng/packages/TestOnly.props b/eng/packages/TestOnly.props index 8bc89ac60c3..be1cb37a609 100644 --- a/eng/packages/TestOnly.props +++ b/eng/packages/TestOnly.props @@ -18,6 +18,7 @@ + diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index d74c5516776..c78b495393b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -209,6 +209,10 @@ private static List ToOpenAIChatContent(IList } break; + + case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase): + parts.Add(ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf")); + break; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs index a922909a415..92224379c10 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs @@ -567,6 +567,11 @@ private static List ToOpenAIResponsesContent(IList + diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs index e20f52aa568..22531e14e22 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs @@ -183,6 +183,24 @@ public virtual async Task MultiModal_DescribeImage() Assert.True(response.Text.IndexOf("net", StringComparison.OrdinalIgnoreCase) >= 0, response.Text); } + [ConditionalFact] + public virtual async Task MultiModal_DescribePdf() + { + SkipIfNotEnabled(); + + var response = await _chatClient.GetResponseAsync( + [ + new(ChatRole.User, + [ + new TextContent("What text does this document contain?"), + new DataContent(ImageDataUri.GetPdfDataUri(), "application/pdf"), + ]) + ], + new() { ModelId = GetModel_MultiModal_DescribeImage() }); + + Assert.True(response.Text.IndexOf("hello", StringComparison.OrdinalIgnoreCase) >= 0, response.Text); + } + [ConditionalFact] public virtual async Task FunctionInvocation_AutomaticallyInvokeFunction_Parameterless() { diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj index 6653838d9ee..ddc72caa90d 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj @@ -43,6 +43,7 @@ + diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj index bd56d0ddfd6..60b6f8c84ab 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj @@ -33,5 +33,6 @@ + diff --git a/test/Shared/ImageDataUri/ImageDataUri.cs b/test/Shared/ImageDataUri/ImageDataUri.cs index 106e936b1ed..b2c91809305 100644 --- a/test/Shared/ImageDataUri/ImageDataUri.cs +++ b/test/Shared/ImageDataUri/ImageDataUri.cs @@ -3,6 +3,10 @@ using System; using System.IO; +using UglyToad.PdfPig.Content; +using UglyToad.PdfPig.Core; +using UglyToad.PdfPig.Fonts.Standard14Fonts; +using UglyToad.PdfPig.Writer; using Xunit; namespace Microsoft.Extensions.AI; @@ -17,4 +21,13 @@ internal static Uri GetImageDataUri() s.CopyTo(ms); return new Uri($"data:image/png;base64,{Convert.ToBase64String(ms.ToArray())}"); } + + internal static Uri GetPdfDataUri() + { + using PdfDocumentBuilder builder = new PdfDocumentBuilder(); + PdfPageBuilder page = builder.AddPage(PageSize.A4); + PdfDocumentBuilder.AddedFont font = builder.AddStandard14Font(Standard14Font.Helvetica); + page.AddText("Hello World!", 12, new PdfPoint(25, 700), font); + return new Uri($"data:application/pdf;base64,{Convert.ToBase64String(builder.Build())}"); + } } From 373580806c000dea655dec136a781b1e20c6c990 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Tue, 29 Apr 2025 11:05:04 -0400 Subject: [PATCH 2/2] Add missing reference --- test/Shared/Shared.Tests.csproj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Shared/Shared.Tests.csproj b/test/Shared/Shared.Tests.csproj index 456e50f67a9..b7e27306f2a 100644 --- a/test/Shared/Shared.Tests.csproj +++ b/test/Shared/Shared.Tests.csproj @@ -21,7 +21,8 @@ - + +