Skip to content

Add PDF support to OpenAI AsIChatClient #6344

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 29, 2025
Merged

Add PDF support to OpenAI AsIChatClient #6344

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4499aed
Add PDF support to OpenAI AsIChatClient
stephentoub Apr 29, 2025
3735808
Add missing reference
stephentoub Apr 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions eng/packages/TestOnly.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
<PackageVersion Include="Moq.AutoMock" Version="3.1.0" />
<PackageVersion Include="Moq" Version="4.18.4" />
<PackageVersion Include="OpenTelemetry.Exporter.InMemory" Version="1.9.0" />
<PackageVersion Include="PdfPig" Version="0.1.10-alpha-20250203-fdb88" />
<PackageVersion Include="Polly.Testing" Version="8.4.2" />
<PackageVersion Include="StrongNamer" Version="0.2.5" />
<PackageVersion Include="System.Configuration.ConfigurationManager" Version="$(SystemConfigurationConfigurationManagerVersion)" />
Expand Down
4 changes: 4 additions & 0 deletions src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,10 @@ private static List<ChatMessageContentPart> ToOpenAIChatContent(IList<AIContent>
}

break;

case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase):
parts.Add(ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf"));
break;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would there be a drawback to just mapping all other DataContent objects, not limiting it to PDF? Are there cases where someone would add a DataContent but not want it to be sent in the request?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just that, at least today, anything else will result in the request failing.

}
}

Expand Down
5 changes: 5 additions & 0 deletions src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,11 @@ private static List<ResponseContentPart> ToOpenAIResponsesContent(IList<AIConten
case DataContent dataContent when dataContent.HasTopLevelMediaType("image"):
parts.Add(ResponseContentPart.CreateInputImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType));
break;

case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase):
parts.Add(ResponseContentPart.CreateInputFilePart(null, $"{Guid.NewGuid():N}.pdf",
BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(dataContent.Uri, ResponseClientJsonContext.Default.String))));
break;
}
}

Expand Down
1 change: 1 addition & 0 deletions ....Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
<PackageReference Include="Azure.Identity" />
<PackageReference Include="Microsoft.Extensions.Hosting" />
<PackageReference Include="OpenAI" />
<PackageReference Include="PdfPig" />
</ItemGroup>

<ItemGroup>
Expand Down
18 changes: 18 additions & 0 deletions test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,24 @@ public virtual async Task MultiModal_DescribeImage()
Assert.True(response.Text.IndexOf("net", StringComparison.OrdinalIgnoreCase) >= 0, response.Text);
}

[ConditionalFact]
public virtual async Task MultiModal_DescribePdf()
{
SkipIfNotEnabled();

var response = await _chatClient.GetResponseAsync(
[
new(ChatRole.User,
[
new TextContent("What text does this document contain?"),
new DataContent(ImageDataUri.GetPdfDataUri(), "application/pdf"),
])
],
new() { ModelId = GetModel_MultiModal_DescribeImage() });

Assert.True(response.Text.IndexOf("hello", StringComparison.OrdinalIgnoreCase) >= 0, response.Text);
}

[ConditionalFact]
public virtual async Task FunctionInvocation_AutomaticallyInvokeFunction_Parameterless()
{
Expand Down
1 change: 1 addition & 0 deletions ...icrosoft.Extensions.AI.Integration.Tests/Microsoft.Extensions.AI.Integration.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
<PackageReference Include="Microsoft.ML.Tokenizers" />
<PackageReference Include="Microsoft.ML.Tokenizers.Data.O200kBase" />
<PackageReference Include="OpenTelemetry.Exporter.InMemory" />
<PackageReference Include="PdfPig" />
<PackageReference Include="System.Numerics.Tensors" />
</ItemGroup>

Expand Down
1 change: 1 addition & 0 deletions ...ibraries/Microsoft.Extensions.AI.OpenAI.Tests/Microsoft.Extensions.AI.OpenAI.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,6 @@
<ItemGroup>
<PackageReference Include="Azure.AI.OpenAI" />
<PackageReference Include="Azure.Identity" />
<PackageReference Include="PdfPig" />
</ItemGroup>
</Project>
13 changes: 13 additions & 0 deletions test/Shared/ImageDataUri/ImageDataUri.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@

using System;
using System.IO;
using UglyToad.PdfPig.Content;
using UglyToad.PdfPig.Core;
using UglyToad.PdfPig.Fonts.Standard14Fonts;
using UglyToad.PdfPig.Writer;
using Xunit;

namespace Microsoft.Extensions.AI;
Expand All @@ -17,4 +21,13 @@ internal static Uri GetImageDataUri()
s.CopyTo(ms);
return new Uri($"data:image/png;base64,{Convert.ToBase64String(ms.ToArray())}");
}

internal static Uri GetPdfDataUri()
{
using PdfDocumentBuilder builder = new PdfDocumentBuilder();
PdfPageBuilder page = builder.AddPage(PageSize.A4);
PdfDocumentBuilder.AddedFont font = builder.AddStandard14Font(Standard14Font.Helvetica);
page.AddText("Hello World!", 12, new PdfPoint(25, 700), font);
return new Uri($"data:application/pdf;base64,{Convert.ToBase64String(builder.Build())}");
}
}
3 changes: 2 additions & 1 deletion test/Shared/Shared.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
<PackageReference Include="JsonSchema.Net" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
<PackageReference Include="PdfPig" />
</ItemGroup>
</Project>
Loading