diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/AgentProvider.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/AgentProvider.cs index daec465020..96795cc6a4 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/AgentProvider.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/AgentProvider.cs @@ -16,6 +16,7 @@ public static class Names public const string Marketing = "MARKETING"; public const string MathChat = "MATHCHAT"; public const string InputArguments = "INPUTARGUMENTS"; + public const string Vision = "VISION"; } public static class Settings @@ -33,6 +34,7 @@ public static AgentProvider Create(IConfiguration configuration, string provider Names.Marketing => new MarketingAgentProvider(configuration), Names.MathChat => new MathChatAgentProvider(configuration), Names.InputArguments => new PoemAgentProvider(configuration), + Names.Vision => new VisionAgentProvider(configuration), _ => new TestAgentProvider(configuration), }; diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/TestAgentProvider.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/TestAgentProvider.cs index 078b6321c0..6cff2c15a0 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/TestAgentProvider.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/TestAgentProvider.cs @@ -20,7 +20,7 @@ protected override async IAsyncEnumerable CreateAgentsAsync(Uri fo await aiProjectClient.CreateAgentAsync( agentName: "TestAgent", agentDefinition: this.DefineMenuAgent(), - agentDescription: "Provides information about the restaurant menu"); + agentDescription: "Basic agent"); } private PromptAgentDefinition DefineMenuAgent() => diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/VisionAgentProvider.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/VisionAgentProvider.cs new file mode 100644 index 0000000000..d9557bd8f9 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/VisionAgentProvider.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Azure.AI.Projects; +using Azure.AI.Projects.OpenAI; +using Azure.Identity; +using Microsoft.Extensions.Configuration; +using Shared.Foundry; + +namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents; + +internal sealed class VisionAgentProvider(IConfiguration configuration) : AgentProvider(configuration) +{ + protected override async IAsyncEnumerable CreateAgentsAsync(Uri foundryEndpoint) + { + AIProjectClient aiProjectClient = new(foundryEndpoint, new AzureCliCredential()); + + yield return + await aiProjectClient.CreateAgentAsync( + agentName: "VisionAgent", + agentDefinition: this.DefineVisionAgent(), + agentDescription: "Use computer vision to describe an image or document."); + } + + private PromptAgentDefinition DefineVisionAgent() => + new(this.GetSetting(Settings.FoundryModelFull)) + { + Instructions = + """ + Describe the image or document contained in the user request, if any; + otherwise, suggest that the user provide an image or document. + """, + }; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/MediaInputTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/MediaInputTest.cs index 33122e2fa8..32adb93ddb 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/MediaInputTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/MediaInputTest.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Azure.AI.Projects; using Azure.Identity; +using Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents; using Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework; using Microsoft.Extensions.AI; using OpenAI.Files; @@ -19,37 +20,42 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests; public sealed class MediaInputTest(ITestOutputHelper output) : IntegrationTest(output) { private const string WorkflowFileName = "MediaInput.yaml"; - private const string ImageReference = "https://sample-files.com/downloads/documents/pdf/basic-text.pdf"; + private const string PdfReference = "https://sample-files.com/downloads/documents/pdf/basic-text.pdf"; + private const string ImageReference = "https://sample-files.com/downloads/images/jpg/web_optimized_1200x800_97kb.jpg"; - [Fact] - public async Task ValidateImageUrlAsync() + [Theory] + [InlineData(ImageReference, "image/jpeg")] + [InlineData(PdfReference, "application/pdf", Skip = "Not currently supported by agent service api")] + public async Task ValidateFileUrlAsync(string fileSource, string mediaType) { - this.Output.WriteLine($"Image: {ImageReference}"); - await this.ValidateImageAsync(new UriContent(ImageReference, "image/jpeg")); + this.Output.WriteLine($"File: {ImageReference}"); + await this.ValidateFileAsync(new UriContent(fileSource, mediaType)); } - [Fact] - public async Task ValidateImageDataAsync() + [Theory] + [InlineData(ImageReference, "image/jpeg")] + [InlineData(PdfReference, "application/pdf")] + public async Task ValidateFileDataAsync(string fileSource, string mediaType) { - byte[] imageData = await DownloadFileAsync(); - string encodedData = Convert.ToBase64String(imageData); - string imageUrl = $"data:image/png;base64,{encodedData}"; - this.Output.WriteLine($"Image: {imageUrl.Substring(0, 112)}..."); - await this.ValidateImageAsync(new DataContent(imageUrl)); + byte[] fileData = await DownloadFileAsync(fileSource); + string encodedData = Convert.ToBase64String(fileData); + string fileUrl = $"data:{mediaType};base64,{encodedData}"; + this.Output.WriteLine($"Content: {fileUrl.Substring(0, 112)}..."); + await this.ValidateFileAsync(new DataContent(fileUrl)); } - [Fact(Skip = "Not behaving will in git-hub build pipeline")] - public async Task ValidateImageUploadAsync() + [Fact(Skip = "Not currently supported by agent service api")] + public async Task ValidateFileUploadAsync() { - byte[] imageData = await DownloadFileAsync(); + byte[] fileData = await DownloadFileAsync(PdfReference); AIProjectClient client = new(this.TestEndpoint, new AzureCliCredential()); - using MemoryStream contentStream = new(imageData); + using MemoryStream contentStream = new(fileData); OpenAIFileClient fileClient = client.GetProjectOpenAIClient().GetOpenAIFileClient(); OpenAIFile fileInfo = await fileClient.UploadFileAsync(contentStream, "basic-text.pdf", FileUploadPurpose.Assistants); try { - this.Output.WriteLine($"Image: {fileInfo.Id}"); - await this.ValidateImageAsync(new HostedFileContent(fileInfo.Id)); + this.Output.WriteLine($"File: {fileInfo.Id}"); + await this.ValidateFileAsync(new HostedFileContent(fileInfo.Id)); } finally { @@ -57,25 +63,29 @@ public async Task ValidateImageUploadAsync() } } - private static async Task DownloadFileAsync() + private static async Task DownloadFileAsync(string uri) { using HttpClient client = new(); client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/110.0"); - return await client.GetByteArrayAsync(new Uri(ImageReference)); + return await client.GetByteArrayAsync(new Uri(uri)); } - private async Task ValidateImageAsync(AIContent imageContent) + private async Task ValidateFileAsync(AIContent fileContent) { - ChatMessage inputMessage = new(ChatRole.User, [new TextContent("Here is my image:"), imageContent]); + AgentProvider agentProvider = AgentProvider.Create(this.Configuration, AgentProvider.Names.Vision); + await agentProvider.CreateAgentsAsync().ConfigureAwait(false); + + ChatMessage inputMessage = new(ChatRole.User, [new TextContent("I've provided a file:"), fileContent]); DeclarativeWorkflowOptions options = await this.CreateOptionsAsync(); Workflow workflow = DeclarativeWorkflowBuilder.Build(Path.Combine(Environment.CurrentDirectory, "Workflows", WorkflowFileName), options); WorkflowHarness harness = new(workflow, runId: Path.GetFileNameWithoutExtension(WorkflowFileName)); WorkflowEvents workflowEvents = await harness.RunWorkflowAsync(inputMessage).ConfigureAwait(false); - Assert.Single(workflowEvents.ConversationEvents); - this.Output.WriteLine("CONVERSATION: " + workflowEvents.ConversationEvents[0].ConversationId); - Assert.Single(workflowEvents.AgentResponseEvents); - this.Output.WriteLine("RESPONSE: " + workflowEvents.AgentResponseEvents[0].Response.Text); + ConversationUpdateEvent conversationEvent = Assert.Single(workflowEvents.ConversationEvents); + this.Output.WriteLine("CONVERSATION: " + conversationEvent.ConversationId); + AgentRunResponseEvent agentResponseEvent = Assert.Single(workflowEvents.AgentResponseEvents); + this.Output.WriteLine("RESPONSE: " + agentResponseEvent.Response.Text); + Assert.NotEmpty(agentResponseEvent.Response.Text); } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/MediaInput.yaml b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/MediaInput.yaml index fce766988d..8d1f451a94 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/MediaInput.yaml +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/MediaInput.yaml @@ -9,8 +9,4 @@ trigger: id: invoke_vision conversationId: =System.ConversationId agent: - name: TestAgent - input: - additionalInstructions: |- - Describe the image contained in the user request, if any; - otherwise, suggest that the user provide an image. + name: VisionAgent