.NET Workflows - Update Declarative Integration Tests for File Cases

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/AgentProvider.cs

@@ -16,6 +16,7 @@ public static class Names
         public const string Marketing = "MARKETING";
         public const string MathChat = "MATHCHAT";
         public const string InputArguments = "INPUTARGUMENTS";
+        public const string Vision = "VISION";
     }
 
     public static class Settings
@@ -33,6 +34,7 @@ public static AgentProvider Create(IConfiguration configuration, string provider
             Names.Marketing => new MarketingAgentProvider(configuration),
             Names.MathChat => new MathChatAgentProvider(configuration),
             Names.InputArguments => new PoemAgentProvider(configuration),
+            Names.Vision => new VisionAgentProvider(configuration),
             _ => new TestAgentProvider(configuration),
         };
 

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/TestAgentProvider.cs

@@ -20,7 +20,7 @@ protected override async IAsyncEnumerable<AgentVersion> CreateAgentsAsync(Uri fo
             await aiProjectClient.CreateAgentAsync(
                 agentName: "TestAgent",
                 agentDefinition: this.DefineMenuAgent(),
-                agentDescription: "Provides information about the restaurant menu");
+                agentDescription: "Basic agent");
     }
 
     private PromptAgentDefinition DefineMenuAgent() =>

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Agents/VisionAgentProvider.cs

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using Azure.AI.Projects;
+using Azure.AI.Projects.OpenAI;
+using Azure.Identity;
+using Microsoft.Extensions.Configuration;
+using Shared.Foundry;
+
+namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents;
+
+internal sealed class VisionAgentProvider(IConfiguration configuration) : AgentProvider(configuration)
+{
+    protected override async IAsyncEnumerable<AgentVersion> CreateAgentsAsync(Uri foundryEndpoint)
+    {
+        AIProjectClient aiProjectClient = new(foundryEndpoint, new AzureCliCredential());
+
+        yield return
+            await aiProjectClient.CreateAgentAsync(
+                agentName: "VisionAgent",
+                agentDefinition: this.DefineVisionAgent(),
+                agentDescription: "Use computer vision to describe an image or document.");
+    }
+
+    private PromptAgentDefinition DefineVisionAgent() =>
+        new(this.GetSetting(Settings.FoundryModelFull))
+        {
+            Instructions =
+                """
+                Describe the image or document contained in the user request, if any;
+                otherwise, suggest that the user provide an image or document.
+                """,
+        };
+}

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/MediaInputTest.cs

@@ -6,6 +6,7 @@
 using System.Threading.Tasks;
 using Azure.AI.Projects;
 using Azure.Identity;
+using Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Agents;
 using Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests.Framework;
 using Microsoft.Extensions.AI;
 using OpenAI.Files;
@@ -19,63 +20,72 @@ namespace Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests;
 public sealed class MediaInputTest(ITestOutputHelper output) : IntegrationTest(output)
 {
     private const string WorkflowFileName = "MediaInput.yaml";
-    private const string ImageReference = "https://sample-files.com/downloads/documents/pdf/basic-text.pdf";
+    private const string PdfReference = "https://sample-files.com/downloads/documents/pdf/basic-text.pdf";
+    private const string ImageReference = "https://sample-files.com/downloads/images/jpg/web_optimized_1200x800_97kb.jpg";
 
-    [Fact]
-    public async Task ValidateImageUrlAsync()
+    [Theory]
+    [InlineData(ImageReference, "image/jpeg")]
+    [InlineData(PdfReference, "application/pdf", Skip = "Not currently supported by agent service api")]
+    public async Task ValidateFileUrlAsync(string fileSource, string mediaType)
     {
-        this.Output.WriteLine($"Image: {ImageReference}");
-        await this.ValidateImageAsync(new UriContent(ImageReference, "image/jpeg"));
+        this.Output.WriteLine($"File: {ImageReference}");
+        await this.ValidateFileAsync(new UriContent(fileSource, mediaType));
     }
 
-    [Fact]
-    public async Task ValidateImageDataAsync()
+    [Theory]
+    [InlineData(ImageReference, "image/jpeg")]
+    [InlineData(PdfReference, "application/pdf")]
+    public async Task ValidateFileDataAsync(string fileSource, string mediaType)
     {
-        byte[] imageData = await DownloadFileAsync();
-        string encodedData = Convert.ToBase64String(imageData);
-        string imageUrl = $"data:image/png;base64,{encodedData}";
-        this.Output.WriteLine($"Image: {imageUrl.Substring(0, 112)}...");
-        await this.ValidateImageAsync(new DataContent(imageUrl));
+        byte[] fileData = await DownloadFileAsync(fileSource);
+        string encodedData = Convert.ToBase64String(fileData);
+        string fileUrl = $"data:{mediaType};base64,{encodedData}";
+        this.Output.WriteLine($"Content: {fileUrl.Substring(0, 112)}...");
+        await this.ValidateFileAsync(new DataContent(fileUrl));
     }
 
-    [Fact(Skip = "Not behaving will in git-hub build pipeline")]
-    public async Task ValidateImageUploadAsync()
+    [Fact(Skip = "Not currently supported by agent service api")]
+    public async Task ValidateFileUploadAsync()
     {
-        byte[] imageData = await DownloadFileAsync();
+        byte[] fileData = await DownloadFileAsync(PdfReference);
         AIProjectClient client = new(this.TestEndpoint, new AzureCliCredential());
-        using MemoryStream contentStream = new(imageData);
+        using MemoryStream contentStream = new(fileData);
         OpenAIFileClient fileClient = client.GetProjectOpenAIClient().GetOpenAIFileClient();
         OpenAIFile fileInfo = await fileClient.UploadFileAsync(contentStream, "basic-text.pdf", FileUploadPurpose.Assistants);
         try
         {
-            this.Output.WriteLine($"Image: {fileInfo.Id}");
-            await this.ValidateImageAsync(new HostedFileContent(fileInfo.Id));
+            this.Output.WriteLine($"File: {fileInfo.Id}");
+            await this.ValidateFileAsync(new HostedFileContent(fileInfo.Id));
         }
         finally
         {
             await fileClient.DeleteFileAsync(fileInfo.Id);
         }
     }
 
-    private static async Task<byte[]> DownloadFileAsync()
+    private static async Task<byte[]> DownloadFileAsync(string uri)
     {
         using HttpClient client = new();
         client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/110.0");
-        return await client.GetByteArrayAsync(new Uri(ImageReference));
+        return await client.GetByteArrayAsync(new Uri(uri));
     }
 
-    private async Task ValidateImageAsync(AIContent imageContent)
+    private async Task ValidateFileAsync(AIContent fileContent)
     {
-        ChatMessage inputMessage = new(ChatRole.User, [new TextContent("Here is my image:"), imageContent]);
+        AgentProvider agentProvider = AgentProvider.Create(this.Configuration, AgentProvider.Names.Vision);
+        await agentProvider.CreateAgentsAsync().ConfigureAwait(false);
+
+        ChatMessage inputMessage = new(ChatRole.User, [new TextContent("I've provided a file:"), fileContent]);
 
         DeclarativeWorkflowOptions options = await this.CreateOptionsAsync();
         Workflow workflow = DeclarativeWorkflowBuilder.Build<ChatMessage>(Path.Combine(Environment.CurrentDirectory, "Workflows", WorkflowFileName), options);
 
         WorkflowHarness harness = new(workflow, runId: Path.GetFileNameWithoutExtension(WorkflowFileName));
         WorkflowEvents workflowEvents = await harness.RunWorkflowAsync(inputMessage).ConfigureAwait(false);
-        Assert.Single(workflowEvents.ConversationEvents);
-        this.Output.WriteLine("CONVERSATION: " + workflowEvents.ConversationEvents[0].ConversationId);
-        Assert.Single(workflowEvents.AgentResponseEvents);
-        this.Output.WriteLine("RESPONSE: " + workflowEvents.AgentResponseEvents[0].Response.Text);
+        ConversationUpdateEvent conversationEvent = Assert.Single(workflowEvents.ConversationEvents);
+        this.Output.WriteLine("CONVERSATION: " + conversationEvent.ConversationId);
+        AgentRunResponseEvent agentResponseEvent = Assert.Single(workflowEvents.AgentResponseEvents);
+        this.Output.WriteLine("RESPONSE: " + agentResponseEvent.Response.Text);
+        Assert.NotEmpty(agentResponseEvent.Response.Text);
     }
 }

dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/Workflows/MediaInput.yaml

@@ -9,8 +9,4 @@ trigger:
       id: invoke_vision
       conversationId: =System.ConversationId
       agent:
-        name: TestAgent
-      input:
-        additionalInstructions: |-
-          Describe the image contained in the user request, if any;
-          otherwise, suggest that the user provide an image.
+        name: VisionAgent