diff --git a/dotnet/src/Microsoft.Agents.AI.Abstractions/AIAgentStructuredOutput.cs b/dotnet/src/Microsoft.Agents.AI.Abstractions/AIAgentStructuredOutput.cs index 796b796317..4be1fd4a4f 100644 --- a/dotnet/src/Microsoft.Agents.AI.Abstractions/AIAgentStructuredOutput.cs +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/AIAgentStructuredOutput.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Text.Json; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.AI; @@ -126,11 +127,77 @@ public async Task> RunAsync( { serializerOptions ??= AgentAbstractionsJsonUtilities.DefaultOptions; + var responseFormat = ChatResponseFormat.ForJsonSchema(serializerOptions); + + (responseFormat, bool isWrappedInObject) = EnsureObjectSchema(responseFormat); + options = options?.Clone() ?? new AgentRunOptions(); - options.ResponseFormat = ChatResponseFormat.ForJsonSchema(serializerOptions); + options.ResponseFormat = responseFormat; AgentResponse response = await this.RunAsync(messages, session, options, cancellationToken).ConfigureAwait(false); - return new AgentResponse(response, serializerOptions); + return new AgentResponse(response, serializerOptions) { IsWrappedInObject = isWrappedInObject }; + } + + private static bool SchemaRepresentsObject(JsonElement? schema) + { + if (schema is not { } schemaElement) + { + return false; + } + + if (schemaElement.ValueKind is JsonValueKind.Object) + { + foreach (var property in schemaElement.EnumerateObject()) + { + if (property.NameEquals("type"u8)) + { + return property.Value.ValueKind == JsonValueKind.String + && property.Value.ValueEquals("object"u8); + } + } + } + + return false; } + + private static (ChatResponseFormatJson ResponseFormat, bool IsWrappedInObject) EnsureObjectSchema(ChatResponseFormatJson responseFormat) + { + if (responseFormat.Schema is null) + { + throw new InvalidOperationException("The response format must have a valid JSON schema."); + } + + var schema = responseFormat.Schema.Value; + bool isWrappedInObject = false; + + if (!SchemaRepresentsObject(responseFormat.Schema)) + { + // For non-object-representing schemas, we wrap them in an object schema, because all + // the real LLM providers today require an object schema as the root. This is currently + // true even for providers that support native structured output. + isWrappedInObject = true; + schema = JsonSerializer.SerializeToElement(new JsonObject + { + { "$schema", "https://json-schema.org/draft/2020-12/schema" }, + { "type", "object" }, + { "properties", new JsonObject { { "data", JsonElementToJsonNode(schema) } } }, + { "additionalProperties", false }, + { "required", new JsonArray("data") }, + }, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(JsonObject))); + + responseFormat = ChatResponseFormat.ForJsonSchema(schema, responseFormat.SchemaName, responseFormat.SchemaDescription); + } + + return (responseFormat, isWrappedInObject); + } + + private static JsonNode? JsonElementToJsonNode(JsonElement element) => + element.ValueKind switch + { + JsonValueKind.Null => null, + JsonValueKind.Array => JsonArray.Create(element), + JsonValueKind.Object => JsonObject.Create(element), + _ => JsonValue.Create(element) + }; } diff --git a/dotnet/src/Microsoft.Agents.AI.Abstractions/AgentResponse{T}.cs b/dotnet/src/Microsoft.Agents.AI.Abstractions/AgentResponse{T}.cs index 8fa16f1d5b..c2fa4ff51c 100644 --- a/dotnet/src/Microsoft.Agents.AI.Abstractions/AgentResponse{T}.cs +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/AgentResponse{T}.cs @@ -43,6 +43,14 @@ public AgentResponse(ChatResponse response, JsonSerializerOptions serializerOpti this._serializerOptions = serializerOptions; } + /// + /// Gets or sets a value indicating whether the JSON schema has an extra object wrapper. + /// + /// + /// The wrapper is required for any non-JSON-object-typed values such as numbers, enum values, and arrays. + /// + internal bool IsWrappedInObject { get; init; } + /// /// Gets the result value of the agent response as an instance of . /// @@ -57,6 +65,11 @@ public virtual T Result throw new InvalidOperationException("The response did not contain JSON to be deserialized."); } + if (this.IsWrappedInObject) + { + json = UnwrapDataProperty(json!); + } + T? deserialized = DeserializeFirstTopLevelObject(json!, (JsonTypeInfo)this._serializerOptions.GetTypeInfo(typeof(T))); if (deserialized is null) { @@ -67,6 +80,19 @@ public virtual T Result } } + private static string UnwrapDataProperty(string json) + { + using var document = JsonDocument.Parse(json); + if (document.RootElement.ValueKind == JsonValueKind.Object && + document.RootElement.TryGetProperty("data", out JsonElement dataElement)) + { + return dataElement.GetRawText(); + } + + // If root is not an object or "data" property is not found, return the original JSON as a fallback + return json; + } + private static T? DeserializeFirstTopLevelObject(string json, JsonTypeInfo typeInfo) { #if NET diff --git a/dotnet/tests/AgentConformance.IntegrationTests/StructuredOutputRunTests.cs b/dotnet/tests/AgentConformance.IntegrationTests/StructuredOutputRunTests.cs index 5901986aa1..6b7556b456 100644 --- a/dotnet/tests/AgentConformance.IntegrationTests/StructuredOutputRunTests.cs +++ b/dotnet/tests/AgentConformance.IntegrationTests/StructuredOutputRunTests.cs @@ -63,6 +63,25 @@ public virtual async Task RunWithGenericTypeReturnsExpectedResultAsync() Assert.Equal("Paris", response.Result.Name); } + [RetryFact(Constants.RetryCount, Constants.RetryDelay)] + public virtual async Task RunWithPrimitiveTypeReturnsExpectedResultAsync() + { + // Arrange + var agent = this.Fixture.Agent; + var session = await agent.CreateSessionAsync(); + await using var cleanup = new SessionCleanup(session, this.Fixture); + + // Act - Request a primitive type, which requires wrapping in an object schema + AgentResponse response = await agent.RunAsync( + new ChatMessage(ChatRole.User, "What is the sum of 15 and 27? Respond with just the number."), + session); + + // Assert + Assert.NotNull(response); + Assert.Single(response.Messages); + Assert.Equal(42, response.Result); + } + protected static bool TryDeserialize(string json, JsonSerializerOptions jsonSerializerOptions, out T structuredOutput) { try diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.UnitTests/AIAgentStructuredOutputTests.cs b/dotnet/tests/Microsoft.Agents.AI.Abstractions.UnitTests/AIAgentStructuredOutputTests.cs new file mode 100644 index 0000000000..a8881ca761 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.UnitTests/AIAgentStructuredOutputTests.cs @@ -0,0 +1,391 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.AI.Abstractions.UnitTests.Models; +using Microsoft.Extensions.AI; +using Moq; +using Moq.Protected; + +namespace Microsoft.Agents.AI.Abstractions.UnitTests; + +/// +/// Unit tests for the structured output functionality in . +/// +public class AIAgentStructuredOutputTests +{ + private readonly Mock _agentMock; + + public AIAgentStructuredOutputTests() + { + this._agentMock = new Mock { CallBase = true }; + } + + #region Schema Wrapping Tests + + /// + /// Verifies that when requesting an object type, the schema is NOT wrapped. + /// + [Fact] + public async Task RunAsyncGeneric_WithObjectType_DoesNotWrapSchemaAsync() + { + // Arrange + Animal expectedAnimal = new() { Id = 1, FullName = "Test", Species = Species.Tiger }; + string responseJson = JsonSerializer.Serialize(expectedAnimal, TestJsonSerializerContext.Default.Animal); + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, responseJson)); + + this._agentMock + .Protected() + .Setup>("RunCoreAsync", + ItExpr.IsAny>(), + ItExpr.IsAny(), + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(response); + + // Act + AgentResponse result = await this._agentMock.Object.RunAsync( + "Get me an animal", + serializerOptions: TestJsonSerializerContext.Default.Options); + + // Assert - Verify the result is NOT marked as wrapped + Assert.False(result.IsWrappedInObject); + } + + /// + /// Verifies that when requesting a primitive type (int), the schema IS wrapped. + /// + [Fact] + public async Task RunAsyncGeneric_WithPrimitiveType_WrapsSchemaAsync() + { + // Arrange + const string ResponseJson = "{\"data\":42}"; + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, ResponseJson)); + + this._agentMock + .Protected() + .Setup>("RunCoreAsync", + ItExpr.IsAny>(), + ItExpr.IsAny(), + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(response); + + // Act + AgentResponse result = await this._agentMock.Object.RunAsync( + "Give me a number", + serializerOptions: TestJsonSerializerContext.Default.Options); + + // Assert - Verify the result is marked as wrapped + Assert.True(result.IsWrappedInObject); + } + + /// + /// Verifies that when requesting an array type, the schema IS wrapped. + /// + [Fact] + public async Task RunAsyncGeneric_WithArrayType_WrapsSchemaAsync() + { + // Arrange + const string ResponseJson = "{\"data\":[\"a\",\"b\",\"c\"]}"; + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, ResponseJson)); + + this._agentMock + .Protected() + .Setup>("RunCoreAsync", + ItExpr.IsAny>(), + ItExpr.IsAny(), + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(response); + + // Act + AgentResponse result = await this._agentMock.Object.RunAsync( + "Give me an array of strings", + serializerOptions: TestJsonSerializerContext.Default.Options); + + // Assert - Verify the result is marked as wrapped + Assert.True(result.IsWrappedInObject); + } + + /// + /// Verifies that when requesting an enum type, the schema IS wrapped. + /// + [Fact] + public async Task RunAsyncGeneric_WithEnumType_WrapsSchemaAsync() + { + // Arrange + const string ResponseJson = "{\"data\":\"Tiger\"}"; + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, ResponseJson)); + + this._agentMock + .Protected() + .Setup>("RunCoreAsync", + ItExpr.IsAny>(), + ItExpr.IsAny(), + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(response); + + // Act + AgentResponse result = await this._agentMock.Object.RunAsync( + "Give me a species", + serializerOptions: TestJsonSerializerContext.Default.Options); + + // Assert - Verify the result is marked as wrapped + Assert.True(result.IsWrappedInObject); + } + + #endregion + + #region AgentResponse.Result Unwrapping Tests + + /// + /// Verifies that AgentResponse{T}.Result correctly deserializes an object without unwrapping. + /// + [Fact] + public void AgentResponseGeneric_Result_DeserializesObjectWithoutUnwrapping() + { + // Arrange + Animal expectedAnimal = new() { Id = 1, FullName = "Tigger", Species = Species.Tiger }; + string responseJson = JsonSerializer.Serialize(expectedAnimal, TestJsonSerializerContext.Default.Animal); + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, responseJson)); + AgentResponse typedResponse = new(response, TestJsonSerializerContext.Default.Options); + + // Act + Animal result = typedResponse.Result; + + // Assert + Assert.Equal(expectedAnimal.Id, result.Id); + Assert.Equal(expectedAnimal.FullName, result.FullName); + Assert.Equal(expectedAnimal.Species, result.Species); + } + + /// + /// Verifies that AgentResponse{T}.Result correctly unwraps and deserializes a primitive value. + /// + [Fact] + public void AgentResponseGeneric_Result_UnwrapsPrimitiveFromDataProperty() + { + // Arrange + const string ResponseJson = "{\"data\":42}"; + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, ResponseJson)); + AgentResponse typedResponse = new(response, TestJsonSerializerContext.Default.Options) { IsWrappedInObject = true }; + + // Act + int result = typedResponse.Result; + + // Assert + Assert.Equal(42, result); + } + + /// + /// Verifies that AgentResponse{T}.Result correctly unwraps and deserializes an array. + /// + [Fact] + public void AgentResponseGeneric_Result_UnwrapsArrayFromDataProperty() + { + // Arrange + const string ResponseJson = "{\"data\":[\"apple\",\"banana\",\"cherry\"]}"; + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, ResponseJson)); + AgentResponse typedResponse = new(response, TestJsonSerializerContext.Default.Options) { IsWrappedInObject = true }; + + // Act + string[] result = typedResponse.Result; + + // Assert + Assert.Equal(["apple", "banana", "cherry"], result); + } + + /// + /// Verifies that AgentResponse{T}.Result correctly unwraps and deserializes an enum. + /// + [Fact] + public void AgentResponseGeneric_Result_UnwrapsEnumFromDataProperty() + { + // Arrange + const string ResponseJson = "{\"data\":\"Walrus\"}"; + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, ResponseJson)); + AgentResponse typedResponse = new(response, TestJsonSerializerContext.Default.Options) { IsWrappedInObject = true }; + + // Act + Species result = typedResponse.Result; + + // Assert + Assert.Equal(Species.Walrus, result); + } + + /// + /// Verifies that AgentResponse{T}.Result falls back to original JSON when data property is missing. + /// + [Fact] + public void AgentResponseGeneric_Result_FallsBackWhenDataPropertyMissing() + { + // Arrange - simulate a case where wrapping was expected but response does not have data + const string ResponseJson = "42"; + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, ResponseJson)); + AgentResponse typedResponse = new(response, TestJsonSerializerContext.Default.Options) { IsWrappedInObject = true }; + + // Act + int result = typedResponse.Result; + + // Assert - should still work by falling back to original JSON + Assert.Equal(42, result); + } + + /// + /// Verifies that AgentResponse{T}.Result throws when response text is empty. + /// + [Fact] + public void AgentResponseGeneric_Result_ThrowsWhenTextIsEmpty() + { + // Arrange + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, string.Empty)); + AgentResponse typedResponse = new(response, TestJsonSerializerContext.Default.Options); + + // Act and Assert + Assert.Throws(() => typedResponse.Result); + } + + /// + /// Verifies that AgentResponse{T}.Result throws when deserialized value is null. + /// + [Fact] + public void AgentResponseGeneric_Result_ThrowsWhenDeserializedValueIsNull() + { + // Arrange + const string ResponseJson = "null"; + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, ResponseJson)); + AgentResponse typedResponse = new(response, TestJsonSerializerContext.Default.Options); + + // Act and Assert + Assert.Throws(() => typedResponse.Result); + } + + #endregion + + #region End-to-End Tests + + /// + /// End-to-end test: Request a primitive type, verify wrapping, and verify correct deserialization. + /// + [Fact] + public async Task RunAsyncGeneric_PrimitiveEndToEnd_WrapsAndDeserializesCorrectlyAsync() + { + // Arrange + const string ResponseJson = "{\"data\":123}"; + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, ResponseJson)); + + this._agentMock + .Protected() + .Setup>("RunCoreAsync", + ItExpr.IsAny>(), + ItExpr.IsAny(), + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(response); + + // Act + AgentResponse result = await this._agentMock.Object.RunAsync( + "Give me a number", + serializerOptions: TestJsonSerializerContext.Default.Options); + + // Assert + Assert.True(result.IsWrappedInObject); + Assert.Equal(123, result.Result); + } + + /// + /// End-to-end test: Request an array type, verify wrapping, and verify correct deserialization. + /// + [Fact] + public async Task RunAsyncGeneric_ArrayEndToEnd_WrapsAndDeserializesCorrectlyAsync() + { + // Arrange + const string ResponseJson = "{\"data\":[\"one\",\"two\",\"three\"]}"; + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, ResponseJson)); + + this._agentMock + .Protected() + .Setup>("RunCoreAsync", + ItExpr.IsAny>(), + ItExpr.IsAny(), + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(response); + + // Act + AgentResponse result = await this._agentMock.Object.RunAsync( + "Give me an array of strings", + serializerOptions: TestJsonSerializerContext.Default.Options); + + // Assert + Assert.True(result.IsWrappedInObject); + Assert.Equal(["one", "two", "three"], result.Result); + } + + /// + /// End-to-end test: Request an object type, verify no wrapping, and verify correct deserialization. + /// + [Fact] + public async Task RunAsyncGeneric_ObjectEndToEnd_NoWrappingAndDeserializesCorrectlyAsync() + { + // Arrange + Animal expectedAnimal = new() { Id = 99, FullName = "Leo", Species = Species.Bear }; + string responseJson = JsonSerializer.Serialize(expectedAnimal, TestJsonSerializerContext.Default.Animal); + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, responseJson)); + + this._agentMock + .Protected() + .Setup>("RunCoreAsync", + ItExpr.IsAny>(), + ItExpr.IsAny(), + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(response); + + // Act + AgentResponse result = await this._agentMock.Object.RunAsync( + "Give me an animal", + serializerOptions: TestJsonSerializerContext.Default.Options); + + // Assert + Assert.False(result.IsWrappedInObject); + Assert.Equal(expectedAnimal.Id, result.Result.Id); + Assert.Equal(expectedAnimal.FullName, result.Result.FullName); + Assert.Equal(expectedAnimal.Species, result.Result.Species); + } + + /// + /// End-to-end test: Request an enum type, verify wrapping, and verify correct deserialization. + /// + [Fact] + public async Task RunAsyncGeneric_EnumEndToEnd_WrapsAndDeserializesCorrectlyAsync() + { + // Arrange + const string ResponseJson = "{\"data\":\"Bear\"}"; + AgentResponse response = new(new ChatMessage(ChatRole.Assistant, ResponseJson)); + + this._agentMock + .Protected() + .Setup>("RunCoreAsync", + ItExpr.IsAny>(), + ItExpr.IsAny(), + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(response); + + // Act + AgentResponse result = await this._agentMock.Object.RunAsync( + "Give me a species", + serializerOptions: TestJsonSerializerContext.Default.Options); + + // Assert + Assert.True(result.IsWrappedInObject); + Assert.Equal(Species.Bear, result.Result); + } + + #endregion +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Abstractions.UnitTests/TestJsonSerializerContext.cs b/dotnet/tests/Microsoft.Agents.AI.Abstractions.UnitTests/TestJsonSerializerContext.cs index 05d13c2e95..94512bd243 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Abstractions.UnitTests/TestJsonSerializerContext.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Abstractions.UnitTests/TestJsonSerializerContext.cs @@ -15,6 +15,7 @@ namespace Microsoft.Agents.AI.Abstractions.UnitTests; [JsonSerializable(typeof(AgentResponseUpdate))] [JsonSerializable(typeof(AgentRunOptions))] [JsonSerializable(typeof(Animal))] +[JsonSerializable(typeof(Species))] [JsonSerializable(typeof(JsonElement))] [JsonSerializable(typeof(Dictionary))] [JsonSerializable(typeof(string[]))]