From 094faa0dae288b665c1dd268c020d0c914d85176 Mon Sep 17 00:00:00 2001 From: Debdatta Kunda Date: Wed, 18 Sep 2024 12:02:36 -0700 Subject: [PATCH] Code changes to update STJ Serializer --- .../Resource/Container/ContainerCore.Items.cs | 45 ++++++++----- .../src/Serializer/CosmosSerializationUtil.cs | 2 +- .../CosmosSystemTextJsonSerializer.cs | 36 +++++++++- .../CosmosItemUnitTests.cs | 66 +++++++++++++------ 4 files changed, 109 insertions(+), 40 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs index 82b4adbf8d..7edf551b9e 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerCore.Items.cs @@ -60,7 +60,7 @@ public async Task CreateItemStreamAsync( requestOptions: requestOptions, trace: trace, cancellationToken: cancellationToken, - targetRequestSerializationFormat: ConfigurationManager.IsBinaryEncodingEnabled() ? JsonSerializationFormat.Binary : JsonSerializationFormat.Text, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), targetResponseSerializationFormat: JsonSerializationFormat.Text); } @@ -81,7 +81,8 @@ public async Task> CreateItemAsync( itemId: null, item: item, operationType: OperationType.Create, - requestOptions: requestOptions, + requestOptions: requestOptions, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), trace: trace, cancellationToken: cancellationToken); @@ -103,7 +104,7 @@ public async Task ReadItemStreamAsync( requestOptions: requestOptions, trace: trace, cancellationToken: cancellationToken, - targetRequestSerializationFormat: ConfigurationManager.IsBinaryEncodingEnabled() ? JsonSerializationFormat.Binary : JsonSerializationFormat.Text, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), targetResponseSerializationFormat: JsonSerializationFormat.Text); } @@ -119,7 +120,8 @@ public async Task> ReadItemAsync( itemId: id, streamPayload: null, operationType: OperationType.Read, - requestOptions: requestOptions, + requestOptions: requestOptions, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), trace: trace, cancellationToken: cancellationToken); @@ -141,7 +143,7 @@ public async Task UpsertItemStreamAsync( requestOptions: requestOptions, trace: trace, cancellationToken: cancellationToken, - targetRequestSerializationFormat: ConfigurationManager.IsBinaryEncodingEnabled() ? JsonSerializationFormat.Binary : JsonSerializationFormat.Text, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), targetResponseSerializationFormat: JsonSerializationFormat.Text); } @@ -162,7 +164,8 @@ public async Task> UpsertItemAsync( itemId: null, item: item, operationType: OperationType.Upsert, - requestOptions: requestOptions, + requestOptions: requestOptions, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), trace: trace, cancellationToken: cancellationToken); @@ -185,7 +188,7 @@ public async Task ReplaceItemStreamAsync( requestOptions: requestOptions, trace: trace, cancellationToken: cancellationToken, - targetRequestSerializationFormat: ConfigurationManager.IsBinaryEncodingEnabled() ? JsonSerializationFormat.Binary : JsonSerializationFormat.Text, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), targetResponseSerializationFormat: JsonSerializationFormat.Text); } @@ -212,7 +215,8 @@ public async Task> ReplaceItemAsync( itemId: id, item: item, operationType: OperationType.Replace, - requestOptions: requestOptions, + requestOptions: requestOptions, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), trace: trace, cancellationToken: cancellationToken); @@ -234,7 +238,7 @@ public async Task DeleteItemStreamAsync( requestOptions: requestOptions, trace: trace, cancellationToken: cancellationToken, - targetRequestSerializationFormat: ConfigurationManager.IsBinaryEncodingEnabled() ? JsonSerializationFormat.Binary : JsonSerializationFormat.Text, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), targetResponseSerializationFormat: JsonSerializationFormat.Text); } @@ -250,7 +254,8 @@ public async Task> DeleteItemAsync( itemId: id, streamPayload: null, operationType: OperationType.Delete, - requestOptions: requestOptions, + requestOptions: requestOptions, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), trace: trace, cancellationToken: cancellationToken); @@ -838,7 +843,8 @@ private async Task ExtractPartitionKeyAndProcessItemStreamAsync string itemId, T item, OperationType operationType, - ItemRequestOptions requestOptions, + ItemRequestOptions requestOptions, + JsonSerializationFormat? targetRequestSerializationFormat, ITrace trace, CancellationToken cancellationToken) { @@ -863,7 +869,8 @@ private async Task ExtractPartitionKeyAndProcessItemStreamAsync operationType, requestOptions, trace: trace, - cancellationToken: cancellationToken); + cancellationToken: cancellationToken, + targetRequestSerializationFormat: targetRequestSerializationFormat); } PartitionKeyMismatchRetryPolicy requestRetryPolicy = null; @@ -876,7 +883,8 @@ private async Task ExtractPartitionKeyAndProcessItemStreamAsync itemId, itemStream, operationType, - requestOptions, + requestOptions, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), trace: trace, cancellationToken: cancellationToken); @@ -908,7 +916,7 @@ private async Task ProcessItemStreamAsync( ItemRequestOptions requestOptions, ITrace trace, CancellationToken cancellationToken, - JsonSerializationFormat? targetRequestSerializationFormat = default, + JsonSerializationFormat? targetRequestSerializationFormat, JsonSerializationFormat? targetResponseSerializationFormat = default) { if (trace == null) @@ -1264,7 +1272,7 @@ public Task PatchItemStreamAsync( requestOptions: requestOptions, trace: trace, cancellationToken: cancellationToken, - targetRequestSerializationFormat: ConfigurationManager.IsBinaryEncodingEnabled() ? JsonSerializationFormat.Binary : JsonSerializationFormat.Text, + targetRequestSerializationFormat: ContainerCore.GetTargetRequestSerializationFormat(), targetResponseSerializationFormat: JsonSerializationFormat.Text); } @@ -1300,6 +1308,13 @@ private ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderPrivate( container: this, changeFeedProcessor: changeFeedProcessor, applyBuilderConfiguration: changeFeedProcessor.ApplyBuildConfiguration).WithChangeFeedMode(mode); + } + + private static JsonSerializationFormat GetTargetRequestSerializationFormat() + { + return ConfigurationManager.IsBinaryEncodingEnabled() + ? JsonSerializationFormat.Binary + : JsonSerializationFormat.Text; } } } diff --git a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializationUtil.cs b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializationUtil.cs index 0b9abe8010..5e124c3fcb 100644 --- a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializationUtil.cs +++ b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializationUtil.cs @@ -150,7 +150,7 @@ internal static Stream ConvertToStreamUsingJsonSerializationFormat( byte[] formattedBytes = writer.GetResult().ToArray(); - return new MemoryStream(formattedBytes, index: 0, count: formattedBytes.Length, writable: false, publiclyVisible: true); + return new MemoryStream(formattedBytes, index: 0, count: formattedBytes.Length, writable: true, publiclyVisible: true); } /// diff --git a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSystemTextJsonSerializer.cs b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSystemTextJsonSerializer.cs index ac5550b128..27387f4865 100644 --- a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSystemTextJsonSerializer.cs +++ b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSystemTextJsonSerializer.cs @@ -9,6 +9,8 @@ namespace Microsoft.Azure.Cosmos using System.Reflection; using System.Text.Json; using System.Text.Json.Serialization; + using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Json; /// /// This class provides a default implementation of System.Text.Json Cosmos Linq Serializer. @@ -49,8 +51,23 @@ public override T FromStream(Stream stream) using (stream) { - using StreamReader reader = new (stream); - return JsonSerializer.Deserialize(reader.ReadToEnd(), this.jsonSerializerOptions); + if (CosmosSerializationUtil.CheckFirstBufferByte( + stream, + JsonSerializationFormat.Binary, + out byte[] content)) + { + if (CosmosObject.TryCreateFromBuffer(content, out CosmosObject cosmosObject)) + { + return System.Text.Json.JsonSerializer.Deserialize(cosmosObject.ToString(), this.jsonSerializerOptions); + } + else + { + using Stream textStream = CosmosSerializationUtil.ConvertToStreamUsingJsonSerializationFormat(content, JsonSerializationFormat.Text); + return this.DeserializeStream(textStream); + } + } + + return this.DeserializeStream(stream); } } @@ -60,7 +77,7 @@ public override Stream ToStream(T input) MemoryStream streamPayload = new (); using Utf8JsonWriter writer = new (streamPayload); - JsonSerializer.Serialize(writer, input, this.jsonSerializerOptions); + System.Text.Json.JsonSerializer.Serialize(writer, input, this.jsonSerializerOptions); streamPayload.Position = 0; return streamPayload; @@ -100,5 +117,18 @@ public override string SerializeMemberName(MemberInfo memberInfo) return memberInfo.Name; } + + /// + /// Deserializes the stream into the specified type using STJ Serializer. + /// + /// The desired type, the input stream to be deserialize into + /// An instance of containing th raw input stream. + /// The deserialized output of type . + private T DeserializeStream( + Stream stream) + { + using StreamReader reader = new (stream); + return System.Text.Json.JsonSerializer.Deserialize(reader.ReadToEnd(), this.jsonSerializerOptions); + } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs index 0aa08e532d..b962b4d3e6 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosItemUnitTests.cs @@ -22,13 +22,18 @@ namespace Microsoft.Azure.Cosmos.Tests public class CosmosItemUnitTests { [TestMethod] - [DataRow(false, true, DisplayName = "Test scenario when binary encoding is disabled at client level and supported in container level.")] - [DataRow(true, true, DisplayName = "Test scenario when binary encoding is enabled at client level and supported in container level.")] - [DataRow(false, false, DisplayName = "Test scenario when binary encoding iis disabled at client level and disabled in container level.")] - [DataRow(true, false, DisplayName = "Test scenario when binary encoding is enabled at client level and disabled in container level.")] + [DataRow(false, true, false, DisplayName = "Test scenario with CosmosJsonDotNetSerializer when binary encoding is disabled at client level and enabled in container level.")] + [DataRow(true, true, false, DisplayName = "Test scenario with CosmosJsonDotNetSerializer when binary encoding is enabled at client level and enabled in container level.")] + [DataRow(false, false, false, DisplayName = "Test scenario with CosmosJsonDotNetSerializer when binary encoding iis disabled at client level and disabled in container level.")] + [DataRow(true, false, false, DisplayName = "Test scenario with CosmosJsonDotNetSerializer when binary encoding is enabled at client level and disabled in container level.")] + [DataRow(false, true, true, DisplayName = "Test scenario with CosmosSystemTextJsonSerializer when binary encoding is disabled at client level and enabled in container level.")] + [DataRow(true, true, true, DisplayName = "Test scenario with CosmosSystemTextJsonSerializer when binary encoding is enabled at client level and enabled in container level.")] + [DataRow(false, false, true, DisplayName = "Test scenario with CosmosSystemTextJsonSerializer when binary encoding iis disabled at client level and disabled in container level.")] + [DataRow(true, false, true, DisplayName = "Test scenario with CosmosSystemTextJsonSerializer when binary encoding is enabled at client level and disabled in container level.")] public async Task TestItemPartitionKeyTypes( bool binaryEncodingEnabledInClient, - bool binaryEncodingEnabledInContainer) + bool binaryEncodingEnabledInContainer, + bool useStjSerializer) { dynamic item = new { @@ -40,7 +45,8 @@ await VerifyItemOperations( "[\"FF627B77-568E-4541-A47E-041EAC10E46F\"]", item, binaryEncodingEnabledInClient, - binaryEncodingEnabledInContainer); + binaryEncodingEnabledInContainer, + useStjSerializer); item = new { @@ -52,7 +58,8 @@ await VerifyItemOperations( "[4567.0]", item, binaryEncodingEnabledInClient, - binaryEncodingEnabledInContainer); + binaryEncodingEnabledInContainer, + useStjSerializer); item = new { @@ -64,7 +71,8 @@ await VerifyItemOperations( "[4567.1234]", item, binaryEncodingEnabledInClient, - binaryEncodingEnabledInContainer); + binaryEncodingEnabledInContainer, + useStjSerializer); item = new { @@ -76,24 +84,30 @@ await VerifyItemOperations( "[true]", item, binaryEncodingEnabledInClient, - binaryEncodingEnabledInContainer); + binaryEncodingEnabledInContainer, + useStjSerializer); } [TestMethod] - [DataRow(false, true, DisplayName = "Test scenario when binary encoding is disabled at client level and supported in container level.")] - [DataRow(true, true, DisplayName = "Test scenario when binary encoding is enabled at client level and supported in container level.")] - [DataRow(false, false, DisplayName = "Test scenario when binary encoding iis disabled at client level and disabled in container level.")] - [DataRow(true, false, DisplayName = "Test scenario when binary encoding is enabled at client level and disabled in container level.")] + [DataRow(false, true, false, DisplayName = "Test scenario with CosmosJsonDotNetSerializer when binary encoding is disabled at client level and enabled in container level.")] + [DataRow(true, true, false, DisplayName = "Test scenario with CosmosJsonDotNetSerializer when binary encoding is enabled at client level and enabled in container level.")] + [DataRow(false, false, false, DisplayName = "Test scenario with CosmosJsonDotNetSerializer when binary encoding iis disabled at client level and disabled in container level.")] + [DataRow(true, false, false, DisplayName = "Test scenario with CosmosJsonDotNetSerializer when binary encoding is enabled at client level and disabled in container level.")] + [DataRow(false, true, true, DisplayName = "Test scenario with CosmosSystemTextJsonSerializer when binary encoding is disabled at client level and enabled in container level.")] + [DataRow(true, true, true, DisplayName = "Test scenario with CosmosSystemTextJsonSerializer when binary encoding is enabled at client level and enabled in container level.")] + [DataRow(false, false, true, DisplayName = "Test scenario with CosmosSystemTextJsonSerializer when binary encoding iis disabled at client level and disabled in container level.")] + [DataRow(true, false, true, DisplayName = "Test scenario with CosmosSystemTextJsonSerializer when binary encoding is enabled at client level and disabled in container level.")] public async Task TestNullItemPartitionKeyFlag( bool binaryEncodingEnabledInClient, - bool binaryEncodingEnabledInContainer) + bool binaryEncodingEnabledInContainer, + bool useStjSerializer) { dynamic testItem = new { id = Guid.NewGuid().ToString() }; - await VerifyItemOperations(new Cosmos.PartitionKey(Undefined.Value), "[{}]", testItem, binaryEncodingEnabledInClient, binaryEncodingEnabledInContainer); + await VerifyItemOperations(new Cosmos.PartitionKey(Undefined.Value), "[{}]", testItem, binaryEncodingEnabledInClient, binaryEncodingEnabledInContainer, useStjSerializer); } [TestMethod] @@ -131,6 +145,7 @@ await VerifyItemOperations( item, true, true, + false, requestOptions); } @@ -843,6 +858,7 @@ private async Task VerifyItemOperations( dynamic testItem, bool binaryEncodingEnabledInClient, bool binaryEncodingEnabledInContainer, + bool useStjSerializer, ItemRequestOptions requestOptions = null) { try @@ -887,7 +903,15 @@ private async Task VerifyItemOperations( }); using CosmosClient client = MockCosmosUtil.CreateMockCosmosClient( - (builder) => builder.AddCustomHandlers(testHandler)); + (builder) => + { + if (useStjSerializer) + { + builder.WithSystemTextJsonSerializerOptions(new System.Text.Json.JsonSerializerOptions()); + } + + builder.AddCustomHandlers(testHandler); + }); Container container = client.GetDatabase("testdb") .GetContainer("testcontainer"); @@ -897,7 +921,7 @@ private async Task VerifyItemOperations( requestOptions: requestOptions); Assert.IsNotNull(itemResponse); Assert.AreEqual(httpStatusCode, itemResponse.StatusCode); - Assert.AreEqual(itemResponseString, itemResponse.Resource); + Assert.AreEqual(itemResponseString, itemResponse.Resource.ToString()); itemResponse = await container.ReadItemAsync( partitionKey: partitionKey, @@ -905,14 +929,14 @@ private async Task VerifyItemOperations( requestOptions: requestOptions); Assert.IsNotNull(itemResponse); Assert.AreEqual(httpStatusCode, itemResponse.StatusCode); - Assert.AreEqual(itemResponseString, itemResponse.Resource); + Assert.AreEqual(itemResponseString, itemResponse.Resource.ToString()); itemResponse = await container.UpsertItemAsync( item: testItem, requestOptions: requestOptions); Assert.IsNotNull(itemResponse); Assert.AreEqual(httpStatusCode, itemResponse.StatusCode); - Assert.AreEqual(itemResponseString, itemResponse.Resource); + Assert.AreEqual(itemResponseString, itemResponse.Resource.ToString()); itemResponse = await container.ReplaceItemAsync( id: testItem.id, @@ -920,7 +944,7 @@ private async Task VerifyItemOperations( requestOptions: requestOptions); Assert.IsNotNull(itemResponse); Assert.AreEqual(httpStatusCode, itemResponse.StatusCode); - Assert.AreEqual(itemResponseString, itemResponse.Resource); + Assert.AreEqual(itemResponseString, itemResponse.Resource.ToString()); itemResponse = await container.DeleteItemAsync( partitionKey: partitionKey, @@ -928,7 +952,7 @@ private async Task VerifyItemOperations( requestOptions: requestOptions); Assert.IsNotNull(itemResponse); Assert.AreEqual(httpStatusCode, itemResponse.StatusCode); - Assert.AreEqual(itemResponseString, itemResponse.Resource); + Assert.AreEqual(itemResponseString, itemResponse.Resource.ToString()); Assert.AreEqual(5, testHandlerHitCount, "An operation did not make it to the handler");