From f629e508b536d6362790671581de366d87329dc2 Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Tue, 3 Dec 2024 19:43:00 -0800 Subject: [PATCH 01/33] Update ChangeFeedItem to include id and pk metadata --- .../Resource/FullFidelity/ChangeFeedItem.cs | 6 ++--- .../FullFidelity/ChangeFeedMetadata.cs | 17 ++++++++++++-- .../FullFidelity/ChangeFeedMetadataFields.cs | 2 ++ .../Converters/ChangeFeedMetadataConverter.cs | 22 +++++++++++++++++++ 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs index 417cc3b1b6..dc856ef553 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs @@ -57,21 +57,21 @@ namespace Microsoft.Azure.Cosmos class ChangeFeedItem { /// - /// The full fidelity change feed current item. + /// The current version of the item for all versions and deletes change feed mode. /// [JsonProperty(PropertyName = "current")] [JsonPropertyName("current")] public T Current { get; set; } /// - /// The full fidelity change feed metadata. + /// The item metadata for all versions and deletes change feed mode. /// [JsonProperty(PropertyName = "metadata", NullValueHandling = NullValueHandling.Ignore)] [JsonPropertyName("metadata")] public ChangeFeedMetadata Metadata { get; set; } /// - /// For delete operations, previous image is always going to be provided. The previous image on replace operations is not going to be exposed by default and requires account-level or container-level opt-in. + /// The previous version of the item for all versions and deletes change feed mode. The previous version on delete and replace operations is not exposed by default and requires container-level opt-in. Refer to https://aka.ms/cosmosdb-change-feed-deletes for more information. /// [JsonProperty(PropertyName = "previous", NullValueHandling = NullValueHandling.Ignore)] [JsonPropertyName("previous")] diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index 1dae4f1e1b..b507644d9f 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos { using System; + using System.Collections.Generic; using System.Text.Json; using Microsoft.Azure.Cosmos.Resource.FullFidelity; using Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters; @@ -21,7 +22,7 @@ namespace Microsoft.Azure.Cosmos #else internal #endif - class ChangeFeedMetadata + class ChangeFeedMetadata { /// /// The change's conflict resolution timestamp. @@ -50,9 +51,21 @@ class ChangeFeedMetadata public long PreviousLsn { get; internal set; } /// - /// Used to distinquish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents). + /// Used to distinguish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents). /// [JsonProperty(PropertyName = ChangeFeedMetadataFields.TimeToLiveExpired, NullValueHandling = NullValueHandling.Ignore)] public bool IsTimeToLiveExpired { get; internal set; } + + /// + /// The id of the previous item version. Used for delete operations only. + /// + [JsonProperty(PropertyName = ChangeFeedMetadataFields.DeletedItemId, NullValueHandling = NullValueHandling.Ignore)] + public string DeletedItemId { get; internal set; } + + /// + /// The partition key of the previous item version. Dictionary Key is the partition key property name and Dictionary Value is the partition key property value. Used for delete operations only. + /// + [JsonProperty(PropertyName = ChangeFeedMetadataFields.DeletedItemPartitionKey, NullValueHandling = NullValueHandling.Ignore)] + public Dictionary DeletedItemPartitionKey { get; internal set; } } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs index db39a386a9..d45bf1bd44 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs @@ -11,5 +11,7 @@ internal class ChangeFeedMetadataFields public const string OperationType = "operationType"; public const string PreviousImageLSN = "previousImageLSN"; public const string TimeToLiveExpired = "timeToLiveExpired"; + public const string DeletedItemId = "id"; + public const string DeletedItemPartitionKey = "partitionKey"; } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs index 0b5056051a..f448772ee2 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters { using System; + using System.Collections.Generic; using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; @@ -56,6 +57,19 @@ public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToCo { metadata.PreviousLsn = property.Value.GetInt64(); } + else if (property.NameEquals(ChangeFeedMetadataFields.DeletedItemId)) + { + metadata.DeletedItemId = property.Value.GetString(); + } + else if (property.NameEquals(ChangeFeedMetadataFields.DeletedItemPartitionKey)) + { + Dictionary partitionKey = new Dictionary(); + foreach (JsonProperty pk in property.Value.EnumerateObject()) + { + partitionKey.Add(pk.Name, pk.Value.GetString()); + } + metadata.DeletedItemPartitionKey = partitionKey; + } } return metadata; @@ -75,6 +89,14 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json writer.WriteNumber(ChangeFeedMetadataFields.Lsn, value.Lsn); writer.WriteString(ChangeFeedMetadataFields.OperationType, value.OperationType.ToString()); writer.WriteNumber(ChangeFeedMetadataFields.PreviousImageLSN, value.PreviousLsn); + writer.WriteString(ChangeFeedMetadataFields.DeletedItemId, value.DeletedItemId); + + writer.WriteStartObject("partitionKey"); + foreach (KeyValuePair kvp in value.DeletedItemPartitionKey) + { + writer.WriteString(kvp.Key, kvp.Value); + } + writer.WriteEndObject(); writer.WriteEndObject(); } From 970b689ef79490295cdd0635488da2e4d9945277 Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Thu, 5 Dec 2024 14:55:58 -0800 Subject: [PATCH 02/33] update emulator tests for avad delete operations --- .../CFP/AllVersionsAndDeletes/BuilderTests.cs | 10 ++++++++++ .../BuilderWithCustomSerializerTests.cs | 16 ++++++++++++++++ .../FeedToken/ChangeFeedIteratorCoreTests.cs | 2 ++ 3 files changed, 28 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs index 669c6bd194..c5c96b0a22 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs @@ -84,6 +84,9 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); // previous + Assert.AreEqual(expected: "1", actual: change.Metadata.DeletedItemId.ToString()); + change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString()); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString()); @@ -155,6 +158,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() ChangeFeedProcessor processor = monitoredContainer .GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(processorName: "processor", onChangesDelegate: (ChangeFeedProcessorContext context, IReadOnlyCollection> docs, CancellationToken token) => { + string metadataId = default; + string metadataPk = default; string id = default; string pk = default; string description = default; @@ -171,6 +176,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() } else { + metadataId = change.Metadata.DeletedItemId.ToString(); + change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out metadataPk).ToString(); id = change.Previous.id.ToString(); pk = change.Previous.pk.ToString(); description = change.Previous.description.ToString(); @@ -211,6 +218,9 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() ChangeFeedItem deleteChange = docs.ElementAt(2); Assert.IsNull(deleteChange.Current.id); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.DeletedItemId.ToString()); + deleteChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); Assert.IsNotNull(deleteChange.Previous); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index a6780e4409..08cf70a150 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -92,6 +92,9 @@ static void ValidateDeserialization(List> activitie Assert.IsTrue(deletedChange.Metadata.IsTimeToLiveExpired); Assert.IsNotNull(deletedChange.Previous); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: deletedChange.Previous.description); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.DeletedItemId.ToString()); + deletedChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: "1", actual: deletedChange.Previous.id); Assert.AreEqual(expected: 5, actual: deletedChange.Previous.ttl); } @@ -295,6 +298,9 @@ static void ValidateDeserialization(List> activitie Assert.IsFalse(deletedChange.Metadata.IsTimeToLiveExpired); Assert.IsNotNull(deletedChange.Previous); Assert.AreEqual(expected: "test after replace", actual: deletedChange.Previous.description); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.DeletedItemId.ToString()); + deletedChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: "1", actual: deletedChange.Previous.id); Assert.AreEqual(expected: 0, actual: deletedChange.Previous.ttl); } @@ -416,6 +422,9 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); // previous + Assert.AreEqual(expected: "1", actual: change.Metadata.DeletedItemId.ToString()); + change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString()); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString()); @@ -508,6 +517,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr { Logger.LogLine($"@ {DateTime.Now}, {nameof(docs)} -> {System.Text.Json.JsonSerializer.Serialize(docs)}"); + string metadataId = default; + string metadataPk = default; string id = default; string pk = default; string description = default; @@ -522,6 +533,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr } else { + metadataId = change.Metadata.DeletedItemId.ToString(); + change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out metadataPk).ToString(); id = change.Previous.id.ToString(); pk = change.Previous.pk.ToString(); description = change.Previous.description.ToString(); @@ -565,6 +578,9 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); Assert.IsNotNull(deleteChange.Previous); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.DeletedItemId.ToString()); + deleteChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: "1", actual: deleteChange.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: deleteChange.Previous.pk.ToString()); Assert.AreEqual(expected: "test after replace", actual: deleteChange.Previous.description.ToString()); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs index a43d86faf5..f4cc9c2ab5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs @@ -896,6 +896,7 @@ private async Task ValidateChangeFeedIteratorCore_WithQuery( foreach (ChangeFeedItem item in feedResponse) { + Assert.AreEqual(expected: "id3", actual: item.Metadata.DeletedItemId.ToString()); Assert.AreEqual("id3", item.Previous.Id); Assert.AreEqual(ChangeFeedOperationType.Delete, item.Metadata.OperationType); } @@ -1094,6 +1095,7 @@ public async Task ChangeFeedIteratorCore_FeedRange_VerifyingWireFormatTests() Assert.AreNotEqual(notExpected: default, actual: deleteOperation.Metadata.Lsn); Assert.AreNotEqual(notExpected: default, actual: deleteOperation.Metadata.PreviousLsn); Assert.IsNotNull(deleteOperation.Previous); + Assert.AreEqual(expected: id, actual: deleteOperation.Metadata.DeletedItemId.ToString()); Assert.AreEqual(expected: id, actual: deleteOperation.Previous.Id); Assert.AreEqual(expected: "205 16th St NW", actual: deleteOperation.Previous.Line1); Assert.AreEqual(expected: "Atlanta", actual: deleteOperation.Previous.City); From 1823f65a7446e63e98f78cc8ecf1e22a63bcae87 Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Tue, 10 Dec 2024 18:00:14 -0800 Subject: [PATCH 03/33] add output of UpdateContracts script --- .../Contracts/DotNetPreviewSDKAPI.json | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index f5ed83ba12..2a48f3ea62 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -137,6 +137,20 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Collections.Generic.Dictionary`2[System.String,System.String] DeletedItemPartitionKey[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"partitionKey\")]": { + "Type": "Property", + "Attributes": [ + "JsonPropertyAttribute" + ], + "MethodInfo": "System.Collections.Generic.Dictionary`2[System.String,System.String] DeletedItemPartitionKey;CanRead:True;CanWrite:True;System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "System.DateTime ConflictResolutionTimestamp[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"crts\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Microsoft.Azure.Documents.UnixDateTimeConverter))]": { "Type": "Property", "Attributes": [ @@ -152,6 +166,20 @@ ], "MethodInfo": "System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.String DeletedItemId[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"id\")]": { + "Type": "Property", + "Attributes": [ + "JsonPropertyAttribute" + ], + "MethodInfo": "System.String DeletedItemId;CanRead:True;CanWrite:True;System.String get_DeletedItemId();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.String get_DeletedItemId()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "System.String get_DeletedItemId();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor()": { "Type": "Constructor", "Attributes": [], From 1e66f3b27ba129793976c2e4411d69da8b65c8e3 Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Wed, 11 Dec 2024 17:16:25 -0800 Subject: [PATCH 04/33] undo encryption contracts changes --- .../DotNetSDKEncryptionCustomAPI.json | 258 +++++++----------- 1 file changed, 93 insertions(+), 165 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json index 35cef3bfdc..ccb4b56fff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json @@ -1,111 +1,5 @@ { "Subclasses": { - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions;System.Object;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { - "Subclasses": {}, - "Members": { - "CompressionAlgorithm Algorithm": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "CompressionAlgorithm Algorithm;CanRead:True;CanWrite:True;CompressionAlgorithm get_Algorithm();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_Algorithm(CompressionAlgorithm);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "CompressionAlgorithm get_Algorithm()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "CompressionAlgorithm get_Algorithm();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Int32 get_MinimalCompressedLength()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Int32 get_MinimalCompressedLength();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Int32 MinimalCompressedLength": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "Int32 MinimalCompressedLength;CanRead:True;CanWrite:True;Int32 get_MinimalCompressedLength();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_MinimalCompressedLength(Int32);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions+CompressionAlgorithm": { - "Type": "NestedType", - "Attributes": [], - "MethodInfo": null - }, - "System.IO.Compression.CompressionLevel CompressionLevel": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "System.IO.Compression.CompressionLevel CompressionLevel;CanRead:True;CanWrite:True;System.IO.Compression.CompressionLevel get_CompressionLevel();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_CompressionLevel(System.IO.Compression.CompressionLevel);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.IO.Compression.CompressionLevel get_CompressionLevel()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "System.IO.Compression.CompressionLevel get_CompressionLevel();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Void .ctor()": { - "Type": "Constructor", - "Attributes": [], - "MethodInfo": "[Void .ctor(), Void .ctor()]" - }, - "Void set_Algorithm(CompressionAlgorithm)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_Algorithm(CompressionAlgorithm);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Void set_CompressionLevel(System.IO.Compression.CompressionLevel)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_CompressionLevel(System.IO.Compression.CompressionLevel);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Void set_MinimalCompressedLength(Int32)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_MinimalCompressedLength(Int32);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - } - }, - "NestedTypes": { - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions+CompressionAlgorithm;System.Enum;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:True;IsClass:False;IsValueType:True;IsNested:True;IsGenericType:False;IsSerializable:True": { - "Subclasses": {}, - "Members": { - "CompressionAlgorithm None": { - "Type": "Field", - "Attributes": [], - "MethodInfo": "CompressionAlgorithm None;IsInitOnly:False;IsStatic:True;" - }, - "Int32 value__": { - "Type": "Field", - "Attributes": [], - "MethodInfo": "Int32 value__;IsInitOnly:False;IsStatic:False;" - } - }, - "NestedTypes": {} - } - } - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions+CompressionAlgorithm;System.Enum;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:True;IsClass:False;IsValueType:True;IsNested:True;IsGenericType:False;IsSerializable:True": { - "Subclasses": {}, - "Members": { - "CompressionAlgorithm None": { - "Type": "Field", - "Attributes": [], - "MethodInfo": "CompressionAlgorithm None;IsInitOnly:False;IsStatic:True;" - }, - "Int32 value__": { - "Type": "Field", - "Attributes": [], - "MethodInfo": "Int32 value__;IsInitOnly:False;IsStatic:False;" - } - }, - "NestedTypes": {} - }, "Microsoft.Azure.Cosmos.Encryption.Custom.CosmosDataEncryptionKeyProvider;Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { @@ -226,20 +120,54 @@ ], "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -1085,30 +1013,6 @@ "Microsoft.Azure.Cosmos.Encryption.Custom.EncryptionOptions;System.Object;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions CompressionOptions": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions CompressionOptions;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions get_CompressionOptions();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_CompressionOptions(Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions get_CompressionOptions()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions get_CompressionOptions();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor get_JsonProcessor()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor get_JsonProcessor();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor JsonProcessor": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor JsonProcessor;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor get_JsonProcessor();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_JsonProcessor(Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "System.Collections.Generic.IEnumerable`1[System.String] get_PathsToEncrypt()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -1150,13 +1054,6 @@ "Attributes": [], "MethodInfo": "[Void .ctor(), Void .ctor()]" }, - "Void set_CompressionOptions(Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_CompressionOptions(Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "Void set_DataEncryptionKeyId(System.String)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -1171,13 +1068,6 @@ ], "MethodInfo": "Void set_EncryptionAlgorithm(System.String);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Void set_JsonProcessor(Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_JsonProcessor(Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "Void set_PathsToEncrypt(System.Collections.Generic.IEnumerable`1[System.String])[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -1242,20 +1132,54 @@ ], "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -1280,22 +1204,26 @@ "Type": "Method", "Attributes": [], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - } - }, - "NestedTypes": {} - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor;System.Enum;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:True;IsClass:False;IsValueType:True;IsNested:False;IsGenericType:False;IsSerializable:True": { - "Subclasses": {}, - "Members": { - "Int32 value__": { - "Type": "Field", + }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", "Attributes": [], - "MethodInfo": "Int32 value__;IsInitOnly:False;IsStatic:False;" + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor Newtonsoft": { - "Type": "Field", + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor Newtonsoft;IsInitOnly:False;IsStatic:True;" + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" } }, "NestedTypes": {} From 60223b4a487b741ad3d314443247b63ff814b262 Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Wed, 11 Dec 2024 17:17:54 -0800 Subject: [PATCH 05/33] update changefeed metadata serialization --- .../Converters/ChangeFeedMetadataConverter.cs | 17 ++++++++++++----- .../BuilderWithCustomSerializerTests.cs | 12 ++++++++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs index f448772ee2..6112c7c22a 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs @@ -89,14 +89,21 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json writer.WriteNumber(ChangeFeedMetadataFields.Lsn, value.Lsn); writer.WriteString(ChangeFeedMetadataFields.OperationType, value.OperationType.ToString()); writer.WriteNumber(ChangeFeedMetadataFields.PreviousImageLSN, value.PreviousLsn); - writer.WriteString(ChangeFeedMetadataFields.DeletedItemId, value.DeletedItemId); - writer.WriteStartObject("partitionKey"); - foreach (KeyValuePair kvp in value.DeletedItemPartitionKey) + if (value.DeletedItemId != null) { - writer.WriteString(kvp.Key, kvp.Value); + writer.WriteString(ChangeFeedMetadataFields.DeletedItemId, value.DeletedItemId); + } + + if (value.DeletedItemPartitionKey != null) + { + writer.WriteStartObject(ChangeFeedMetadataFields.DeletedItemPartitionKey); + foreach (KeyValuePair kvp in value.DeletedItemPartitionKey) + { + writer.WriteString(kvp.Key, kvp.Value); + } + writer.WriteEndObject(); } - writer.WriteEndObject(); writer.WriteEndObject(); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 08cf70a150..c8c35b069b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -39,7 +39,11 @@ public void ValidateNSJAndSTJSerializationOfChangeFeedItemDeleteTimeToLiveExpire ""crts"": 1722511591, ""operationType"": ""delete"", ""timeToLiveExpired"": true, - ""previousImageLSN"": 16 + ""previousImageLSN"": 16, + ""id"": ""1"", + ""partitionKey"": { + ""pk"": ""1"" + } }, ""previous"": { ""id"": ""1"", @@ -219,7 +223,11 @@ public void ValidateNSJAndSTJSerializationOfChangeFeedItemTest(bool propertyName ""lsn"": 376, ""operationType"": ""delete"", ""previousImageLSN"": 375, - ""timeToLiveExpired"": false + ""timeToLiveExpired"": false, + ""id"": ""1"", + ""partitionKey"": { + ""pk"": ""1"" + } }, ""previous"": { ""id"": ""1"", From aacd447996607222d31c0b4056549a584386d87c Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Wed, 11 Dec 2024 17:34:13 -0800 Subject: [PATCH 06/33] update ttl delete test --- .../BuilderWithCustomSerializerTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index c8c35b069b..fcbabdc291 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -390,7 +390,7 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Container leaseContainer = await database.CreateContainerIfNotExistsAsync(containerProperties: new ContainerProperties(id: "leases", partitionKeyPath: "/id")); ContainerInternal monitoredContainer = await this.CreateMonitoredContainer(ChangeFeedMode.AllVersionsAndDeletes, database); Exception exception = default; - int ttlInSeconds = 5; + int ttlInSeconds = 1; Stopwatch stopwatch = new(); ManualResetEvent allDocsProcessed = new ManualResetEvent(false); @@ -428,11 +428,11 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); - - // previous Assert.AreEqual(expected: "1", actual: change.Metadata.DeletedItemId.ToString()); change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); Assert.AreEqual(expected: "1", actual: partitionKey); + + // previous Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString()); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString()); From b2f0e2a47075891d22c791b90ab71c59add34fbe Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Mon, 13 Jan 2025 20:12:50 -0800 Subject: [PATCH 07/33] update change feed metadata contract --- .../FullFidelity/ChangeFeedMetadata.cs | 8 +-- .../FullFidelity/ChangeFeedMetadataFields.cs | 4 +- .../Converters/ChangeFeedMetadataConverter.cs | 52 ++++++++++++++----- .../CFP/AllVersionsAndDeletes/BuilderTests.cs | 14 +++-- .../BuilderWithCustomSerializerTests.cs | 24 ++++----- .../FeedToken/ChangeFeedIteratorCoreTests.cs | 4 +- 6 files changed, 64 insertions(+), 42 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index b507644d9f..5909fee2c2 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -59,13 +59,13 @@ class ChangeFeedMetadata /// /// The id of the previous item version. Used for delete operations only. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.DeletedItemId, NullValueHandling = NullValueHandling.Ignore)] - public string DeletedItemId { get; internal set; } + [JsonProperty(PropertyName = ChangeFeedMetadataFields.Id, NullValueHandling = NullValueHandling.Ignore)] + public string Id { get; internal set; } /// /// The partition key of the previous item version. Dictionary Key is the partition key property name and Dictionary Value is the partition key property value. Used for delete operations only. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.DeletedItemPartitionKey, NullValueHandling = NullValueHandling.Ignore)] - public Dictionary DeletedItemPartitionKey { get; internal set; } + [JsonProperty(PropertyName = ChangeFeedMetadataFields.PartitionKey, NullValueHandling = NullValueHandling.Ignore)] + public List<(string, object)> PartitionKey { get; internal set; } } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs index d45bf1bd44..e0005b8c05 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs @@ -11,7 +11,7 @@ internal class ChangeFeedMetadataFields public const string OperationType = "operationType"; public const string PreviousImageLSN = "previousImageLSN"; public const string TimeToLiveExpired = "timeToLiveExpired"; - public const string DeletedItemId = "id"; - public const string DeletedItemPartitionKey = "partitionKey"; + public const string Id = "id"; + public const string PartitionKey = "partitionKey"; } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs index 6112c7c22a..ac70541357 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs @@ -57,18 +57,18 @@ public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToCo { metadata.PreviousLsn = property.Value.GetInt64(); } - else if (property.NameEquals(ChangeFeedMetadataFields.DeletedItemId)) + else if (property.NameEquals(ChangeFeedMetadataFields.Id)) { - metadata.DeletedItemId = property.Value.GetString(); + metadata.Id = property.Value.GetString(); } - else if (property.NameEquals(ChangeFeedMetadataFields.DeletedItemPartitionKey)) + else if (property.NameEquals(ChangeFeedMetadataFields.PartitionKey)) { - Dictionary partitionKey = new Dictionary(); + List<(string, object)> partitionKey = new List<(string, object)>(); foreach (JsonProperty pk in property.Value.EnumerateObject()) { - partitionKey.Add(pk.Name, pk.Value.GetString()); + partitionKey.Add((pk.Name, pk.Value)); } - metadata.DeletedItemPartitionKey = partitionKey; + metadata.PartitionKey = partitionKey; } } @@ -90,18 +90,46 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json writer.WriteString(ChangeFeedMetadataFields.OperationType, value.OperationType.ToString()); writer.WriteNumber(ChangeFeedMetadataFields.PreviousImageLSN, value.PreviousLsn); - if (value.DeletedItemId != null) + if (value.Id != null) { - writer.WriteString(ChangeFeedMetadataFields.DeletedItemId, value.DeletedItemId); + writer.WriteString(ChangeFeedMetadataFields.Id, value.Id); } - if (value.DeletedItemPartitionKey != null) + if (value.PartitionKey != null) { - writer.WriteStartObject(ChangeFeedMetadataFields.DeletedItemPartitionKey); - foreach (KeyValuePair kvp in value.DeletedItemPartitionKey) + writer.WriteStartObject(ChangeFeedMetadataFields.PartitionKey); + + foreach ((string, object) pk in value.PartitionKey) { - writer.WriteString(kvp.Key, kvp.Value); + JsonElement pkValue = (JsonElement)pk.Item2; + + switch (pkValue.ValueKind) + { + case JsonValueKind.String: + writer.WriteString(pk.Item1, pkValue.GetString()); + break; + + case JsonValueKind.Number: + writer.WriteNumber(pk.Item1, pkValue.GetDouble()); + break; + + case JsonValueKind.True: + case JsonValueKind.False: + writer.WriteBoolean(pk.Item1, pkValue.GetBoolean()); + break; + + case JsonValueKind.Null: + writer.WriteNull(pk.Item1); + break; + + case JsonValueKind.Undefined: + break; + + default: + throw new JsonException(string.Format(CultureInfo.CurrentCulture, RMResources.JsonUnexpectedToken)); + } } + writer.WriteEndObject(); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs index c5c96b0a22..c1c03f1abf 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs @@ -84,9 +84,8 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); // previous - Assert.AreEqual(expected: "1", actual: change.Metadata.DeletedItemId.ToString()); - change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: change.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.FirstOrDefault().Item2); Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString()); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString()); @@ -176,8 +175,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() } else { - metadataId = change.Metadata.DeletedItemId.ToString(); - change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out metadataPk).ToString(); + metadataId = change.Metadata.Id.ToString(); + metadataPk = change.Metadata.PartitionKey.FirstOrDefault().Item2.ToString(); id = change.Previous.id.ToString(); pk = change.Previous.pk.ToString(); description = change.Previous.description.ToString(); @@ -218,9 +217,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() ChangeFeedItem deleteChange = docs.ElementAt(2); Assert.IsNull(deleteChange.Current.id); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.DeletedItemId.ToString()); - deleteChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.FirstOrDefault().Item2); Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); Assert.IsNotNull(deleteChange.Previous); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index fcbabdc291..5f3c7169cb 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -96,9 +96,8 @@ static void ValidateDeserialization(List> activitie Assert.IsTrue(deletedChange.Metadata.IsTimeToLiveExpired); Assert.IsNotNull(deletedChange.Previous); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: deletedChange.Previous.description); - Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.DeletedItemId.ToString()); - deletedChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.PartitionKey.FirstOrDefault().Item2.ToString()); Assert.AreEqual(expected: "1", actual: deletedChange.Previous.id); Assert.AreEqual(expected: 5, actual: deletedChange.Previous.ttl); } @@ -306,9 +305,8 @@ static void ValidateDeserialization(List> activitie Assert.IsFalse(deletedChange.Metadata.IsTimeToLiveExpired); Assert.IsNotNull(deletedChange.Previous); Assert.AreEqual(expected: "test after replace", actual: deletedChange.Previous.description); - Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.DeletedItemId.ToString()); - deletedChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.PartitionKey.FirstOrDefault().Item2.ToString()); Assert.AreEqual(expected: "1", actual: deletedChange.Previous.id); Assert.AreEqual(expected: 0, actual: deletedChange.Previous.ttl); } @@ -428,9 +426,8 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); - Assert.AreEqual(expected: "1", actual: change.Metadata.DeletedItemId.ToString()); - change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: change.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.FirstOrDefault().Item2); // previous Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); @@ -541,8 +538,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr } else { - metadataId = change.Metadata.DeletedItemId.ToString(); - change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out metadataPk).ToString(); + metadataId = change.Metadata.Id.ToString(); + metadataPk = change.Metadata.PartitionKey.FirstOrDefault().Item2.ToString(); id = change.Previous.id.ToString(); pk = change.Previous.pk.ToString(); description = change.Previous.description.ToString(); @@ -586,9 +583,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); Assert.IsNotNull(deleteChange.Previous); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.DeletedItemId.ToString()); - deleteChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.FirstOrDefault().Item2); Assert.AreEqual(expected: "1", actual: deleteChange.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: deleteChange.Previous.pk.ToString()); Assert.AreEqual(expected: "test after replace", actual: deleteChange.Previous.description.ToString()); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs index f4cc9c2ab5..ca679ab6a8 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs @@ -896,7 +896,7 @@ private async Task ValidateChangeFeedIteratorCore_WithQuery( foreach (ChangeFeedItem item in feedResponse) { - Assert.AreEqual(expected: "id3", actual: item.Metadata.DeletedItemId.ToString()); + Assert.AreEqual(expected: "id3", actual: item.Metadata.Id.ToString()); Assert.AreEqual("id3", item.Previous.Id); Assert.AreEqual(ChangeFeedOperationType.Delete, item.Metadata.OperationType); } @@ -1095,7 +1095,7 @@ public async Task ChangeFeedIteratorCore_FeedRange_VerifyingWireFormatTests() Assert.AreNotEqual(notExpected: default, actual: deleteOperation.Metadata.Lsn); Assert.AreNotEqual(notExpected: default, actual: deleteOperation.Metadata.PreviousLsn); Assert.IsNotNull(deleteOperation.Previous); - Assert.AreEqual(expected: id, actual: deleteOperation.Metadata.DeletedItemId.ToString()); + Assert.AreEqual(expected: id, actual: deleteOperation.Metadata.Id.ToString()); Assert.AreEqual(expected: id, actual: deleteOperation.Previous.Id); Assert.AreEqual(expected: "205 16th St NW", actual: deleteOperation.Previous.Line1); Assert.AreEqual(expected: "Atlanta", actual: deleteOperation.Previous.City); From 4129a953991f351c35c313255ace85867561cfc8 Mon Sep 17 00:00:00 2001 From: Dikshi Bahl Date: Fri, 9 May 2025 13:05:10 -0500 Subject: [PATCH 08/33] fix: adding newsoft converter to handle partitionKey --- .../FullFidelity/ChangeFeedMetadata.cs | 2 +- .../Converters/ChangeFeedMetadataConverter.cs | 39 ++-- .../ChangeFeedMetadataNewtonSoftConverter.cs | 207 ++++++++++++++++++ 3 files changed, 230 insertions(+), 18 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index 5909fee2c2..7a4184f5e8 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -6,7 +6,6 @@ namespace Microsoft.Azure.Cosmos { using System; using System.Collections.Generic; - using System.Text.Json; using Microsoft.Azure.Cosmos.Resource.FullFidelity; using Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters; using Microsoft.Azure.Documents; @@ -17,6 +16,7 @@ namespace Microsoft.Azure.Cosmos /// The metadata of a change feed resource with is initialized to . /// [System.Text.Json.Serialization.JsonConverter(typeof(ChangeFeedMetadataConverter))] + [JsonConverter(typeof(ChangeFeedMetadataNewtonSoftConverter))] #if PREVIEW public #else diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs index ac70541357..6975c65c97 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs @@ -66,7 +66,15 @@ public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToCo List<(string, object)> partitionKey = new List<(string, object)>(); foreach (JsonProperty pk in property.Value.EnumerateObject()) { - partitionKey.Add((pk.Name, pk.Value)); + object actualValue = pk.Value.ValueKind switch + { + JsonValueKind.String => pk.Value.GetString(), + JsonValueKind.Number => pk.Value.TryGetInt64(out long longValue) ? longValue : (object)pk.Value.GetDouble(), + JsonValueKind.True or JsonValueKind.False => pk.Value.GetBoolean(), + JsonValueKind.Null => null, + _ => throw new JsonException($"Unexpected JsonValueKind '{pk.Value.ValueKind}' for PartitionKey property."), + }; + partitionKey.Add((pk.Name, actualValue)); } metadata.PartitionKey = partitionKey; } @@ -99,37 +107,34 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json { writer.WriteStartObject(ChangeFeedMetadataFields.PartitionKey); - foreach ((string, object) pk in value.PartitionKey) + foreach ((string key, object objectValue) in value.PartitionKey) { - JsonElement pkValue = (JsonElement)pk.Item2; - - switch (pkValue.ValueKind) + switch (objectValue) { - case JsonValueKind.String: - writer.WriteString(pk.Item1, pkValue.GetString()); + case string stringValue: + writer.WriteString(key, stringValue); break; - case JsonValueKind.Number: - writer.WriteNumber(pk.Item1, pkValue.GetDouble()); + case long longValue: + writer.WriteNumber(key, longValue); break; - case JsonValueKind.True: - case JsonValueKind.False: - writer.WriteBoolean(pk.Item1, pkValue.GetBoolean()); + case double doubleValue: + writer.WriteNumber(key, doubleValue); break; - case JsonValueKind.Null: - writer.WriteNull(pk.Item1); + case bool boolValue: + writer.WriteBoolean(key, boolValue); break; - case JsonValueKind.Undefined: + case null: + writer.WriteNull(key); break; default: - throw new JsonException(string.Format(CultureInfo.CurrentCulture, RMResources.JsonUnexpectedToken)); + throw new JsonException($"Unexpected value type '{value.GetType()}' for PartitionKey property."); } } - writer.WriteEndObject(); } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs new file mode 100644 index 0000000000..b9c84da4c5 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs @@ -0,0 +1,207 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters +{ + using System; + using System.Collections.Generic; + using Microsoft.Azure.Cosmos.Spatial; + using Newtonsoft.Json; + + internal class ChangeFeedMetadataNewtonSoftConverter : JsonConverter + { + private readonly static DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + + /// + /// Writes the JSON representation of the object. + /// + /// The to write to. + /// The object value to write. + /// The calling serializer. + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + if (value is ChangeFeedMetadata metadata) + { + writer.WriteStartObject(); + + writer.WritePropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp); + serializer.Serialize(writer, metadata.ConflictResolutionTimestamp); + + writer.WritePropertyName(ChangeFeedMetadataFields.Lsn); + writer.WriteValue(metadata.Lsn); + + writer.WritePropertyName(ChangeFeedMetadataFields.OperationType); + serializer.Serialize(writer, metadata.OperationType); + + writer.WritePropertyName(ChangeFeedMetadataFields.PreviousImageLSN); + writer.WriteValue(metadata.PreviousLsn); + + writer.WritePropertyName(ChangeFeedMetadataFields.TimeToLiveExpired); + writer.WriteValue(metadata.IsTimeToLiveExpired); + + writer.WritePropertyName(ChangeFeedMetadataFields.Id); + writer.WriteValue(metadata.Id); + + writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); + writer.WriteStartArray(); + if (metadata.PartitionKey != null) + { + writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); + writer.WriteStartObject(); + + foreach ((string key, object objectValue) in metadata.PartitionKey) + { + writer.WritePropertyName(key); + + if (objectValue == null) + { + writer.WriteNull(); + } + else + { + switch (objectValue) + { + case string stringValue: + writer.WriteValue(stringValue); + break; + + case long longValue: + writer.WriteValue(longValue); + break; + + case double doubleValue: + writer.WriteValue(doubleValue); + break; + + case bool boolValue: + writer.WriteValue(boolValue); + break; + + default: + throw new JsonSerializationException($"Unexpected value type: {objectValue.GetType()} for PartitionKey property."); + } + } + } + + writer.WriteEndObject(); + } + + writer.WriteEndObject(); + + } + else + { + throw new JsonSerializationException($"Unexpected value when converting {nameof(ChangeFeedMetadata)}."); + } + } + + /// + /// Reads the JSON representation of the object. + /// + /// The to read from. + /// Type of the object. + /// The existing value of object being read. + /// The calling serializer. + /// The deserialized object. + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null) + { + return null; + } + + ChangeFeedMetadata metadata = new ChangeFeedMetadata(); + List<(string, object)> partitionKey = new List<(string, object)>(); + + reader.Read(); // StartObject + + while (reader.TokenType == JsonToken.PropertyName) + { + string propertyName = reader.Value.ToString(); + reader.Read(); // Move to property value + + switch (propertyName) + { + case ChangeFeedMetadataFields.ConflictResolutionTimestamp: + metadata.ConflictResolutionTimestamp = ChangeFeedMetadataNewtonSoftConverter.ToDateTimeFromUnixTimeInSeconds(Convert.ToInt64(reader.Value)); + break; + + case ChangeFeedMetadataFields.Lsn: + metadata.Lsn = reader.Value != null ? Convert.ToInt64(reader.Value) : 0; + break; + + case ChangeFeedMetadataFields.OperationType: + metadata.OperationType = serializer.Deserialize(reader); + break; + + case ChangeFeedMetadataFields.PreviousImageLSN: + metadata.PreviousLsn = reader.Value != null ? Convert.ToInt64(reader.Value) : 0; + break; + + case ChangeFeedMetadataFields.TimeToLiveExpired: + metadata.IsTimeToLiveExpired = reader.Value != null && Convert.ToBoolean(reader.Value); + break; + + case ChangeFeedMetadataFields.Id: + metadata.Id = reader.Value?.ToString(); + break; + + case ChangeFeedMetadataFields.PartitionKey: + if (reader.TokenType == JsonToken.StartObject) + { + reader.Read(); // Move to the first property in the object + + while (reader.TokenType == JsonToken.PropertyName) + { + string key = reader.Value.ToString(); + reader.Read(); // Move to the value of the property + + object value = reader.TokenType switch + { + JsonToken.String => reader.Value.ToString(), + JsonToken.Integer => Convert.ToInt64(reader.Value), + JsonToken.Float => Convert.ToDouble(reader.Value), + JsonToken.Boolean => Convert.ToBoolean(reader.Value), + JsonToken.Null => null, + _ => throw new JsonSerializationException($"Unexpected token type: {reader.TokenType} for PartitionKey property.") + }; + + partitionKey.Add((key, value)); + reader.Read(); // Move to the next property or EndObject + } + } + break; + + default: + reader.Skip(); + break; + } + + reader.Read(); // Move to next property or EndObject + } + + metadata.PartitionKey = partitionKey; + return metadata; + } + /// + /// Determines whether this instance can convert the specified object type. + /// + /// Type of the object. + /// true if this instance can convert the specified object type; otherwise, false. + public override bool CanConvert(Type objectType) + { + return objectType == typeof(ChangeFeedMetadata); + } + + private static long ToUnixTimeInSecondsFromDateTime(DateTime date) + { + return (long)(date - ChangeFeedMetadataNewtonSoftConverter.UnixEpoch).TotalSeconds; + } + + private static DateTime ToDateTimeFromUnixTimeInSeconds(long unixTimeInSeconds) + { + return ChangeFeedMetadataNewtonSoftConverter.UnixEpoch.AddSeconds(unixTimeInSeconds); + } + } +} From c41d59c0adf1876158aff36e361611e0902de794 Mon Sep 17 00:00:00 2001 From: Dikshi Bahl Date: Fri, 9 May 2025 15:00:57 -0500 Subject: [PATCH 09/33] fix:updated contract --- .../Contracts/DotNetPreviewSDKAPI.json | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index 2a48f3ea62..ae48b42248 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -137,19 +137,19 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.Dictionary`2[System.String,System.String] DeletedItemPartitionKey[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"partitionKey\")]": { - "Type": "Property", + "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", "Attributes": [ - "JsonPropertyAttribute" + "CompilerGeneratedAttribute" ], - "MethodInfo": "System.Collections.Generic.Dictionary`2[System.String,System.String] DeletedItemPartitionKey;CanRead:True;CanWrite:True;System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", + "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] PartitionKey[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"partitionKey\")]": { + "Type": "Property", "Attributes": [ - "CompilerGeneratedAttribute" + "JsonPropertyAttribute" ], - "MethodInfo": "System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] PartitionKey;CanRead:True;CanWrite:True;System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "System.DateTime ConflictResolutionTimestamp[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"crts\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Microsoft.Azure.Documents.UnixDateTimeConverter))]": { "Type": "Property", @@ -166,19 +166,19 @@ ], "MethodInfo": "System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.String DeletedItemId[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"id\")]": { - "Type": "Property", + "System.String get_Id()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", "Attributes": [ - "JsonPropertyAttribute" + "CompilerGeneratedAttribute" ], - "MethodInfo": "System.String DeletedItemId;CanRead:True;CanWrite:True;System.String get_DeletedItemId();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.String get_Id();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.String get_DeletedItemId()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", + "System.String Id[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"id\")]": { + "Type": "Property", "Attributes": [ - "CompilerGeneratedAttribute" + "JsonPropertyAttribute" ], - "MethodInfo": "System.String get_DeletedItemId();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.String Id;CanRead:True;CanWrite:True;System.String get_Id();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Void .ctor()": { "Type": "Constructor", From 5ddaec18588c67683b51ed322c33038fa9652a3b Mon Sep 17 00:00:00 2001 From: Dikshi Bahl Date: Sat, 10 May 2025 12:02:31 -0500 Subject: [PATCH 10/33] fix:refactoring --- .../ChangeFeedMetadataNewtonSoftConverter.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs index b9c84da4c5..e685099a9e 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs @@ -24,9 +24,8 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s if (value is ChangeFeedMetadata metadata) { writer.WriteStartObject(); - writer.WritePropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp); - serializer.Serialize(writer, metadata.ConflictResolutionTimestamp); + serializer.Serialize(writer, ChangeFeedMetadataNewtonSoftConverter.ToUnixTimeInSecondsFromDateTime(metadata.ConflictResolutionTimestamp)); writer.WritePropertyName(ChangeFeedMetadataFields.Lsn); writer.WriteValue(metadata.Lsn); @@ -42,13 +41,10 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s writer.WritePropertyName(ChangeFeedMetadataFields.Id); writer.WriteValue(metadata.Id); - - writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); - writer.WriteStartArray(); if (metadata.PartitionKey != null) { writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); - writer.WriteStartObject(); + writer.WriteStartObject(); foreach ((string key, object objectValue) in metadata.PartitionKey) { @@ -84,11 +80,10 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s } } - writer.WriteEndObject(); + writer.WriteEndObject(); // End PartitionKey object } writer.WriteEndObject(); - } else { From 31bc8b82c29e7c19af6512b5a17f8370d9e51636 Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Tue, 27 May 2025 13:25:16 -0500 Subject: [PATCH 11/33] fix:addressing comments --- .../src/Resource/FullFidelity/ChangeFeedMetadata.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index 7a4184f5e8..a6de520a25 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -57,13 +57,15 @@ class ChangeFeedMetadata public bool IsTimeToLiveExpired { get; internal set; } /// - /// The id of the previous item version. Used for delete operations only. + /// Used for delete operations only. + /// The id of the previous item version. /// [JsonProperty(PropertyName = ChangeFeedMetadataFields.Id, NullValueHandling = NullValueHandling.Ignore)] public string Id { get; internal set; } /// - /// The partition key of the previous item version. Dictionary Key is the partition key property name and Dictionary Value is the partition key property value. Used for delete operations only. + /// Used for delete operations only. + /// The partition key of the previous item version. string is the partition key property name and object is the partition key property value. All levels of hierarchy will be represented in order if a HPK is used. /// [JsonProperty(PropertyName = ChangeFeedMetadataFields.PartitionKey, NullValueHandling = NullValueHandling.Ignore)] public List<(string, object)> PartitionKey { get; internal set; } From d0d775dc62abfd91c4f74e1c0a7ae8e5c612700b Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 28 May 2025 11:23:33 -0500 Subject: [PATCH 12/33] Update Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs Co-authored-by: Kiran Kumar Kolli --- .../src/Resource/FullFidelity/ChangeFeedMetadata.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index a6de520a25..f27f08a405 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -57,7 +57,7 @@ class ChangeFeedMetadata public bool IsTimeToLiveExpired { get; internal set; } /// - /// Used for delete operations only. + /// Applicable for delete operations only, otherwise null. /// The id of the previous item version. /// [JsonProperty(PropertyName = ChangeFeedMetadataFields.Id, NullValueHandling = NullValueHandling.Ignore)] From 59f53a5c3b303b5ef4296e5e042c0ac35981e8dc Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 28 May 2025 11:23:46 -0500 Subject: [PATCH 13/33] Update Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs Co-authored-by: Kiran Kumar Kolli --- .../src/Resource/FullFidelity/ChangeFeedMetadata.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index f27f08a405..3b1ab146d0 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -64,7 +64,7 @@ class ChangeFeedMetadata public string Id { get; internal set; } /// - /// Used for delete operations only. + /// Applicable for delete operations only, otherwise null. /// The partition key of the previous item version. string is the partition key property name and object is the partition key property value. All levels of hierarchy will be represented in order if a HPK is used. /// [JsonProperty(PropertyName = ChangeFeedMetadataFields.PartitionKey, NullValueHandling = NullValueHandling.Ignore)] From 2870d206aa158f62edbb5848b6ddbdd4ca21eba5 Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 28 May 2025 15:56:34 -0500 Subject: [PATCH 14/33] fix: addressing comments --- .../src/Resource/FullFidelity/ChangeFeedMetadata.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index 3b1ab146d0..7c6d982475 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -27,47 +27,38 @@ class ChangeFeedMetadata /// /// The change's conflict resolution timestamp. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.ConflictResolutionTimestamp, NullValueHandling = NullValueHandling.Ignore)] - [JsonConverter(typeof(UnixDateTimeConverter))] public DateTime ConflictResolutionTimestamp { get; internal set; } /// /// The current change's logical sequence number. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.Lsn, NullValueHandling = NullValueHandling.Ignore)] public long Lsn { get; internal set; } /// /// The change's feed operation type . /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.OperationType, NullValueHandling = NullValueHandling.Ignore)] - [JsonConverter(typeof(StringEnumConverter))] public ChangeFeedOperationType OperationType { get; internal set; } /// /// The previous change's logical sequence number. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.PreviousImageLSN, NullValueHandling = NullValueHandling.Ignore)] public long PreviousLsn { get; internal set; } /// /// Used to distinguish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents). /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.TimeToLiveExpired, NullValueHandling = NullValueHandling.Ignore)] public bool IsTimeToLiveExpired { get; internal set; } /// /// Applicable for delete operations only, otherwise null. /// The id of the previous item version. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.Id, NullValueHandling = NullValueHandling.Ignore)] public string Id { get; internal set; } /// /// Applicable for delete operations only, otherwise null. /// The partition key of the previous item version. string is the partition key property name and object is the partition key property value. All levels of hierarchy will be represented in order if a HPK is used. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.PartitionKey, NullValueHandling = NullValueHandling.Ignore)] public List<(string, object)> PartitionKey { get; internal set; } } } From 079e9aa0ee91e3b18671cdad12df4c8cb7b58bc3 Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Sun, 1 Jun 2025 13:11:35 -0500 Subject: [PATCH 15/33] fix: addressing comments --- .../Converters/ChangeFeedMetadataNewtonSoftConverter.cs | 4 ++-- .../CFP/AllVersionsAndDeletes/BuilderTests.cs | 3 ++- .../AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs index e685099a9e..ac5e8f2a7a 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs @@ -107,7 +107,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist } ChangeFeedMetadata metadata = new ChangeFeedMetadata(); - List<(string, object)> partitionKey = new List<(string, object)>(); + List<(string, object)> partitionKey = null; reader.Read(); // StartObject @@ -145,8 +145,8 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist case ChangeFeedMetadataFields.PartitionKey: if (reader.TokenType == JsonToken.StartObject) { + partitionKey ??= new List<(string, object)>(); reader.Read(); // Move to the first property in the object - while (reader.TokenType == JsonToken.PropertyName) { string key = reader.Value.ToString(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs index c1c03f1abf..914e256220 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs @@ -69,7 +69,8 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); Assert.IsFalse(change.Metadata.IsTimeToLiveExpired); - + Assert.IsNull(change.Metadata.Id); + Assert.IsNull(change.Metadata.PartitionKey); // previous Assert.IsNull(change.Previous); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 5f3c7169cb..061ffbfcd0 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -413,6 +413,8 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); Assert.IsFalse(change.Metadata.IsTimeToLiveExpired); + Assert.IsNull(change.Metadata.Id); + Assert.IsNull(change.Metadata.PartitionKey); // previous Assert.IsNull(change.Previous); From aa60a72fa69c9ba815f30250485a6472e66eb52b Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 4 Jun 2025 12:34:55 -0500 Subject: [PATCH 16/33] fix: refactoring --- .../Converters/ChangeFeedMetadataNewtonSoftConverter.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs index ac5e8f2a7a..911cd98589 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs @@ -165,6 +165,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist partitionKey.Add((key, value)); reader.Read(); // Move to the next property or EndObject } + metadata.PartitionKey = partitionKey; } break; @@ -175,8 +176,6 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist reader.Read(); // Move to next property or EndObject } - - metadata.PartitionKey = partitionKey; return metadata; } /// From 2658b33f134ee9eff006883cd80ed85de567967b Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 17 Sep 2025 16:51:45 -0500 Subject: [PATCH 17/33] updating contract file --- .../Contracts/DotNetPreviewSDKAPI.json | 44 ++++++------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index ae48b42248..6ae5f8fa96 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -87,11 +87,9 @@ ], "MethodInfo": "Boolean get_IsTimeToLiveExpired();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Boolean IsTimeToLiveExpired[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"timeToLiveExpired\")]": { + "Boolean IsTimeToLiveExpired": { "Type": "Property", - "Attributes": [ - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "Boolean IsTimeToLiveExpired;CanRead:True;CanWrite:True;Boolean get_IsTimeToLiveExpired();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Int64 get_Lsn()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { @@ -108,18 +106,14 @@ ], "MethodInfo": "Int64 get_PreviousLsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Int64 Lsn[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"lsn\")]": { + "Int64 Lsn": { "Type": "Property", - "Attributes": [ - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "Int64 Lsn;CanRead:True;CanWrite:True;Int64 get_Lsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Int64 PreviousLsn[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"previousImageLSN\")]": { + "Int64 PreviousLsn": { "Type": "Property", - "Attributes": [ - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "Int64 PreviousLsn;CanRead:True;CanWrite:True;Int64 get_PreviousLsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { @@ -129,12 +123,9 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"operationType\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Newtonsoft.Json.Converters.StringEnumConverter))]": { + "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType": { "Type": "Property", - "Attributes": [ - "JsonConverterAttribute", - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { @@ -144,19 +135,14 @@ ], "MethodInfo": "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] PartitionKey[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"partitionKey\")]": { + "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] PartitionKey": { "Type": "Property", - "Attributes": [ - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] PartitionKey;CanRead:True;CanWrite:True;System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.DateTime ConflictResolutionTimestamp[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"crts\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Microsoft.Azure.Documents.UnixDateTimeConverter))]": { + "System.DateTime ConflictResolutionTimestamp": { "Type": "Property", - "Attributes": [ - "JsonConverterAttribute", - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "System.DateTime ConflictResolutionTimestamp;CanRead:True;CanWrite:True;System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "System.DateTime get_ConflictResolutionTimestamp()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { @@ -173,11 +159,9 @@ ], "MethodInfo": "System.String get_Id();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.String Id[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"id\")]": { + "System.String Id": { "Type": "Property", - "Attributes": [ - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "System.String Id;CanRead:True;CanWrite:True;System.String get_Id();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Void .ctor()": { From 8694811cc668898d252019daa4ed117f6194442d Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 29 Oct 2025 14:34:41 -0500 Subject: [PATCH 18/33] enhanced logging --- .../Converters/ChangeFeedMetadataConverter.cs | 13 +++++++++++-- .../ChangeFeedMetadataNewtonSoftConverter.cs | 4 ++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs index 6975c65c97..3ef24a7213 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs @@ -72,13 +72,22 @@ public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToCo JsonValueKind.Number => pk.Value.TryGetInt64(out long longValue) ? longValue : (object)pk.Value.GetDouble(), JsonValueKind.True or JsonValueKind.False => pk.Value.GetBoolean(), JsonValueKind.Null => null, - _ => throw new JsonException($"Unexpected JsonValueKind '{pk.Value.ValueKind}' for PartitionKey property."), + _ => throw new JsonException($"Unexpected JsonValueKind '{pk.Value.ValueKind}' for PartitionKey property '{pk.Name}'."), }; partitionKey.Add((pk.Name, actualValue)); } metadata.PartitionKey = partitionKey; } } + + // validate delete operation requirements + if (metadata.OperationType == ChangeFeedOperationType.Delete) + { + if (metadata.Id == null || metadata.PartitionKey == null) + { + throw new JsonException("Delete operations require both 'id' and 'partitionKey' to be present."); + } + } return metadata; } @@ -132,7 +141,7 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json break; default: - throw new JsonException($"Unexpected value type '{value.GetType()}' for PartitionKey property."); + throw new JsonException($"Unexpected value type '{objectValue.GetType()}' for PartitionKey property '{key}'."); } } writer.WriteEndObject(); diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs index 911cd98589..5ab7070192 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs @@ -75,7 +75,7 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s break; default: - throw new JsonSerializationException($"Unexpected value type: {objectValue.GetType()} for PartitionKey property."); + throw new JsonSerializationException($"Unexpected value type '{objectValue.GetType()}' for PartitionKey property '{key}'."); } } } @@ -87,7 +87,7 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s } else { - throw new JsonSerializationException($"Unexpected value when converting {nameof(ChangeFeedMetadata)}."); + throw new JsonSerializationException($"Unexpected value '{value}' of type '{value?.GetType()}' when converting {nameof(ChangeFeedMetadata)}."); } } From b64ffa15beaa0c6da2f788282026ca69f23fee95 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Fri, 31 Oct 2025 10:43:14 -0700 Subject: [PATCH 19/33] Simplify converter --- Microsoft.Azure.Cosmos.sln | 16 +++- .../FullFidelity/ChangeFeedMetadata.cs | 2 +- .../Converters/ChangeFeedMetadataConverter.cs | 71 +++------------ .../ChangeFeedMetadataNewtonSoftConverter.cs | 90 ++++--------------- .../CFP/AllVersionsAndDeletes/BuilderTests.cs | 20 ++--- .../BuilderWithCustomSerializerTests.cs | 41 ++++----- 6 files changed, 64 insertions(+), 176 deletions(-) diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index cec2d9a5f4..5dd1a4d66a 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29123.88 +# Visual Studio Version 17 +VisualStudioVersion = 17.14.36623.8 d17.14 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject @@ -181,6 +181,18 @@ Global {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.Build.0 = Release|Any CPU {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.ActiveCfg = Release|Any CPU {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.Build.0 = Release|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Cover|Any CPU.ActiveCfg = Debug|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Cover|Any CPU.Build.0 = Debug|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Cover|x64.ActiveCfg = Debug|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Cover|x64.Build.0 = Debug|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Debug|x64.ActiveCfg = Debug|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Debug|x64.Build.0 = Debug|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Release|Any CPU.Build.0 = Release|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Release|x64.ActiveCfg = Release|Any CPU + {D744906A-1091-403F-B0B6-794DE045169A}.Release|x64.Build.0 = Release|Any CPU {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|Any CPU.ActiveCfg = Debug|Any CPU {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|Any CPU.Build.0 = Debug|Any CPU {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|x64.ActiveCfg = Debug|Any CPU diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index 7c6d982475..6f9cfb3d3a 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -59,6 +59,6 @@ class ChangeFeedMetadata /// Applicable for delete operations only, otherwise null. /// The partition key of the previous item version. string is the partition key property name and object is the partition key property value. All levels of hierarchy will be represented in order if a HPK is used. /// - public List<(string, object)> PartitionKey { get; internal set; } + public Dictionary PartitionKey { get; internal set; } } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs index 3ef24a7213..e75eb0acbb 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs @@ -17,7 +17,7 @@ namespace Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters /// internal class ChangeFeedMetadataConverter : JsonConverter { - private readonly static DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { @@ -43,7 +43,9 @@ public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToCo } else if (property.NameEquals(ChangeFeedMetadataFields.ConflictResolutionTimestamp)) { - metadata.ConflictResolutionTimestamp = ChangeFeedMetadataConverter.ToDateTimeFromUnixTimeInSeconds(property.Value.GetInt64()); + // Read the Unix timestamp and convert to DateTime + long unixTimeInSeconds = property.Value.GetInt64(); + metadata.ConflictResolutionTimestamp = UnixEpoch.AddSeconds(unixTimeInSeconds); } else if (property.NameEquals(ChangeFeedMetadataFields.OperationType)) { @@ -63,20 +65,8 @@ public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToCo } else if (property.NameEquals(ChangeFeedMetadataFields.PartitionKey)) { - List<(string, object)> partitionKey = new List<(string, object)>(); - foreach (JsonProperty pk in property.Value.EnumerateObject()) - { - object actualValue = pk.Value.ValueKind switch - { - JsonValueKind.String => pk.Value.GetString(), - JsonValueKind.Number => pk.Value.TryGetInt64(out long longValue) ? longValue : (object)pk.Value.GetDouble(), - JsonValueKind.True or JsonValueKind.False => pk.Value.GetBoolean(), - JsonValueKind.Null => null, - _ => throw new JsonException($"Unexpected JsonValueKind '{pk.Value.ValueKind}' for PartitionKey property '{pk.Name}'."), - }; - partitionKey.Add((pk.Name, actualValue)); - } - metadata.PartitionKey = partitionKey; + // Dictionary is handled by default System.Text.Json deserialization + metadata.PartitionKey = JsonSerializer.Deserialize>(property.Value.GetRawText(), options); } } @@ -101,7 +91,10 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json writer.WriteStartObject(); - writer.WriteNumber(ChangeFeedMetadataFields.ConflictResolutionTimestamp, ChangeFeedMetadataConverter.ToUnixTimeInSecondsFromDateTime(value.ConflictResolutionTimestamp)); + writer.WritePropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp); + long unixTimeInSeconds = (long)(value.ConflictResolutionTimestamp - UnixEpoch).TotalSeconds; + writer.WriteNumberValue(unixTimeInSeconds); + writer.WriteBoolean(ChangeFeedMetadataFields.TimeToLiveExpired, value.IsTimeToLiveExpired); writer.WriteNumber(ChangeFeedMetadataFields.Lsn, value.Lsn); writer.WriteString(ChangeFeedMetadataFields.OperationType, value.OperationType.ToString()); @@ -114,50 +107,12 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json if (value.PartitionKey != null) { - writer.WriteStartObject(ChangeFeedMetadataFields.PartitionKey); - - foreach ((string key, object objectValue) in value.PartitionKey) - { - switch (objectValue) - { - case string stringValue: - writer.WriteString(key, stringValue); - break; - - case long longValue: - writer.WriteNumber(key, longValue); - break; - - case double doubleValue: - writer.WriteNumber(key, doubleValue); - break; - - case bool boolValue: - writer.WriteBoolean(key, boolValue); - break; - - case null: - writer.WriteNull(key); - break; - - default: - throw new JsonException($"Unexpected value type '{objectValue.GetType()}' for PartitionKey property '{key}'."); - } - } - writer.WriteEndObject(); + // Dictionary is handled by default System.Text.Json serialization + writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); + JsonSerializer.Serialize(writer, value.PartitionKey, options); } writer.WriteEndObject(); } - - private static long ToUnixTimeInSecondsFromDateTime(DateTime date) - { - return (long)(date - ChangeFeedMetadataConverter.UnixEpoch).TotalSeconds; - } - - private static DateTime ToDateTimeFromUnixTimeInSeconds(long unixTimeInSeconds) - { - return ChangeFeedMetadataConverter.UnixEpoch.AddSeconds(unixTimeInSeconds); - } } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs index 5ab7070192..0c7f8e0b48 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs @@ -11,7 +11,7 @@ namespace Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters internal class ChangeFeedMetadataNewtonSoftConverter : JsonConverter { - private readonly static DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); /// /// Writes the JSON representation of the object. @@ -24,8 +24,10 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s if (value is ChangeFeedMetadata metadata) { writer.WriteStartObject(); + writer.WritePropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp); - serializer.Serialize(writer, ChangeFeedMetadataNewtonSoftConverter.ToUnixTimeInSecondsFromDateTime(metadata.ConflictResolutionTimestamp)); + long unixTimeInSeconds = (long)(metadata.ConflictResolutionTimestamp - UnixEpoch).TotalSeconds; + writer.WriteValue(unixTimeInSeconds); writer.WritePropertyName(ChangeFeedMetadataFields.Lsn); writer.WriteValue(metadata.Lsn); @@ -43,44 +45,9 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s writer.WriteValue(metadata.Id); if (metadata.PartitionKey != null) { + // Dictionary is handled by default Newtonsoft.Json serialization writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); - writer.WriteStartObject(); - - foreach ((string key, object objectValue) in metadata.PartitionKey) - { - writer.WritePropertyName(key); - - if (objectValue == null) - { - writer.WriteNull(); - } - else - { - switch (objectValue) - { - case string stringValue: - writer.WriteValue(stringValue); - break; - - case long longValue: - writer.WriteValue(longValue); - break; - - case double doubleValue: - writer.WriteValue(doubleValue); - break; - - case bool boolValue: - writer.WriteValue(boolValue); - break; - - default: - throw new JsonSerializationException($"Unexpected value type '{objectValue.GetType()}' for PartitionKey property '{key}'."); - } - } - } - - writer.WriteEndObject(); // End PartitionKey object + serializer.Serialize(writer, metadata.PartitionKey); } writer.WriteEndObject(); @@ -107,7 +74,6 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist } ChangeFeedMetadata metadata = new ChangeFeedMetadata(); - List<(string, object)> partitionKey = null; reader.Read(); // StartObject @@ -119,7 +85,12 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist switch (propertyName) { case ChangeFeedMetadataFields.ConflictResolutionTimestamp: - metadata.ConflictResolutionTimestamp = ChangeFeedMetadataNewtonSoftConverter.ToDateTimeFromUnixTimeInSeconds(Convert.ToInt64(reader.Value)); + // Read the Unix timestamp and convert to DateTime + if (reader.Value != null) + { + long unixTimeInSeconds = Convert.ToInt64(reader.Value); + metadata.ConflictResolutionTimestamp = UnixEpoch.AddSeconds(unixTimeInSeconds); + } break; case ChangeFeedMetadataFields.Lsn: @@ -143,30 +114,8 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist break; case ChangeFeedMetadataFields.PartitionKey: - if (reader.TokenType == JsonToken.StartObject) - { - partitionKey ??= new List<(string, object)>(); - reader.Read(); // Move to the first property in the object - while (reader.TokenType == JsonToken.PropertyName) - { - string key = reader.Value.ToString(); - reader.Read(); // Move to the value of the property - - object value = reader.TokenType switch - { - JsonToken.String => reader.Value.ToString(), - JsonToken.Integer => Convert.ToInt64(reader.Value), - JsonToken.Float => Convert.ToDouble(reader.Value), - JsonToken.Boolean => Convert.ToBoolean(reader.Value), - JsonToken.Null => null, - _ => throw new JsonSerializationException($"Unexpected token type: {reader.TokenType} for PartitionKey property.") - }; - - partitionKey.Add((key, value)); - reader.Read(); // Move to the next property or EndObject - } - metadata.PartitionKey = partitionKey; - } + // Dictionary is handled by default Newtonsoft.Json deserialization + metadata.PartitionKey = serializer.Deserialize>(reader); break; default: @@ -176,6 +125,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist reader.Read(); // Move to next property or EndObject } + return metadata; } /// @@ -187,15 +137,5 @@ public override bool CanConvert(Type objectType) { return objectType == typeof(ChangeFeedMetadata); } - - private static long ToUnixTimeInSecondsFromDateTime(DateTime date) - { - return (long)(date - ChangeFeedMetadataNewtonSoftConverter.UnixEpoch).TotalSeconds; - } - - private static DateTime ToDateTimeFromUnixTimeInSeconds(long unixTimeInSeconds) - { - return ChangeFeedMetadataNewtonSoftConverter.UnixEpoch.AddSeconds(unixTimeInSeconds); - } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs index 914e256220..da3b29057a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs @@ -86,11 +86,8 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA // previous Assert.AreEqual(expected: "1", actual: change.Metadata.Id.ToString()); - Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.FirstOrDefault().Item2); - Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); - Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString()); - Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString()); - Assert.AreEqual(expected: ttlInSeconds, actual: change.Previous.ttl); + Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.Values.FirstOrDefault()); + Assert.IsNull(change.Previous); // stop after reading delete since it is the last document in feed. stopwatch.Stop(); @@ -177,10 +174,7 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() else { metadataId = change.Metadata.Id.ToString(); - metadataPk = change.Metadata.PartitionKey.FirstOrDefault().Item2.ToString(); - id = change.Previous.id.ToString(); - pk = change.Previous.pk.ToString(); - description = change.Previous.description.ToString(); + metadataPk = change.Metadata.PartitionKey.Values.FirstOrDefault().ToString(); } ChangeFeedOperationType operationType = change.Metadata.OperationType; @@ -219,13 +213,9 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() ChangeFeedItem deleteChange = docs.ElementAt(2); Assert.IsNull(deleteChange.Current.id); Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.FirstOrDefault().Item2); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.Values.FirstOrDefault()); Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); - Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); - Assert.IsNotNull(deleteChange.Previous); - Assert.AreEqual(expected: "1", actual: deleteChange.Previous.id.ToString()); - Assert.AreEqual(expected: "1", actual: deleteChange.Previous.pk.ToString()); - Assert.AreEqual(expected: "test after replace", actual: deleteChange.Previous.description.ToString()); + Assert.IsNull(deleteChange.Previous); Assert.IsTrue(condition: createChange.Metadata.ConflictResolutionTimestamp < replaceChange.Metadata.ConflictResolutionTimestamp, message: "The create operation must happen before the replace operation."); Assert.IsTrue(condition: replaceChange.Metadata.ConflictResolutionTimestamp < deleteChange.Metadata.ConflictResolutionTimestamp, message: "The replace operation must happen before the delete operation."); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 061ffbfcd0..7ab92ebadb 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -97,7 +97,7 @@ static void ValidateDeserialization(List> activitie Assert.IsNotNull(deletedChange.Previous); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: deletedChange.Previous.description); Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.Id.ToString()); - Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.PartitionKey.FirstOrDefault().Item2.ToString()); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.PartitionKey.Values.FirstOrDefault().ToString()); Assert.AreEqual(expected: "1", actual: deletedChange.Previous.id); Assert.AreEqual(expected: 5, actual: deletedChange.Previous.ttl); } @@ -306,7 +306,7 @@ static void ValidateDeserialization(List> activitie Assert.IsNotNull(deletedChange.Previous); Assert.AreEqual(expected: "test after replace", actual: deletedChange.Previous.description); Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.Id.ToString()); - Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.PartitionKey.FirstOrDefault().Item2.ToString()); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.PartitionKey.Values.FirstOrDefault().ToString()); Assert.AreEqual(expected: "1", actual: deletedChange.Previous.id); Assert.AreEqual(expected: 0, actual: deletedChange.Previous.ttl); } @@ -395,12 +395,12 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA ChangeFeedProcessor processor = monitoredContainer .GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(processorName: "processor", onChangesDelegate: (ChangeFeedProcessorContext context, IReadOnlyCollection> docs, CancellationToken token) => { - // NOTE(philipthomas-MSFT): Please allow these Logger.LogLine because TTL on items will purge at random times so I am using this to test when ran locally using emulator. + // NOTE(philipthomas-MSFT): Please allow these Logger.LogLine because TTL on items will purge at random times so I am using this to test when ran locally using emulator. - Logger.LogLine($"@ {DateTime.Now}, {nameof(stopwatch)} -> CFP AVAD took '{stopwatch.ElapsedMilliseconds}' to read document CRUD in feed."); + Logger.LogLine($"@ {DateTime.Now}, {nameof(stopwatch)} -> CFP AVAD took '{stopwatch.ElapsedMilliseconds}' to read document CRUD in feed."); - foreach (ChangeFeedItem change in docs) - { + foreach (ChangeFeedItem change in docs) + { if (change.Metadata.OperationType == ChangeFeedOperationType.Create) { // current @@ -429,23 +429,20 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); Assert.AreEqual(expected: "1", actual: change.Metadata.Id.ToString()); - Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.FirstOrDefault().Item2); + Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.Values.FirstOrDefault()); // previous - Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); - Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString()); - Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString()); - Assert.AreEqual(expected: ttlInSeconds, actual: change.Previous.ttl); + Assert.IsNull(change.Previous); // stop after reading delete since it is the last document in feed. stopwatch.Stop(); allDocsProcessed.Set(); } - else - { - Assert.Fail("Invalid operation."); + else + { + Assert.Fail("Invalid operation."); + } } - } return Task.CompletedTask; }) @@ -541,10 +538,7 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr else { metadataId = change.Metadata.Id.ToString(); - metadataPk = change.Metadata.PartitionKey.FirstOrDefault().Item2.ToString(); - id = change.Previous.id.ToString(); - pk = change.Previous.pk.ToString(); - description = change.Previous.description.ToString(); + metadataPk = change.Metadata.PartitionKey.Values.FirstOrDefault().ToString(); } ChangeFeedOperationType operationType = change.Metadata.OperationType; @@ -584,13 +578,10 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr Assert.IsNull(deleteChange.Current.id); Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); - Assert.IsNotNull(deleteChange.Previous); + Assert.IsNull(deleteChange.Previous); Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.FirstOrDefault().Item2); - Assert.AreEqual(expected: "1", actual: deleteChange.Previous.id.ToString()); - Assert.AreEqual(expected: "1", actual: deleteChange.Previous.pk.ToString()); - Assert.AreEqual(expected: "test after replace", actual: deleteChange.Previous.description.ToString()); - + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.Values.FirstOrDefault().ToString()); + Assert.IsTrue(condition: createChange.Metadata.ConflictResolutionTimestamp < replaceChange.Metadata.ConflictResolutionTimestamp, message: "The create operation must happen before the replace operation."); Assert.IsTrue(condition: replaceChange.Metadata.ConflictResolutionTimestamp < deleteChange.Metadata.ConflictResolutionTimestamp, message: "The replace operation must happen before the delete operation."); Assert.IsTrue(condition: createChange.Metadata.Lsn < replaceChange.Metadata.Lsn, message: "The create operation must happen before the replace operation."); From 3810bab2b65a1728a8604f015a223c39d0c1c389 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Fri, 31 Oct 2025 12:08:07 -0700 Subject: [PATCH 20/33] Move to annotation converters --- .../FullFidelity/ChangeFeedMetadata.cs | 33 +++- .../Converters/ChangeFeedMetadataConverter.cs | 118 --------------- .../ChangeFeedMetadataNewtonSoftConverter.cs | 141 ------------------ .../BuilderWithCustomSerializerTests.cs | 14 +- 4 files changed, 37 insertions(+), 269 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs delete mode 100644 Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index 6f9cfb3d3a..6e509e1c61 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -7,7 +7,6 @@ namespace Microsoft.Azure.Cosmos using System; using System.Collections.Generic; using Microsoft.Azure.Cosmos.Resource.FullFidelity; - using Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters; using Microsoft.Azure.Documents; using Newtonsoft.Json; using Newtonsoft.Json.Converters; @@ -15,8 +14,6 @@ namespace Microsoft.Azure.Cosmos /// /// The metadata of a change feed resource with is initialized to . /// - [System.Text.Json.Serialization.JsonConverter(typeof(ChangeFeedMetadataConverter))] - [JsonConverter(typeof(ChangeFeedMetadataNewtonSoftConverter))] #if PREVIEW public #else @@ -24,41 +21,69 @@ namespace Microsoft.Azure.Cosmos #endif class ChangeFeedMetadata { + private readonly static DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + /// /// The change's conflict resolution timestamp. /// - public DateTime ConflictResolutionTimestamp { get; internal set; } + [System.Text.Json.Serialization.JsonIgnore] + public DateTime ConflictResolutionTimestamp => UnixEpoch.AddSeconds(this.ConflictResolutionTimestampInSeconds.Value); + + [System.Text.Json.Serialization.JsonInclude] + [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp)] + [JsonProperty(PropertyName = ChangeFeedMetadataFields.ConflictResolutionTimestamp, NullValueHandling = NullValueHandling.Ignore)] + internal double? ConflictResolutionTimestampInSeconds { get; set; } /// /// The current change's logical sequence number. /// + [System.Text.Json.Serialization.JsonInclude] + [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.Lsn)] + [JsonProperty(PropertyName = ChangeFeedMetadataFields.Lsn, NullValueHandling = NullValueHandling.Ignore)] public long Lsn { get; internal set; } /// /// The change's feed operation type . /// + [Newtonsoft.Json.JsonConverter(typeof(StringEnumConverter))] + [System.Text.Json.Serialization.JsonInclude] + [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.OperationType)] + [System.Text.Json.Serialization.JsonConverter(typeof(System.Text.Json.Serialization.JsonStringEnumConverter))] + [JsonProperty(PropertyName = ChangeFeedMetadataFields.OperationType, NullValueHandling = NullValueHandling.Ignore)] public ChangeFeedOperationType OperationType { get; internal set; } /// /// The previous change's logical sequence number. /// + [System.Text.Json.Serialization.JsonInclude] + [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.PreviousImageLSN)] + [JsonProperty(PropertyName = ChangeFeedMetadataFields.PreviousImageLSN, NullValueHandling = NullValueHandling.Ignore)] public long PreviousLsn { get; internal set; } /// /// Used to distinguish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents). /// + [System.Text.Json.Serialization.JsonInclude] + [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.TimeToLiveExpired)] + [JsonProperty(PropertyName = ChangeFeedMetadataFields.TimeToLiveExpired, NullValueHandling = NullValueHandling.Ignore)] public bool IsTimeToLiveExpired { get; internal set; } /// /// Applicable for delete operations only, otherwise null. /// The id of the previous item version. /// + [System.Text.Json.Serialization.JsonInclude] + [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.Id)] + [JsonProperty(PropertyName = ChangeFeedMetadataFields.Id, NullValueHandling = NullValueHandling.Ignore)] public string Id { get; internal set; } /// /// Applicable for delete operations only, otherwise null. /// The partition key of the previous item version. string is the partition key property name and object is the partition key property value. All levels of hierarchy will be represented in order if a HPK is used. /// + [System.Text.Json.Serialization.JsonInclude] + [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.PartitionKey)] + [JsonProperty(PropertyName = ChangeFeedMetadataFields.PartitionKey, NullValueHandling = NullValueHandling.Ignore)] public Dictionary PartitionKey { get; internal set; } } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs deleted file mode 100644 index e75eb0acbb..0000000000 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ /dev/null @@ -1,118 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters -{ - using System; - using System.Collections.Generic; - using System.Globalization; - using System.Text.Json; - using System.Text.Json.Serialization; - using Microsoft.Azure.Cosmos.Resource.FullFidelity; - using Microsoft.Azure.Documents; - - /// - /// Converter used to support System.Text.Json de/serialization of type ChangeFeedMetadata/>. - /// - internal class ChangeFeedMetadataConverter : JsonConverter - { - private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); - - public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - if (reader.TokenType == JsonTokenType.Null) - { - return null; - } - - if (reader.TokenType != JsonTokenType.StartObject) - { - throw new JsonException(string.Format(CultureInfo.CurrentCulture, RMResources.JsonUnexpectedToken)); - } - - JsonElement element = JsonDocument.ParseValue(ref reader).RootElement; - - ChangeFeedMetadata metadata = new (); - - foreach (JsonProperty property in element.EnumerateObject()) - { - if (property.NameEquals(ChangeFeedMetadataFields.Lsn)) - { - metadata.Lsn = property.Value.GetInt64(); - } - else if (property.NameEquals(ChangeFeedMetadataFields.ConflictResolutionTimestamp)) - { - // Read the Unix timestamp and convert to DateTime - long unixTimeInSeconds = property.Value.GetInt64(); - metadata.ConflictResolutionTimestamp = UnixEpoch.AddSeconds(unixTimeInSeconds); - } - else if (property.NameEquals(ChangeFeedMetadataFields.OperationType)) - { - metadata.OperationType = (ChangeFeedOperationType)Enum.Parse(enumType: typeof(ChangeFeedOperationType), value: property.Value.GetString(), ignoreCase: true); - } - else if (property.NameEquals(ChangeFeedMetadataFields.TimeToLiveExpired)) - { - metadata.IsTimeToLiveExpired = property.Value.GetBoolean(); - } - else if (property.NameEquals(ChangeFeedMetadataFields.PreviousImageLSN)) - { - metadata.PreviousLsn = property.Value.GetInt64(); - } - else if (property.NameEquals(ChangeFeedMetadataFields.Id)) - { - metadata.Id = property.Value.GetString(); - } - else if (property.NameEquals(ChangeFeedMetadataFields.PartitionKey)) - { - // Dictionary is handled by default System.Text.Json deserialization - metadata.PartitionKey = JsonSerializer.Deserialize>(property.Value.GetRawText(), options); - } - } - - // validate delete operation requirements - if (metadata.OperationType == ChangeFeedOperationType.Delete) - { - if (metadata.Id == null || metadata.PartitionKey == null) - { - throw new JsonException("Delete operations require both 'id' and 'partitionKey' to be present."); - } - } - - return metadata; - } - - public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, JsonSerializerOptions options) - { - if (value == null) - { - return; - } - - writer.WriteStartObject(); - - writer.WritePropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp); - long unixTimeInSeconds = (long)(value.ConflictResolutionTimestamp - UnixEpoch).TotalSeconds; - writer.WriteNumberValue(unixTimeInSeconds); - - writer.WriteBoolean(ChangeFeedMetadataFields.TimeToLiveExpired, value.IsTimeToLiveExpired); - writer.WriteNumber(ChangeFeedMetadataFields.Lsn, value.Lsn); - writer.WriteString(ChangeFeedMetadataFields.OperationType, value.OperationType.ToString()); - writer.WriteNumber(ChangeFeedMetadataFields.PreviousImageLSN, value.PreviousLsn); - - if (value.Id != null) - { - writer.WriteString(ChangeFeedMetadataFields.Id, value.Id); - } - - if (value.PartitionKey != null) - { - // Dictionary is handled by default System.Text.Json serialization - writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); - JsonSerializer.Serialize(writer, value.PartitionKey, options); - } - - writer.WriteEndObject(); - } - } -} diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs deleted file mode 100644 index 0c7f8e0b48..0000000000 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs +++ /dev/null @@ -1,141 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters -{ - using System; - using System.Collections.Generic; - using Microsoft.Azure.Cosmos.Spatial; - using Newtonsoft.Json; - - internal class ChangeFeedMetadataNewtonSoftConverter : JsonConverter - { - private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); - - /// - /// Writes the JSON representation of the object. - /// - /// The to write to. - /// The object value to write. - /// The calling serializer. - public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) - { - if (value is ChangeFeedMetadata metadata) - { - writer.WriteStartObject(); - - writer.WritePropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp); - long unixTimeInSeconds = (long)(metadata.ConflictResolutionTimestamp - UnixEpoch).TotalSeconds; - writer.WriteValue(unixTimeInSeconds); - - writer.WritePropertyName(ChangeFeedMetadataFields.Lsn); - writer.WriteValue(metadata.Lsn); - - writer.WritePropertyName(ChangeFeedMetadataFields.OperationType); - serializer.Serialize(writer, metadata.OperationType); - - writer.WritePropertyName(ChangeFeedMetadataFields.PreviousImageLSN); - writer.WriteValue(metadata.PreviousLsn); - - writer.WritePropertyName(ChangeFeedMetadataFields.TimeToLiveExpired); - writer.WriteValue(metadata.IsTimeToLiveExpired); - - writer.WritePropertyName(ChangeFeedMetadataFields.Id); - writer.WriteValue(metadata.Id); - if (metadata.PartitionKey != null) - { - // Dictionary is handled by default Newtonsoft.Json serialization - writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); - serializer.Serialize(writer, metadata.PartitionKey); - } - - writer.WriteEndObject(); - } - else - { - throw new JsonSerializationException($"Unexpected value '{value}' of type '{value?.GetType()}' when converting {nameof(ChangeFeedMetadata)}."); - } - } - - /// - /// Reads the JSON representation of the object. - /// - /// The to read from. - /// Type of the object. - /// The existing value of object being read. - /// The calling serializer. - /// The deserialized object. - public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) - { - if (reader.TokenType == JsonToken.Null) - { - return null; - } - - ChangeFeedMetadata metadata = new ChangeFeedMetadata(); - - reader.Read(); // StartObject - - while (reader.TokenType == JsonToken.PropertyName) - { - string propertyName = reader.Value.ToString(); - reader.Read(); // Move to property value - - switch (propertyName) - { - case ChangeFeedMetadataFields.ConflictResolutionTimestamp: - // Read the Unix timestamp and convert to DateTime - if (reader.Value != null) - { - long unixTimeInSeconds = Convert.ToInt64(reader.Value); - metadata.ConflictResolutionTimestamp = UnixEpoch.AddSeconds(unixTimeInSeconds); - } - break; - - case ChangeFeedMetadataFields.Lsn: - metadata.Lsn = reader.Value != null ? Convert.ToInt64(reader.Value) : 0; - break; - - case ChangeFeedMetadataFields.OperationType: - metadata.OperationType = serializer.Deserialize(reader); - break; - - case ChangeFeedMetadataFields.PreviousImageLSN: - metadata.PreviousLsn = reader.Value != null ? Convert.ToInt64(reader.Value) : 0; - break; - - case ChangeFeedMetadataFields.TimeToLiveExpired: - metadata.IsTimeToLiveExpired = reader.Value != null && Convert.ToBoolean(reader.Value); - break; - - case ChangeFeedMetadataFields.Id: - metadata.Id = reader.Value?.ToString(); - break; - - case ChangeFeedMetadataFields.PartitionKey: - // Dictionary is handled by default Newtonsoft.Json deserialization - metadata.PartitionKey = serializer.Deserialize>(reader); - break; - - default: - reader.Skip(); - break; - } - - reader.Read(); // Move to next property or EndObject - } - - return metadata; - } - /// - /// Determines whether this instance can convert the specified object type. - /// - /// Type of the object. - /// true if this instance can convert the specified object type; otherwise, false. - public override bool CanConvert(Type objectType) - { - return objectType == typeof(ChangeFeedMetadata); - } - } -} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 7ab92ebadb..765a3ff7ac 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -325,18 +325,19 @@ public void ValidateChangeFeedMetadataSerializationReplaceAnDeleteWriteTest(bool Lsn = 374, OperationType = ChangeFeedOperationType.Create, IsTimeToLiveExpired = true, - ConflictResolutionTimestamp = DateTime.Parse("7/31/2024 7:59:30 PM") + ConflictResolutionTimestampInSeconds = 1722455970 }; string json = System.Text.Json.JsonSerializer.Serialize( value: metadata, options: new JsonSerializerOptions { - PropertyNameCaseInsensitive = propertyNameCaseInsensitive + PropertyNameCaseInsensitive = propertyNameCaseInsensitive, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }); Assert.AreEqual( - expected: @"{""crts"":1722455970,""timeToLiveExpired"":true,""lsn"":374,""operationType"":""Create"",""previousImageLSN"":15}", + expected: @"{""crts"":1722455970,""lsn"":374,""operationType"":""Create"",""previousImageLSN"":15,""timeToLiveExpired"":true}", actual: json); } @@ -351,18 +352,19 @@ public void ValidateChangeFeedMetadataSerializationCreateWriteTest(bool property { Lsn = 374, OperationType = ChangeFeedOperationType.Create, - ConflictResolutionTimestamp = DateTime.Parse("7/31/2024 7:59:30 PM") + ConflictResolutionTimestampInSeconds = 1722455970 }; string json = System.Text.Json.JsonSerializer.Serialize( value: metadata, options: new JsonSerializerOptions { - PropertyNameCaseInsensitive = propertyNameCaseInsensitive + PropertyNameCaseInsensitive = propertyNameCaseInsensitive, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }); Assert.AreEqual( - expected: @"{""crts"":1722455970,""timeToLiveExpired"":false,""lsn"":374,""operationType"":""Create"",""previousImageLSN"":0}", + expected: @"{""crts"":1722455970,""lsn"":374,""operationType"":""Create"",""previousImageLSN"":0,""timeToLiveExpired"":false}", actual: json); } From 617b4abc8bf6e87b94cc59ac1afa4ff0ea60db37 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Fri, 31 Oct 2025 15:23:11 -0700 Subject: [PATCH 21/33] Address comments --- Microsoft.Azure.Cosmos.sln | 16 +- .../FullFidelity/ChangeFeedMetadata.cs | 48 +++++- .../CFP/AllVersionsAndDeletes/BuilderTests.cs | 2 +- .../BuilderWithCustomSerializerTests.cs | 162 +++++++++++++++++- .../CFP/AllVersionsAndDeletes/ToDoActivity.cs | 16 ++ 5 files changed, 224 insertions(+), 20 deletions(-) diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index 5dd1a4d66a..cec2d9a5f4 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.14.36623.8 d17.14 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29123.88 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject @@ -181,18 +181,6 @@ Global {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.Build.0 = Release|Any CPU {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.ActiveCfg = Release|Any CPU {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.Build.0 = Release|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Cover|Any CPU.ActiveCfg = Debug|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Cover|Any CPU.Build.0 = Debug|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Cover|x64.ActiveCfg = Debug|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Cover|x64.Build.0 = Debug|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Debug|x64.ActiveCfg = Debug|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Debug|x64.Build.0 = Debug|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Release|Any CPU.Build.0 = Release|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Release|x64.ActiveCfg = Release|Any CPU - {D744906A-1091-403F-B0B6-794DE045169A}.Release|x64.Build.0 = Release|Any CPU {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|Any CPU.ActiveCfg = Debug|Any CPU {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|Any CPU.Build.0 = Debug|Any CPU {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|x64.ActiveCfg = Debug|Any CPU diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index 6e509e1c61..bb7ad0ba21 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -27,11 +27,12 @@ class ChangeFeedMetadata /// The change's conflict resolution timestamp. /// [System.Text.Json.Serialization.JsonIgnore] - public DateTime ConflictResolutionTimestamp => UnixEpoch.AddSeconds(this.ConflictResolutionTimestampInSeconds.Value); + [Newtonsoft.Json.JsonIgnore] + public DateTime? ConflictResolutionTimestamp => this.ConflictResolutionTimestampInSeconds.HasValue ? UnixEpoch.AddSeconds(this.ConflictResolutionTimestampInSeconds.Value) : null; [System.Text.Json.Serialization.JsonInclude] [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp)] - [JsonProperty(PropertyName = ChangeFeedMetadataFields.ConflictResolutionTimestamp, NullValueHandling = NullValueHandling.Ignore)] + [JsonProperty(PropertyName = ChangeFeedMetadataFields.ConflictResolutionTimestamp)] internal double? ConflictResolutionTimestampInSeconds { get; set; } /// @@ -78,9 +79,48 @@ class ChangeFeedMetadata public string Id { get; internal set; } /// - /// Applicable for delete operations only, otherwise null. - /// The partition key of the previous item version. string is the partition key property name and object is the partition key property value. All levels of hierarchy will be represented in order if a HPK is used. + /// Applicable for delete operations only, otherwise null. + /// The partition key of the previous item version represented as a dictionary where the key is the partition key property name + /// and the value is the partition key property value. All levels of hierarchy will be present if a hierarchical partition key (HPK) is used. /// + /// + /// + /// For single partition key containers, the dictionary will contain one entry with the partition key path name (without the leading '/') + /// as the key and the partition key value as the value. + /// + /// + /// For hierarchical partition key containers, the dictionary will contain multiple entries, one for each level of the hierarchy, + /// in the order they were defined in the container's partition key definition. + /// + /// + /// Example for a single partition key container with partition key path "/tenantId": + /// + /// { + /// "tenantId": "tenant123" + /// } + /// + /// + /// + /// Example for a hierarchical partition key container with partition key paths ["/tenantId", "/userId", "/sessionId"]: + /// + /// { + /// "tenantId": "tenant123", + /// "userId": "user456", + /// "sessionId": "session789" + /// } + /// + /// + /// + /// The partition key values can be of different types (string, number, boolean, null) depending on the document's schema. + /// For example, with partition key paths ["/category", "/priority"]: + /// + /// { + /// "category": "electronics", + /// "priority": 1 + /// } + /// + /// + /// [System.Text.Json.Serialization.JsonInclude] [System.Text.Json.Serialization.JsonPropertyName(ChangeFeedMetadataFields.PartitionKey)] [JsonProperty(PropertyName = ChangeFeedMetadataFields.PartitionKey, NullValueHandling = NullValueHandling.Ignore)] diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs index da3b29057a..90b7d8c357 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs @@ -179,7 +179,7 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() ChangeFeedOperationType operationType = change.Metadata.OperationType; long previousLsn = change.Metadata.PreviousLsn; - DateTime m = change.Metadata.ConflictResolutionTimestamp; + DateTime? m = change.Metadata.ConflictResolutionTimestamp; long lsn = change.Metadata.Lsn; bool isTimeToLiveExpired = change.Metadata.IsTimeToLiveExpired; } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 765a3ff7ac..1c749f08b3 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -312,6 +312,166 @@ static void ValidateDeserialization(List> activitie } } + [TestMethod] + [Owner("trivediyash")] + [Description("Validating to deserization using NSJ and STJ of ChangeFeedItem with HPK (Hierarchical Partition Key) with Create, Replace, and Delete payload.")] + [DataRow(true)] + [DataRow(false)] + public void ValidateNSJAndSTJSerializationOfChangeFeedItemWithHPKTest(bool propertyNameCaseInsensitive) + { + string json = @"[ + { + ""current"": { + ""id"": ""1"", + ""pk1"": ""value1"", + ""pk2"": ""value2"", + ""pk3"": ""value3"", + ""description"": ""original test with HPK"", + ""_rid"": ""HpxDAL+dzLQBAAAAAAAAAA=="", + ""_self"": ""dbs/HpxDAA==/colls/HpxDAL+dzLQ=/docs/HpxDAL+dzLQBAAAAAAAAAA==/"", + ""_etag"": ""\""00000000-0000-0000-e384-28095c1a01da\"""", + ""_attachments"": ""attachments/"", + ""_ts"": 1722455970 + }, + ""metadata"": { + ""crts"": 1722455970, + ""lsn"": 374, + ""operationType"": ""create"", + ""previousImageLSN"": 0, + ""timeToLiveExpired"": false + } + }, + { + ""current"": { + ""id"": ""1"", + ""pk1"": ""value1"", + ""pk2"": ""value2"", + ""pk3"": ""value3"", + ""description"": ""test after replace with HPK"", + ""_rid"": ""HpxDAL+dzLQBAAAAAAAAAA=="", + ""_self"": ""dbs/HpxDAA==/colls/HpxDAL+dzLQ=/docs/HpxDAL+dzLQBAAAAAAAAAA==/"", + ""_etag"": ""\""00000000-0000-0000-e384-28a5abdd01da\"""", + ""_attachments"": ""attachments/"", + ""_ts"": 1722455971 + }, + ""metadata"": { + ""crts"": 1722455971, + ""lsn"": 375, + ""operationType"": ""replace"", + ""previousImageLSN"": 374, + ""timeToLiveExpired"": false + } + }, + { + ""current"": {}, + ""metadata"": { + ""crts"": 1722455972, + ""lsn"": 376, + ""operationType"": ""delete"", + ""previousImageLSN"": 375, + ""timeToLiveExpired"": false, + ""id"": ""1"", + ""partitionKey"": { + ""pk1"": ""value1"", + ""pk2"": ""value2"", + ""pk3"": ""value3"" + } + }, + ""previous"": { + ""id"": ""1"", + ""pk1"": ""value1"", + ""pk2"": ""value2"", + ""pk3"": ""value3"", + ""description"": ""test after replace with HPK"", + ""_rid"": ""HpxDAL+dzLQBAAAAAAAAAA=="", + ""_self"": ""dbs/HpxDAA==/colls/HpxDAL+dzLQ=/docs/HpxDAL+dzLQBAAAAAAAAAA==/"", + ""_etag"": ""\""00000000-0000-0000-e384-28a5abdd01da\"""", + ""_attachments"": ""attachments/"", + ""_ts"": 1722455971 + } + } + ]"; + + ValidateSystemTextJsonDeserialization(json, propertyNameCaseInsensitive); + ValidateNewtonsoftJsonDeserialization(json); + + static void ValidateNewtonsoftJsonDeserialization(string json) + { + ValidateDeserialization(JsonConvert.DeserializeObject>>(json)); + } + + static void ValidateSystemTextJsonDeserialization(string json, bool propertyNameCaseInsensitive) + { + ValidateDeserialization(System.Text.Json.JsonSerializer.Deserialize>>( + json: json, + options: new JsonSerializerOptions() + { + PropertyNameCaseInsensitive = propertyNameCaseInsensitive + })); + } + + static void ValidateDeserialization(List> activities) + { + Assert.IsNotNull(activities); + + ChangeFeedItem createdUpdate = activities.ElementAt(0); + Assert.IsNotNull(createdUpdate); + Assert.IsNotNull(createdUpdate.Current); + Assert.AreEqual(expected: "original test with HPK", actual: createdUpdate.Current.description); + Assert.AreEqual(expected: "1", actual: createdUpdate.Current.id); + Assert.AreEqual(expected: "value1", actual: createdUpdate.Current.pk1); + Assert.AreEqual(expected: "value2", actual: createdUpdate.Current.pk2); + Assert.AreEqual(expected: "value3", actual: createdUpdate.Current.pk3); + Assert.IsNotNull(createdUpdate.Metadata); + Assert.AreEqual(expected: DateTime.Parse("7/31/2024 7:59:30 PM"), actual: createdUpdate.Metadata.ConflictResolutionTimestamp); + Assert.AreEqual(expected: 374, actual: createdUpdate.Metadata.Lsn); + Assert.AreEqual(expected: ChangeFeedOperationType.Create, actual: createdUpdate.Metadata.OperationType); + Assert.AreEqual(expected: 0, actual: createdUpdate.Metadata.PreviousLsn); + Assert.IsFalse(createdUpdate.Metadata.IsTimeToLiveExpired); + Assert.IsNull(createdUpdate.Previous); // No Previous for a Create change. + + ChangeFeedItem replacedChange = activities.ElementAt(1); + Assert.IsNotNull(replacedChange); + Assert.IsNotNull(replacedChange.Current); + Assert.AreEqual(expected: "test after replace with HPK", actual: replacedChange.Current.description); + Assert.AreEqual(expected: "1", actual: replacedChange.Current.id); + Assert.AreEqual(expected: "value1", actual: replacedChange.Current.pk1); + Assert.AreEqual(expected: "value2", actual: replacedChange.Current.pk2); + Assert.AreEqual(expected: "value3", actual: replacedChange.Current.pk3); + Assert.IsNotNull(replacedChange.Metadata); + Assert.AreEqual(expected: DateTime.Parse("7/31/2024 7:59:31 PM"), actual: replacedChange.Metadata.ConflictResolutionTimestamp); + Assert.AreEqual(expected: 375, actual: replacedChange.Metadata.Lsn); + Assert.AreEqual(expected: ChangeFeedOperationType.Replace, actual: replacedChange.Metadata.OperationType); + Assert.AreEqual(expected: 374, actual: replacedChange.Metadata.PreviousLsn); + Assert.IsFalse(replacedChange.Metadata.IsTimeToLiveExpired); + Assert.IsNull(replacedChange.Previous); // No Previous for a Replace change. + + ChangeFeedItem deletedChange = activities.ElementAt(2); + Assert.IsNotNull(deletedChange); + Assert.IsNotNull(deletedChange.Current); // Current is not null, but no data. + Assert.AreEqual(expected: default, actual: deletedChange.Current.description); // No current description for Delete + Assert.AreEqual(expected: default, actual: deletedChange.Current.id); // No current id for Delete + Assert.IsNotNull(deletedChange.Metadata); + Assert.AreEqual(expected: DateTime.Parse("7/31/2024 7:59:32 PM"), actual: deletedChange.Metadata.ConflictResolutionTimestamp); + Assert.AreEqual(expected: 376, actual: deletedChange.Metadata.Lsn); + Assert.AreEqual(expected: ChangeFeedOperationType.Delete, actual: deletedChange.Metadata.OperationType); + Assert.AreEqual(expected: 375, actual: deletedChange.Metadata.PreviousLsn); + Assert.IsFalse(deletedChange.Metadata.IsTimeToLiveExpired); + Assert.IsNotNull(deletedChange.Previous); + Assert.AreEqual(expected: "test after replace with HPK", actual: deletedChange.Previous.description); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.Id.ToString()); + Assert.IsNotNull(deletedChange.Metadata.PartitionKey); + Assert.AreEqual(expected: 3, actual: deletedChange.Metadata.PartitionKey.Count); + Assert.AreEqual(expected: "value1", actual: deletedChange.Metadata.PartitionKey["pk1"].ToString()); + Assert.AreEqual(expected: "value2", actual: deletedChange.Metadata.PartitionKey["pk2"].ToString()); + Assert.AreEqual(expected: "value3", actual: deletedChange.Metadata.PartitionKey["pk3"].ToString()); + Assert.AreEqual(expected: "1", actual: deletedChange.Previous.id); + Assert.AreEqual(expected: "value1", actual: deletedChange.Previous.pk1); + Assert.AreEqual(expected: "value2", actual: deletedChange.Previous.pk2); + Assert.AreEqual(expected: "value3", actual: deletedChange.Previous.pk3); + } + } + [TestMethod] [Owner("philipthomas-MSFT")] [Description("Replace and Deletes have full ChangeFeedMetadata.")] @@ -545,7 +705,7 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr ChangeFeedOperationType operationType = change.Metadata.OperationType; long previousLsn = change.Metadata.PreviousLsn; - DateTime m = change.Metadata.ConflictResolutionTimestamp; + DateTime? m = change.Metadata.ConflictResolutionTimestamp; long lsn = change.Metadata.Lsn; bool isTimeToLiveExpired = change.Metadata.IsTimeToLiveExpired; } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/ToDoActivity.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/ToDoActivity.cs index 1058c8f3fe..d8c6df721a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/ToDoActivity.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/ToDoActivity.cs @@ -15,4 +15,20 @@ public class ToDoActivity public int ttl { get; set; } } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Used for CFP AllVersionsAndDeletes builder tests with HPK without having attribute annotations from STJ or NSJ.")] + public class ToDoActivityWithHPK + { + public string id { get; set; } + + public string pk1 { get; set; } + + public string pk2 { get; set; } + + public string pk3 { get; set; } + + public string description { get; set; } + + public int ttl { get; set; } + } } From 9964fd00971b8f0f73a82d4a21026a6ed0cc63d3 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Tue, 4 Nov 2025 12:10:35 -0800 Subject: [PATCH 22/33] Add E2E live account test --- .../CosmosItemIntegrationTests.cs | 976 ++++++++++-------- 1 file changed, 533 insertions(+), 443 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemIntegrationTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemIntegrationTests.cs index 646cc28f8d..c636351fdc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemIntegrationTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemIntegrationTests.cs @@ -1,20 +1,20 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests { using System; - using System.Collections.Generic; - using System.IO; + using System.Collections.Generic; + using System.IO; using System.Linq; - using System.Net; - using System.Net.Http; - using System.Text; + using System.Net; + using System.Net.Http; + using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Diagnostics; using Microsoft.Azure.Cosmos.FaultInjection; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using Newtonsoft.Json.Linq; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json.Linq; using static Microsoft.Azure.Cosmos.Routing.GlobalPartitionEndpointManagerCore; using static Microsoft.Azure.Cosmos.SDK.EmulatorTests.MultiRegionSetupHelpers; @@ -29,15 +29,15 @@ public class CosmosItemIntegrationTests private static string region1; private static string region2; - private static string region3; + private static string region3; private IDictionary readRegionsMapping; private IList thinClientreadRegionalEndpoints; private CosmosSystemTextJsonSerializer cosmosSystemTextJsonSerializer; [TestInitialize] public async Task TestInitAsync() - { - this.connectionString = ConfigurationManager.GetEnvironmentVariable("COSMOSDB_MULTI_REGION", null); + { + this.connectionString = ConfigurationManager.GetEnvironmentVariable("COSMOSDB_MULTI_REGION", null); JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions() { @@ -153,93 +153,93 @@ public async Task ReadMany2UnreachablePartitionsTest() rule.Disable(); fiClient.Dispose(); } - } - - [TestMethod] + } + + [TestMethod] [Timeout(70000)] - [TestCategory("MultiRegion")] - public async Task DateTimeArrayRoundtrip_BinaryEncoding_CompareExtraDates_IntegrationTest() - { - string binaryEncodingEnabled = "binaryEncodingEnabled" + Guid.NewGuid().ToString("N"); - string binaryEncodingDisabled = "binaryEncodingDisabled" + Guid.NewGuid().ToString("N"); - string pk = "pk"; - string testId = Guid.NewGuid().ToString(); - - string[] dateStrings = + [TestCategory("MultiRegion")] + public async Task DateTimeArrayRoundtrip_BinaryEncoding_CompareExtraDates_IntegrationTest() + { + string binaryEncodingEnabled = "binaryEncodingEnabled" + Guid.NewGuid().ToString("N"); + string binaryEncodingDisabled = "binaryEncodingDisabled" + Guid.NewGuid().ToString("N"); + string pk = "pk"; + string testId = Guid.NewGuid().ToString(); + + string[] dateStrings = { "12/25/2023","2023-12-25","12-25-2023","25.12.2023","25/12/2023", "Dec 25, 2023","Dec 25 2023","2023-12-25T10:00:00","2023-12-25T10:00:00.123", "12/25/2023 10:00 AM","12/25/2023 10:00:00 AM","12/25/2023 10:00:00.123 AM","9999-12-31T23:59:59", "2023-12-25T10:00:00.1","2023-12-25T10:00:00.12", "2023-12-25T10:00:00.1234","2023-12-25T10:00:00.1234567" - }; - string[] formats = + }; + string[] formats = { "MM/dd/yyyy","yyyy-MM-dd","MM-dd-yyyy","dd.MM.yyyy","dd/MM/yyyy", "MMM dd, yyyy","MMM dd yyyy","yyyy-MM-ddTHH:mm:ss","yyyy-MM-ddTHH:mm:ss.fff", "yyyy-MM-ddTHH:mm:ss.f","yyyy-MM-ddTHH:mm:ss.ff","yyyy-MM-ddTHH:mm:ss.ffff", "yyyy-MM-ddTHH:mm:ss.fffffff","MM/dd/yyyy hh:mm tt","MM/dd/yyyy hh:mm:ss tt", "MM/dd/yyyy hh:mm:ss.fff tt" - }; - DateTime[] parsedDates = dateStrings - .Select(s => DateTime.ParseExact(s, formats, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.None)) - .ToArray(); - - TestCosmosItem testItem = new TestCosmosItem( - id: testId, - pk: pk, - title: "title", - email: "test@example.com", - body: "Binary encoding test document.", - createdUtc: DateTime.UtcNow, - modifiedUtc: DateTime.Parse("2025-03-26T20:22:20Z", null, System.Globalization.DateTimeStyles.AdjustToUniversal), - extraDates: parsedDates); - - Database db = this.database; - ContainerResponse containerBEEnabledResponse = await db.CreateContainerAsync(binaryEncodingEnabled, "/pk"); - ContainerResponse containerBEDisabledResponse = await db.CreateContainerAsync(binaryEncodingDisabled, "/pk"); - - try - { - // BinaryEncodingEnabled = True - Environment.SetEnvironmentVariable(ConfigurationManager.BinaryEncodingEnabled, "True"); - string rawJsonBEEnabled; - string rawJsonBEDisabled; - using (CosmosClient clientBinaryEncodingEnabled = new CosmosClient(this.connectionString)) - { - Container containerBinaryEncodingEnabled = clientBinaryEncodingEnabled.GetDatabase(db.Id).GetContainer(binaryEncodingEnabled); - await containerBinaryEncodingEnabled.CreateItemAsync(testItem, new Microsoft.Azure.Cosmos.PartitionKey(pk)); - using ResponseMessage response = await containerBinaryEncodingEnabled.ReadItemStreamAsync(testId, new Microsoft.Azure.Cosmos.PartitionKey(pk)); - using StreamReader reader = new StreamReader(response.Content, Encoding.UTF8); + }; + DateTime[] parsedDates = dateStrings + .Select(s => DateTime.ParseExact(s, formats, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.None)) + .ToArray(); + + TestCosmosItem testItem = new TestCosmosItem( + id: testId, + pk: pk, + title: "title", + email: "test@example.com", + body: "Binary encoding test document.", + createdUtc: DateTime.UtcNow, + modifiedUtc: DateTime.Parse("2025-03-26T20:22:20Z", null, System.Globalization.DateTimeStyles.AdjustToUniversal), + extraDates: parsedDates); + + Database db = this.database; + ContainerResponse containerBEEnabledResponse = await db.CreateContainerAsync(binaryEncodingEnabled, "/pk"); + ContainerResponse containerBEDisabledResponse = await db.CreateContainerAsync(binaryEncodingDisabled, "/pk"); + + try + { + // BinaryEncodingEnabled = True + Environment.SetEnvironmentVariable(ConfigurationManager.BinaryEncodingEnabled, "True"); + string rawJsonBEEnabled; + string rawJsonBEDisabled; + using (CosmosClient clientBinaryEncodingEnabled = new CosmosClient(this.connectionString)) + { + Container containerBinaryEncodingEnabled = clientBinaryEncodingEnabled.GetDatabase(db.Id).GetContainer(binaryEncodingEnabled); + await containerBinaryEncodingEnabled.CreateItemAsync(testItem, new Microsoft.Azure.Cosmos.PartitionKey(pk)); + using ResponseMessage response = await containerBinaryEncodingEnabled.ReadItemStreamAsync(testId, new Microsoft.Azure.Cosmos.PartitionKey(pk)); + using StreamReader reader = new StreamReader(response.Content, Encoding.UTF8); rawJsonBEEnabled = await reader.ReadToEndAsync(); - - } - - // BinaryEncodingEnabled = False - Environment.SetEnvironmentVariable(ConfigurationManager.BinaryEncodingEnabled, "False"); - using (CosmosClient clientBinaryEncodingDisabled = new CosmosClient(this.connectionString)) - { - Container containerBinaryEncodingDisabled = clientBinaryEncodingDisabled.GetDatabase(db.Id).GetContainer(binaryEncodingDisabled); - await containerBinaryEncodingDisabled.CreateItemAsync(testItem, new Microsoft.Azure.Cosmos.PartitionKey(pk)); - using ResponseMessage response = await containerBinaryEncodingDisabled.ReadItemStreamAsync(testId, new Microsoft.Azure.Cosmos.PartitionKey(pk)); - using StreamReader reader = new StreamReader(response.Content, Encoding.UTF8); - rawJsonBEDisabled = await reader.ReadToEndAsync(); - } - - using JsonDocument docTrue = JsonDocument.Parse(rawJsonBEEnabled); - using JsonDocument docFalse = JsonDocument.Parse(rawJsonBEDisabled); - - string extraDatesTrue = docTrue.RootElement.GetProperty("ExtraDates").GetRawText(); - string extraDatesFalse = docFalse.RootElement.GetProperty("ExtraDates").GetRawText(); - + + } + + // BinaryEncodingEnabled = False + Environment.SetEnvironmentVariable(ConfigurationManager.BinaryEncodingEnabled, "False"); + using (CosmosClient clientBinaryEncodingDisabled = new CosmosClient(this.connectionString)) + { + Container containerBinaryEncodingDisabled = clientBinaryEncodingDisabled.GetDatabase(db.Id).GetContainer(binaryEncodingDisabled); + await containerBinaryEncodingDisabled.CreateItemAsync(testItem, new Microsoft.Azure.Cosmos.PartitionKey(pk)); + using ResponseMessage response = await containerBinaryEncodingDisabled.ReadItemStreamAsync(testId, new Microsoft.Azure.Cosmos.PartitionKey(pk)); + using StreamReader reader = new StreamReader(response.Content, Encoding.UTF8); + rawJsonBEDisabled = await reader.ReadToEndAsync(); + } + + using JsonDocument docTrue = JsonDocument.Parse(rawJsonBEEnabled); + using JsonDocument docFalse = JsonDocument.Parse(rawJsonBEDisabled); + + string extraDatesTrue = docTrue.RootElement.GetProperty("ExtraDates").GetRawText(); + string extraDatesFalse = docFalse.RootElement.GetProperty("ExtraDates").GetRawText(); + Assert.AreEqual(extraDatesTrue, extraDatesFalse, $"ExtraDates JSON mismatch:\nTrue: {extraDatesTrue}\nFalse: {extraDatesFalse}"); - } - finally - { - await containerBEEnabledResponse.Container.DeleteContainerAsync(); - await containerBEDisabledResponse.Container.DeleteContainerAsync(); - } - } + } + finally + { + await containerBEEnabledResponse.Container.DeleteContainerAsync(); + await containerBEDisabledResponse.Container.DeleteContainerAsync(); + } + } [TestMethod] [TestCategory("MultiRegion")] @@ -483,8 +483,8 @@ public async Task ReadItemAsync_WithCircuitBreakerEnabledAndSingleMasterAccountA { // Arrange. Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelCircuitBreakerEnabled, "True"); - Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, circuitBreakerConsecutiveFailureCount); - + Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, circuitBreakerConsecutiveFailureCount); + // Enabling fault injection rule to simulate a 503 service unavailable scenario. string serviceUnavailableRuleId = "503-rule-" + Guid.NewGuid().ToString(); FaultInjectionRule serviceUnavailableRule = new FaultInjectionRuleBuilder( @@ -540,7 +540,7 @@ public async Task ReadItemAsync_WithCircuitBreakerEnabledAndSingleMasterAccountA ItemResponse readResponse = await container.ReadItemAsync( id: itemsList[0].Id, partitionKey: new PartitionKey(itemsList[0].Pk)); - + IReadOnlyList<(string regionName, Uri uri)> contactedRegionMapping = readResponse.Diagnostics.GetContactedRegions(); HashSet contactedRegions = new(contactedRegionMapping.Select(r => r.regionName)); @@ -595,12 +595,12 @@ public async Task ReadItemAsync_WithCircuitBreakerEnabledAndSingleMasterAccountA finally { Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelCircuitBreakerEnabled, null); - Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, null); + Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, null); await this.TryDeleteItems(itemsList); } - } - + } + [TestMethod] [TestCategory("MultiRegion")] [DataRow(ConnectionMode.Direct, DisplayName ="Direct Mode")] @@ -611,7 +611,7 @@ public async Task ReadItemAsync_WithCircuitBreakerEnabledAndTimeoutCounterOverwr ConnectionMode connectionMode) { // Arrange. - Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelCircuitBreakerEnabled, "True"); + Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelCircuitBreakerEnabled, "True"); Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerTimeoutCounterResetWindowInMinutes, "0.0833"); // setting to 5 seconds // Enabling fault injection rule to simulate a 503 service unavailable scenario. @@ -657,9 +657,9 @@ public async Task ReadItemAsync_WithCircuitBreakerEnabledAndTimeoutCounterOverwr await this.TryCreateItems(itemsList); //Must Ensure the data is replicated to all regions - await Task.Delay(3000); - - int readErrorCount = 0; + await Task.Delay(3000); + + int readErrorCount = 0; PartitionKeyRangeFailoverInfo failoverInfo; for (int i = 1; i <= 3; i++) @@ -681,10 +681,10 @@ public async Task ReadItemAsync_WithCircuitBreakerEnabledAndTimeoutCounterOverwr failoverInfo = TestCommon.GetFailoverInfoForFirstPartitionUsingReflection( globalPartitionEndpointManager: cosmosClient.ClientContext.DocumentClient.PartitionKeyRangeLocation, - isReadOnlyOrMultiMaster: true); - + isReadOnlyOrMultiMaster: true); + failoverInfo.SnapshotConsecutiveRequestFailureCount(out readErrorCount, out _); - + Assert.IsTrue(readErrorCount > 0); } catch (CosmosException) @@ -695,28 +695,28 @@ public async Task ReadItemAsync_WithCircuitBreakerEnabledAndTimeoutCounterOverwr { Assert.Fail($"Unhandled Exception was thrown during ReadItemAsync call. Message: {ex.Message}"); } - } - - await Task.Delay(6000); // Wait for the timeout counter to reset - - try - { + } + + await Task.Delay(6000); // Wait for the timeout counter to reset + + try + { ItemResponse readResponse = await container.ReadItemAsync( id: itemsList[0].Id, - partitionKey: new PartitionKey(itemsList[0].Pk)); - } - catch (CosmosException) - { - Assert.Fail("Read Item operation should succeed after the timeout counter is overwritten."); - } - + partitionKey: new PartitionKey(itemsList[0].Pk)); + } + catch (CosmosException) + { + Assert.Fail("Read Item operation should succeed after the timeout counter is overwritten."); + } + failoverInfo = TestCommon.GetFailoverInfoForFirstPartitionUsingReflection( globalPartitionEndpointManager: cosmosClient.ClientContext.DocumentClient.PartitionKeyRangeLocation, - isReadOnlyOrMultiMaster: true); - - failoverInfo.SnapshotConsecutiveRequestFailureCount(out int currentReadErrorCount, out _); - - Assert.AreEqual(1, currentReadErrorCount, "The read error count should be reset after the timeout counter is overwritten. Then after one more failure it should be incremented by 1."); + isReadOnlyOrMultiMaster: true); + + failoverInfo.SnapshotConsecutiveRequestFailureCount(out int currentReadErrorCount, out _); + + Assert.AreEqual(1, currentReadErrorCount, "The read error count should be reset after the timeout counter is overwritten. Then after one more failure it should be incremented by 1."); Assert.IsTrue(readErrorCount > currentReadErrorCount, "The read error count should be greater than the current before the timeout counter is overwritten."); } finally @@ -1457,7 +1457,7 @@ public async Task CreateAndReadItemAsync_WithCircuitBreakerEnabledAndMultiMaster [TestMethod] [Owner("dkunda")] - [TestCategory("MultiRegion")] + [TestCategory("MultiRegion")] [Timeout(70000)] [DataRow(true, DisplayName = "Test scenario when PPAF is enabled at client level.")] [DataRow(false, DisplayName = "Test scenario when PPAF is disabled at client level.")] @@ -1481,35 +1481,35 @@ public async Task ReadItemAsync_WithPPAFEnabledAndSingleMasterAccountWithRespons .Build(); List rules = new List { serviceUnavailableRule }; - FaultInjector faultInjector = new FaultInjector(rules); - - // Now that the ppaf enablement flag is returned from gateway, we need to intercept the response and remove the flag from the response, so that - // the environment variable set above is honored. - HttpClientHandlerHelper httpClientHandlerHelper = new HttpClientHandlerHelper() - { - ResponseIntercepter = async (response, request) => - { - string json = await response?.Content?.ReadAsStringAsync(); - if (json.Length > 0 && json.Contains("enablePerPartitionFailoverBehavior")) - { - JObject parsedDatabaseAccountResponse = JObject.Parse(json); - parsedDatabaseAccountResponse.Property("enablePerPartitionFailoverBehavior").Value = enablePartitionLevelFailover.ToString(); - - HttpResponseMessage interceptedResponse = new() - { - StatusCode = response.StatusCode, - Content = new StringContent(parsedDatabaseAccountResponse.ToString()), - Version = response.Version, - ReasonPhrase = response.ReasonPhrase, - RequestMessage = response.RequestMessage, - }; - - return interceptedResponse; - } - - return response; - }, - }; + FaultInjector faultInjector = new FaultInjector(rules); + + // Now that the ppaf enablement flag is returned from gateway, we need to intercept the response and remove the flag from the response, so that + // the environment variable set above is honored. + HttpClientHandlerHelper httpClientHandlerHelper = new HttpClientHandlerHelper() + { + ResponseIntercepter = async (response, request) => + { + string json = await response?.Content?.ReadAsStringAsync(); + if (json.Length > 0 && json.Contains("enablePerPartitionFailoverBehavior")) + { + JObject parsedDatabaseAccountResponse = JObject.Parse(json); + parsedDatabaseAccountResponse.Property("enablePerPartitionFailoverBehavior").Value = enablePartitionLevelFailover.ToString(); + + HttpResponseMessage interceptedResponse = new() + { + StatusCode = response.StatusCode, + Content = new StringContent(parsedDatabaseAccountResponse.ToString()), + Version = response.Version, + ReasonPhrase = response.ReasonPhrase, + RequestMessage = response.RequestMessage, + }; + + return interceptedResponse; + } + + return response; + }, + }; List preferredRegions = new List { region1, region2, region3 }; CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() @@ -1517,8 +1517,8 @@ public async Task ReadItemAsync_WithPPAFEnabledAndSingleMasterAccountWithRespons ConsistencyLevel = ConsistencyLevel.Session, FaultInjector = faultInjector, RequestTimeout = TimeSpan.FromSeconds(5), - ApplicationPreferredRegions = preferredRegions, - HttpClientFactory = () => new HttpClient(httpClientHandlerHelper), + ApplicationPreferredRegions = preferredRegions, + HttpClientFactory = () => new HttpClient(httpClientHandlerHelper), }; List itemsList = new() @@ -1540,7 +1540,7 @@ public async Task ReadItemAsync_WithPPAFEnabledAndSingleMasterAccountWithRespons ItemResponse readResponse = await container.ReadItemAsync( id: itemsList[0].Id, - partitionKey: new PartitionKey(itemsList[0].Pk)); + partitionKey: new PartitionKey(itemsList[0].Pk)); IReadOnlyList<(string regionName, Uri uri)> contactedRegionMapping = readResponse.Diagnostics.GetContactedRegions(); HashSet contactedRegions = new(contactedRegionMapping.Select(r => r.regionName)); @@ -1574,15 +1574,15 @@ public async Task ReadItemAsync_WithPPAFEnabledAndSingleMasterAccountWithRespons { await this.TryDeleteItems(itemsList); } - } - + } + [TestMethod] [Owner("ntripician")] - [TestCategory("MultiRegion")] - [Timeout(70000 *100)] + [TestCategory("MultiRegion")] + [Timeout(70000 *100)] [DataRow(ConnectionMode.Direct, false, DisplayName = "Test dynamic PPAF enablement with Direct mode.")] - public async Task ReadItemAsync_WithPPAFDynamicOverride_ShouldEnableOrDisablePPAFInSDK( - ConnectionMode connectionMode, + public async Task ReadItemAsync_WithPPAFDynamicOverride_ShouldEnableOrDisablePPAFInSDK( + ConnectionMode connectionMode, bool isThinClientEnabled) { // Arrange. @@ -1602,68 +1602,68 @@ public async Task ReadItemAsync_WithPPAFDynamicOverride_ShouldEnableOrDisablePPA .Build(); List rules = new List { serviceUnavailableRule }; - FaultInjector faultInjector = new FaultInjector(rules); - - bool enablePPAF = false; - - // Now that the ppaf enablement flag is returned from gateway, we need to intercept the response and remove the flag from the response, so that - // the environment variable set above is honored. - HttpClientHandlerHelper httpClientHandlerHelper = new HttpClientHandlerHelper() - { - ResponseIntercepter = async (response, request) => - { - string json = await response?.Content?.ReadAsStringAsync(); - if (json.Length > 0 && json.Contains("enablePerPartitionFailoverBehavior")) - { - if (enablePPAF) - { - JObject parsedDatabaseAccountResponse = JObject.Parse(json); - parsedDatabaseAccountResponse.Property("enablePerPartitionFailoverBehavior").Value = true; - - HttpResponseMessage interceptedResponse = new() - { - StatusCode = response.StatusCode, - Content = new StringContent(parsedDatabaseAccountResponse.ToString()), - Version = response.Version, - ReasonPhrase = response.ReasonPhrase, - RequestMessage = response.RequestMessage, - }; - - return interceptedResponse; - } - else - { - JObject parsedDatabaseAccountResponse = JObject.Parse(json); - parsedDatabaseAccountResponse.Property("enablePerPartitionFailoverBehavior").Value = false; - - HttpResponseMessage interceptedResponse = new() - { - StatusCode = response.StatusCode, - Content = new StringContent(parsedDatabaseAccountResponse.ToString()), - Version = response.Version, - ReasonPhrase = response.ReasonPhrase, - RequestMessage = response.RequestMessage, - }; - - return interceptedResponse; - } - - } - - return response; - }, - }; - + FaultInjector faultInjector = new FaultInjector(rules); + + bool enablePPAF = false; + + // Now that the ppaf enablement flag is returned from gateway, we need to intercept the response and remove the flag from the response, so that + // the environment variable set above is honored. + HttpClientHandlerHelper httpClientHandlerHelper = new HttpClientHandlerHelper() + { + ResponseIntercepter = async (response, request) => + { + string json = await response?.Content?.ReadAsStringAsync(); + if (json.Length > 0 && json.Contains("enablePerPartitionFailoverBehavior")) + { + if (enablePPAF) + { + JObject parsedDatabaseAccountResponse = JObject.Parse(json); + parsedDatabaseAccountResponse.Property("enablePerPartitionFailoverBehavior").Value = true; + + HttpResponseMessage interceptedResponse = new() + { + StatusCode = response.StatusCode, + Content = new StringContent(parsedDatabaseAccountResponse.ToString()), + Version = response.Version, + ReasonPhrase = response.ReasonPhrase, + RequestMessage = response.RequestMessage, + }; + + return interceptedResponse; + } + else + { + JObject parsedDatabaseAccountResponse = JObject.Parse(json); + parsedDatabaseAccountResponse.Property("enablePerPartitionFailoverBehavior").Value = false; + + HttpResponseMessage interceptedResponse = new() + { + StatusCode = response.StatusCode, + Content = new StringContent(parsedDatabaseAccountResponse.ToString()), + Version = response.Version, + ReasonPhrase = response.ReasonPhrase, + RequestMessage = response.RequestMessage, + }; + + return interceptedResponse; + } + + } + + return response; + }, + }; + List preferredRegions = new List { region1, region2, region3 }; CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() { ConsistencyLevel = ConsistencyLevel.Session, FaultInjector = faultInjector, RequestTimeout = TimeSpan.FromSeconds(5), - ApplicationPreferredRegions = preferredRegions, - HttpClientFactory = () => new HttpClient(httpClientHandlerHelper), - ConnectionMode = connectionMode, - ApplicationName = "ppafDynamicOverrideTest", + ApplicationPreferredRegions = preferredRegions, + HttpClientFactory = () => new HttpClient(httpClientHandlerHelper), + ConnectionMode = connectionMode, + ApplicationName = "ppafDynamicOverrideTest", }; List itemsList = new() @@ -1681,12 +1681,12 @@ public async Task ReadItemAsync_WithPPAFDynamicOverride_ShouldEnableOrDisablePPA await this.TryCreateItems(itemsList); //Must Ensure the data is replicated to all regions - await Task.Delay(3000); - + await Task.Delay(3000); + ItemResponse readResponse = await container.ReadItemAsync( id: itemsList[0].Id, - partitionKey: new PartitionKey(itemsList[0].Pk)); - + partitionKey: new PartitionKey(itemsList[0].Pk)); + IReadOnlyList<(string regionName, Uri uri)> contactedRegionMapping = readResponse.Diagnostics.GetContactedRegions(); HashSet contactedRegions = new(contactedRegionMapping.Select(r => r.regionName)); @@ -1697,22 +1697,22 @@ public async Task ReadItemAsync_WithPPAFDynamicOverride_ShouldEnableOrDisablePPA CosmosTraceDiagnostics traceDiagnostic = readResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContextNoPPAF); - - Assert.IsNull(hedgeContextNoPPAF); - Assert.IsNull(cosmosClient.DocumentClient.ConnectionPolicy.AvailabilityStrategy); - Assert.IsFalse(cosmosClient.DocumentClient.PartitionKeyRangeLocation.IsPartitionLevelAutomaticFailoverEnabled()); - - // Enable PPAF At the Gateway Layer. - enablePPAF = true; - - //force database account refresh + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContextNoPPAF); + + Assert.IsNull(hedgeContextNoPPAF); + Assert.IsNull(cosmosClient.DocumentClient.ConnectionPolicy.AvailabilityStrategy); + Assert.IsFalse(cosmosClient.DocumentClient.PartitionKeyRangeLocation.IsPartitionLevelAutomaticFailoverEnabled()); + + // Enable PPAF At the Gateway Layer. + enablePPAF = true; + + //force database account refresh await cosmosClient.DocumentClient.GlobalEndpointManager.RefreshLocationAsync(true); readResponse = await container.ReadItemAsync( id: itemsList[0].Id, - partitionKey: new PartitionKey(itemsList[0].Pk)); - + partitionKey: new PartitionKey(itemsList[0].Pk)); + contactedRegionMapping = readResponse.Diagnostics.GetContactedRegions(); contactedRegions = new(contactedRegionMapping.Select(r => r.regionName)); @@ -1729,18 +1729,18 @@ public async Task ReadItemAsync_WithPPAFDynamicOverride_ShouldEnableOrDisablePPA List hedgedRegions = ((IEnumerable)hedgeContext).ToList(); Assert.IsTrue(hedgedRegions.Count >= 1, "Since the first region is not available, the request should atleast hedge to the next region."); - Assert.IsTrue(cosmosClient.DocumentClient.PartitionKeyRangeLocation.IsPartitionLevelAutomaticFailoverEnabled()); - + Assert.IsTrue(cosmosClient.DocumentClient.PartitionKeyRangeLocation.IsPartitionLevelAutomaticFailoverEnabled()); + // Disable PPAF At the Gateway Layer. - enablePPAF = false; - - //force database account refresh - await cosmosClient.DocumentClient.GlobalEndpointManager.RefreshLocationAsync(true); - + enablePPAF = false; + + //force database account refresh + await cosmosClient.DocumentClient.GlobalEndpointManager.RefreshLocationAsync(true); + readResponse = await container.ReadItemAsync( id: itemsList[0].Id, - partitionKey: new PartitionKey(itemsList[0].Pk)); - + partitionKey: new PartitionKey(itemsList[0].Pk)); + contactedRegionMapping = readResponse.Diagnostics.GetContactedRegions(); contactedRegions = new(contactedRegionMapping.Select(r => r.regionName)); @@ -1751,30 +1751,30 @@ public async Task ReadItemAsync_WithPPAFDynamicOverride_ShouldEnableOrDisablePPA traceDiagnostic = readResponse.Diagnostics as CosmosTraceDiagnostics; Assert.IsNotNull(traceDiagnostic); - traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContextNoPPAF2); - - Assert.IsNull(hedgeContextNoPPAF2); - Assert.IsNull(cosmosClient.DocumentClient.ConnectionPolicy.AvailabilityStrategy); + traceDiagnostic.Value.Data.TryGetValue("Hedge Context", out object hedgeContextNoPPAF2); + + Assert.IsNull(hedgeContextNoPPAF2); + Assert.IsNull(cosmosClient.DocumentClient.ConnectionPolicy.AvailabilityStrategy); Assert.IsFalse(cosmosClient.DocumentClient.PartitionKeyRangeLocation.IsPartitionLevelAutomaticFailoverEnabled()); } finally { - await this.TryDeleteItems(itemsList); - - if (isThinClientEnabled) - { - Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, null); + await this.TryDeleteItems(itemsList); + + if (isThinClientEnabled) + { + Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, null); } } - } - + } + [TestMethod] [Owner("nalutripician")] - [TestCategory("MultiRegion")] + [TestCategory("MultiRegion")] [Timeout(70000)] [DataRow(true, DisplayName = "Test scenario when PPAF is enabled at client level.")] [DataRow(false, DisplayName = "Test scenario when PPAF is disabled at client level.")] - public async Task ReadItemAsync_WithPPAFDiableOverride( + public async Task ReadItemAsync_WithPPAFDiableOverride( bool enablePartitionLevelFailover) { // Arrange. @@ -1794,35 +1794,35 @@ public async Task ReadItemAsync_WithPPAFDiableOverride( .Build(); List rules = new List { serviceUnavailableRule }; - FaultInjector faultInjector = new FaultInjector(rules); - - // Now that the ppaf enablement flag is returned from gateway, we need to intercept the response and remove the flag from the response, so that - // the environment variable set above is honored. - HttpClientHandlerHelper httpClientHandlerHelper = new HttpClientHandlerHelper() - { - ResponseIntercepter = async (response, request) => - { - string json = await response?.Content?.ReadAsStringAsync(); - if (json.Length > 0 && json.Contains("enablePerPartitionFailoverBehavior")) - { - JObject parsedDatabaseAccountResponse = JObject.Parse(json); - parsedDatabaseAccountResponse.Property("enablePerPartitionFailoverBehavior").Value = enablePartitionLevelFailover.ToString(); - - HttpResponseMessage interceptedResponse = new() - { - StatusCode = response.StatusCode, - Content = new StringContent(parsedDatabaseAccountResponse.ToString()), - Version = response.Version, - ReasonPhrase = response.ReasonPhrase, - RequestMessage = response.RequestMessage, - }; - - return interceptedResponse; - } - - return response; - }, - }; + FaultInjector faultInjector = new FaultInjector(rules); + + // Now that the ppaf enablement flag is returned from gateway, we need to intercept the response and remove the flag from the response, so that + // the environment variable set above is honored. + HttpClientHandlerHelper httpClientHandlerHelper = new HttpClientHandlerHelper() + { + ResponseIntercepter = async (response, request) => + { + string json = await response?.Content?.ReadAsStringAsync(); + if (json.Length > 0 && json.Contains("enablePerPartitionFailoverBehavior")) + { + JObject parsedDatabaseAccountResponse = JObject.Parse(json); + parsedDatabaseAccountResponse.Property("enablePerPartitionFailoverBehavior").Value = enablePartitionLevelFailover.ToString(); + + HttpResponseMessage interceptedResponse = new() + { + StatusCode = response.StatusCode, + Content = new StringContent(parsedDatabaseAccountResponse.ToString()), + Version = response.Version, + ReasonPhrase = response.ReasonPhrase, + RequestMessage = response.RequestMessage, + }; + + return interceptedResponse; + } + + return response; + }, + }; List preferredRegions = new List { region1, region2, region3 }; CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() @@ -1830,9 +1830,9 @@ public async Task ReadItemAsync_WithPPAFDiableOverride( ConsistencyLevel = ConsistencyLevel.Session, FaultInjector = faultInjector, RequestTimeout = TimeSpan.FromSeconds(5), - ApplicationPreferredRegions = preferredRegions, - HttpClientFactory = () => new HttpClient(httpClientHandlerHelper), - DisablePartitionLevelFailover = true, // This will disable the PPAF override for this test. + ApplicationPreferredRegions = preferredRegions, + HttpClientFactory = () => new HttpClient(httpClientHandlerHelper), + DisablePartitionLevelFailover = true, // This will disable the PPAF override for this test. }; List itemsList = new() @@ -1854,7 +1854,7 @@ public async Task ReadItemAsync_WithPPAFDiableOverride( ItemResponse readResponse = await container.ReadItemAsync( id: itemsList[0].Id, - partitionKey: new PartitionKey(itemsList[0].Pk)); + partitionKey: new PartitionKey(itemsList[0].Pk)); IReadOnlyList<(string regionName, Uri uri)> contactedRegionMapping = readResponse.Diagnostics.GetContactedRegions(); HashSet contactedRegions = new(contactedRegionMapping.Select(r => r.regionName)); @@ -1877,76 +1877,76 @@ public async Task ReadItemAsync_WithPPAFDiableOverride( { await this.TryDeleteItems(itemsList); } - } - - [TestMethod] - [TestCategory("MultiRegion")] - [Owner("ntripician")] - public async Task AddressRefreshInternalServerErrorTest() - { - FaultInjectionRule internalServerError = new FaultInjectionRuleBuilder( - id: "rule1", - condition: new FaultInjectionConditionBuilder() - .WithOperationType(FaultInjectionOperationType.MetadataRefreshAddresses) - .WithRegion(region1) - .Build(), - result: - FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.InternalServerError) - .Build()) - .Build(); - - List rules = new List() { internalServerError }; - FaultInjector faultInjector = new FaultInjector(rules); - - internalServerError.Disable(); - - CosmosClientOptions clientOptions = new CosmosClientOptions() - { - ConnectionMode = ConnectionMode.Direct, - Serializer = this.cosmosSystemTextJsonSerializer, - ApplicationRegion = region1, - }; - - using (CosmosClient faultInjectionClient = new CosmosClient( - connectionString: this.connectionString, - clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions))) - { - Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); - Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); - - internalServerError.Enable(); - - try - { - ItemResponse response = await container.ReadItemAsync("testId", new PartitionKey("pk")); - Assert.IsTrue(internalServerError.GetHitCount() > 0); - Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); - } - catch (CosmosException ex) - { - Assert.Fail(ex.Message); - } - } } - + + [TestMethod] + [TestCategory("MultiRegion")] + [Owner("ntripician")] + public async Task AddressRefreshInternalServerErrorTest() + { + FaultInjectionRule internalServerError = new FaultInjectionRuleBuilder( + id: "rule1", + condition: new FaultInjectionConditionBuilder() + .WithOperationType(FaultInjectionOperationType.MetadataRefreshAddresses) + .WithRegion(region1) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.InternalServerError) + .Build()) + .Build(); + + List rules = new List() { internalServerError }; + FaultInjector faultInjector = new FaultInjector(rules); + + internalServerError.Disable(); + + CosmosClientOptions clientOptions = new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Direct, + Serializer = this.cosmosSystemTextJsonSerializer, + ApplicationRegion = region1, + }; + + using (CosmosClient faultInjectionClient = new CosmosClient( + connectionString: this.connectionString, + clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions))) + { + Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName); + Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + + internalServerError.Enable(); + + try + { + ItemResponse response = await container.ReadItemAsync("testId", new PartitionKey("pk")); + Assert.IsTrue(internalServerError.GetHitCount() > 0); + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + } + catch (CosmosException ex) + { + Assert.Fail(ex.Message); + } + } + } + [TestMethod] [TestCategory("MultiRegion")] [Ignore("We will enable this test once the test staging account used for multi master validation starts supporting thin proxy.")] [DataRow(ConnectionMode.Gateway, "15", "10", DisplayName = "Thin Client Mode - Scenario when the total iteration count is 15 and circuit breaker consecutive failure threshold is set to 10.")] [DataRow(ConnectionMode.Gateway, "25", "20", DisplayName = "Thin Client Mode - Scenario when the total iteration count is 25 and circuit breaker consecutive failure threshold is set to 20.")] - [DataRow(ConnectionMode.Gateway, "35", "30", DisplayName = "Thin Client Mode - Scenario when the total iteration count is 35 and circuit breaker consecutive failure threshold is set to 30.")] + [DataRow(ConnectionMode.Gateway, "35", "30", DisplayName = "Thin Client Mode - Scenario when the total iteration count is 35 and circuit breaker consecutive failure threshold is set to 30.")] [Owner("dkunda")] [Timeout(70000)] public async Task ReadItemAsync_WithThinClientCircuitBreakerEnabledAndSingleMasterAccountAndServiceUnavailableReceived_ShouldApplyPartitionLevelOverride( ConnectionMode connectionMode, string iterationCount, string circuitBreakerConsecutiveFailureCount) - { - // Arrange. - Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "True"); + { + // Arrange. + Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "True"); Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelCircuitBreakerEnabled, "True"); - Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, circuitBreakerConsecutiveFailureCount); - + Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, circuitBreakerConsecutiveFailureCount); + // Enabling fault injection rule to simulate a 503 service unavailable scenario. string serviceUnavailableRuleId = "503-rule-" + Guid.NewGuid().ToString(); FaultInjectionRule serviceUnavailableRule = new FaultInjectionRuleBuilder( @@ -1982,11 +1982,11 @@ public async Task ReadItemAsync_WithThinClientCircuitBreakerEnabledAndSingleMast try { - using CosmosClient cosmosClient = new(connectionString: this.connectionString, clientOptions: cosmosClientOptions); - AccountProperties accountInfo = await cosmosClient.ReadAccountAsync(); - - Assert.IsTrue(cosmosClient.DocumentClient.GlobalEndpointManager.ThinClientReadEndpoints.Count() >= 2); - this.thinClientreadRegionalEndpoints = cosmosClient.DocumentClient.GlobalEndpointManager.ThinClientReadEndpoints; + using CosmosClient cosmosClient = new(connectionString: this.connectionString, clientOptions: cosmosClientOptions); + AccountProperties accountInfo = await cosmosClient.ReadAccountAsync(); + + Assert.IsTrue(cosmosClient.DocumentClient.GlobalEndpointManager.ThinClientReadEndpoints.Count() >= 2); + this.thinClientreadRegionalEndpoints = cosmosClient.DocumentClient.GlobalEndpointManager.ThinClientReadEndpoints; Database database = cosmosClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); @@ -2007,7 +2007,7 @@ public async Task ReadItemAsync_WithThinClientCircuitBreakerEnabledAndSingleMast ItemResponse readResponse = await container.ReadItemAsync( id: itemsList[0].Id, partitionKey: new PartitionKey(itemsList[0].Pk)); - + IReadOnlyList<(string regionName, Uri uri)> contactedRegionMapping = readResponse.Diagnostics.GetContactedRegions(); HashSet contactedRegions = new(contactedRegionMapping.Select(r => r.regionName)); @@ -2050,31 +2050,31 @@ public async Task ReadItemAsync_WithThinClientCircuitBreakerEnabledAndSingleMast finally { Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelCircuitBreakerEnabled, null); - Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, null); - Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, null); + Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, null); + Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, null); await this.TryDeleteItems(itemsList); } - } - + } + [TestMethod] - [TestCategory("MultiMaster")] + [TestCategory("MultiMaster")] [Ignore ("We will enable this test once the test staging account used for multi master validation starts supporting thin proxy.")] [DataRow(ConnectionMode.Gateway, "15", "10", DisplayName = "Thin Client Mode - Scenario when the total iteration count is 15 and circuit breaker consecutive failure threshold is set to 10.")] [DataRow(ConnectionMode.Gateway, "25", "20", DisplayName = "Thin Client Mode - Scenario when the total iteration count is 25 and circuit breaker consecutive failure threshold is set to 20.")] - [DataRow(ConnectionMode.Gateway, "35", "30", DisplayName = "Thin Client Mode - Scenario when the total iteration count is 35 and circuit breaker consecutive failure threshold is set to 30.")] + [DataRow(ConnectionMode.Gateway, "35", "30", DisplayName = "Thin Client Mode - Scenario when the total iteration count is 35 and circuit breaker consecutive failure threshold is set to 30.")] [Owner("dkunda")] [Timeout(70000)] public async Task CreateItemAsync_WithThinClientEnabledAndCircuitBreakerEnabledAndMultiMasterAccountAndServiceUnavailableReceived_ShouldApplyPartitionLevelOverride( ConnectionMode connectionMode, string iterationCount, string circuitBreakerConsecutiveFailureCount) - { - // Arrange. - Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "True"); + { + // Arrange. + Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "True"); Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelCircuitBreakerEnabled, "True"); - Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, circuitBreakerConsecutiveFailureCount); - + Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, circuitBreakerConsecutiveFailureCount); + // Enabling fault injection rule to simulate a 503 service unavailable scenario. string serviceUnavailableRuleId = "503-rule-" + Guid.NewGuid().ToString(); FaultInjectionRule serviceUnavailableRule = new FaultInjectionRuleBuilder( @@ -2091,9 +2091,9 @@ public async Task CreateItemAsync_WithThinClientEnabledAndCircuitBreakerEnabledA .Build(); List rules = new List { serviceUnavailableRule }; - FaultInjector faultInjector = new FaultInjector(rules); - - Random random = new(); + FaultInjector faultInjector = new FaultInjector(rules); + + Random random = new(); List itemsCleanupList = new(); List preferredRegions = new List { Regions.WestUS, Regions.EastAsia }; CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() @@ -2112,11 +2112,11 @@ public async Task CreateItemAsync_WithThinClientEnabledAndCircuitBreakerEnabledA try { - using CosmosClient cosmosClient = new(connectionString: this.connectionString, clientOptions: cosmosClientOptions); - AccountProperties accountInfo = await cosmosClient.ReadAccountAsync(); - - Assert.IsTrue(cosmosClient.DocumentClient.GlobalEndpointManager.ThinClientReadEndpoints.Count() >= 2); - this.thinClientreadRegionalEndpoints = cosmosClient.DocumentClient.GlobalEndpointManager.ThinClientReadEndpoints; + using CosmosClient cosmosClient = new(connectionString: this.connectionString, clientOptions: cosmosClientOptions); + AccountProperties accountInfo = await cosmosClient.ReadAccountAsync(); + + Assert.IsTrue(cosmosClient.DocumentClient.GlobalEndpointManager.ThinClientReadEndpoints.Count() >= 2); + this.thinClientreadRegionalEndpoints = cosmosClient.DocumentClient.GlobalEndpointManager.ThinClientReadEndpoints; Database database = cosmosClient.GetDatabase(MultiRegionSetupHelpers.dbName); Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); @@ -2189,76 +2189,166 @@ public async Task CreateItemAsync_WithThinClientEnabledAndCircuitBreakerEnabledA finally { Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelCircuitBreakerEnabled, null); - Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, null); - Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, null); + Environment.SetEnvironmentVariable(ConfigurationManager.CircuitBreakerConsecutiveFailureCountForReads, null); + Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, null); await this.TryDeleteItems(itemsList); } - } - + } + [TestMethod] [Owner("ntripician")] - [TestCategory("MultiRegion")] + [TestCategory("MultiRegion")] [Timeout(70000)] public async Task ClinetOverrides0msRequestTimeoutValueForPPAF() { // Arrange. - - // Now that the ppaf enablement flag is returned from gateway, we need to intercept the response and remove the flag from the response, so that - // the environment variable set above is honored. - HttpClientHandlerHelper httpClientHandlerHelper = new HttpClientHandlerHelper() - { - ResponseIntercepter = async (response, request) => - { - string json = await response?.Content?.ReadAsStringAsync(); - if (json.Length > 0 && json.Contains("enablePerPartitionFailoverBehavior")) - { - JObject parsedDatabaseAccountResponse = JObject.Parse(json); - parsedDatabaseAccountResponse.Property("enablePerPartitionFailoverBehavior").Value = "true"; - - HttpResponseMessage interceptedResponse = new() - { - StatusCode = response.StatusCode, - Content = new StringContent(parsedDatabaseAccountResponse.ToString()), - Version = response.Version, - ReasonPhrase = response.ReasonPhrase, - RequestMessage = response.RequestMessage, - }; - - return interceptedResponse; - } - - return response; - }, - }; + + // Now that the ppaf enablement flag is returned from gateway, we need to intercept the response and remove the flag from the response, so that + // the environment variable set above is honored. + HttpClientHandlerHelper httpClientHandlerHelper = new HttpClientHandlerHelper() + { + ResponseIntercepter = async (response, request) => + { + string json = await response?.Content?.ReadAsStringAsync(); + if (json.Length > 0 && json.Contains("enablePerPartitionFailoverBehavior")) + { + JObject parsedDatabaseAccountResponse = JObject.Parse(json); + parsedDatabaseAccountResponse.Property("enablePerPartitionFailoverBehavior").Value = "true"; + + HttpResponseMessage interceptedResponse = new() + { + StatusCode = response.StatusCode, + Content = new StringContent(parsedDatabaseAccountResponse.ToString()), + Version = response.Version, + ReasonPhrase = response.ReasonPhrase, + RequestMessage = response.RequestMessage, + }; + + return interceptedResponse; + } + + return response; + }, + }; List preferredRegions = new List { region1, region2, region3 }; CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() { ConsistencyLevel = ConsistencyLevel.Session, RequestTimeout = TimeSpan.FromSeconds(0), - ApplicationPreferredRegions = preferredRegions, - HttpClientFactory = () => new HttpClient(httpClientHandlerHelper), + ApplicationPreferredRegions = preferredRegions, + HttpClientFactory = () => new HttpClient(httpClientHandlerHelper), }; using CosmosClient cosmosClient = new(connectionString: this.connectionString, clientOptions: cosmosClientOptions); Database database = cosmosClient.GetDatabase(MultiRegionSetupHelpers.dbName); - Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); - - try - { - //request to start document client initiation - _ = await container.ReadItemAsync("id", new PartitionKey("pk1")); - } + Container container = database.GetContainer(MultiRegionSetupHelpers.containerName); + + try + { + //request to start document client initiation + _ = await container.ReadItemAsync("id", new PartitionKey("pk1")); + } catch { } - // Act and Assert. - - CrossRegionHedgingAvailabilityStrategy strat = cosmosClient.DocumentClient.ConnectionPolicy.AvailabilityStrategy as CrossRegionHedgingAvailabilityStrategy; - Assert.IsNotNull(strat); + // Act and Assert. + + CrossRegionHedgingAvailabilityStrategy strat = cosmosClient.DocumentClient.ConnectionPolicy.AvailabilityStrategy as CrossRegionHedgingAvailabilityStrategy; + Assert.IsNotNull(strat); Assert.AreNotEqual(0, strat.Threshold); - } + } + + + [TestMethod] + [TestCategory("MultiRegion")] + [Owner("trivediyash")] + [Description("Scenario: When a document is created, then updated, and finally deleted, the operations must reflect on Change Feed.")] + public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedCFPTests() + { + string testId = "testDoc" + Guid.NewGuid().ToString("N"); + string testPk = "testPk" + Guid.NewGuid().ToString("N"); + + try + { + // Create the document + CosmosIntegrationTestObject createItem = new CosmosIntegrationTestObject + { + Id = testId, + Pk = testPk, + Other = "original test" + }; + + ItemResponse createResponse = await this.container.CreateItemAsync( + createItem, + new PartitionKey(testPk)); + + Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode); + Assert.IsNotNull(createResponse.Resource); + Assert.AreEqual(testId, createResponse.Resource.Id); + Assert.AreEqual(testPk, createResponse.Resource.Pk); + Assert.AreEqual("original test", createResponse.Resource.Other); + + // Wait 1 second to ensure different timestamps + await Task.Delay(1000); + + // Update the document + CosmosIntegrationTestObject updateItem = new CosmosIntegrationTestObject + { + Id = testId, + Pk = testPk, + Other = "test after replace" + }; + + ItemResponse updateResponse = await this.container.ReplaceItemAsync( + updateItem, + testId, + new PartitionKey(testPk)); + + Assert.AreEqual(HttpStatusCode.OK, updateResponse.StatusCode); + Assert.IsNotNull(updateResponse.Resource); + Assert.AreEqual(testId, updateResponse.Resource.Id); + Assert.AreEqual(testPk, updateResponse.Resource.Pk); + Assert.AreEqual("test after replace", updateResponse.Resource.Other); + + // Verify the ETag changed + Assert.AreNotEqual(createResponse.ETag, updateResponse.ETag); + + // Wait 1 second to ensure different timestamps + await Task.Delay(1000); + + // Delete the document + ItemResponse deleteResponse = await this.container.DeleteItemAsync( + testId, + new PartitionKey(testPk)); + + Assert.AreEqual(HttpStatusCode.NoContent, deleteResponse.StatusCode); + + // Verify the document no longer exists + try + { + await this.container.ReadItemAsync(testId, new PartitionKey(testPk)); + Assert.Fail("Document should not exist after deletion"); + } + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + // Expected - document was successfully deleted + } + } + finally + { + // Cleanup in case test failed before deletion + try + { + await this.container.DeleteItemAsync(testId, new PartitionKey(testPk)); + } + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + // Ignore - document already deleted + } + } + } private async Task TryCreateItems(List testItems) { @@ -2300,41 +2390,41 @@ await this.container.DeleteItemAsync( { // Ignore } - } - - public sealed class TestCosmosItem - { - [JsonConstructor] - public TestCosmosItem( - string id, - string pk, - string title, - string email, - string body, - DateTime createdUtc, - DateTime modifiedUtc, - DateTime[] extraDates) - { - this.id = id; - this.pk = pk; - this.title = title; - this.email = email; - this.body = body; - this.CreatedUtc = createdUtc; - this.ModifiedUtc = modifiedUtc; - this.ExtraDates = extraDates; - } - -#pragma warning disable IDE1006 - public string id { get; } - public string pk { get; } - public string title { get; } - public string email { get; } - public string body { get; } -#pragma warning restore IDE1006 // Naming Styles - public DateTime CreatedUtc { get; } - public DateTime ModifiedUtc { get; } - public DateTime[] ExtraDates { get; } + } + + public sealed class TestCosmosItem + { + [JsonConstructor] + public TestCosmosItem( + string id, + string pk, + string title, + string email, + string body, + DateTime createdUtc, + DateTime modifiedUtc, + DateTime[] extraDates) + { + this.id = id; + this.pk = pk; + this.title = title; + this.email = email; + this.body = body; + this.CreatedUtc = createdUtc; + this.ModifiedUtc = modifiedUtc; + this.ExtraDates = extraDates; + } + +#pragma warning disable IDE1006 + public string id { get; } + public string pk { get; } + public string title { get; } + public string email { get; } + public string body { get; } +#pragma warning restore IDE1006 // Naming Styles + public DateTime CreatedUtc { get; } + public DateTime ModifiedUtc { get; } + public DateTime[] ExtraDates { get; } } } } From e6fe51cc3621abbcc5e6740906fe0e3c92614e63 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Thu, 6 Nov 2025 12:10:16 -0800 Subject: [PATCH 23/33] Update code-doc --- .../src/Resource/FullFidelity/ChangeFeedItem.cs | 1 + .../src/Resource/FullFidelity/ChangeFeedMetadata.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs index dc856ef553..5e8ac7d438 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs @@ -58,6 +58,7 @@ class ChangeFeedItem { /// /// The current version of the item for all versions and deletes change feed mode. + /// It is always null for delete change feed operations. /// [JsonProperty(PropertyName = "current")] [JsonPropertyName("current")] diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index bb7ad0ba21..baae832c46 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -90,7 +90,7 @@ class ChangeFeedMetadata /// /// /// For hierarchical partition key containers, the dictionary will contain multiple entries, one for each level of the hierarchy, - /// in the order they were defined in the container's partition key definition. + /// as defined in the container's partition key definition. /// /// /// Example for a single partition key container with partition key path "/tenantId": From 40cb84a5a473bc6fef4f44d8749ca5aa6c4ddf48 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Fri, 7 Nov 2025 14:26:20 -0800 Subject: [PATCH 24/33] Fix tests and update contract --- .../DotNetSDKEncryptionCustomAPI.net6.json | 133 ++++++------------ .../BuilderWithCustomSerializerTests.cs | 23 ++- .../Contracts/DotNetPreviewSDKAPI.net6.json | 73 +++++++--- 3 files changed, 110 insertions(+), 119 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.net6.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.net6.json index 4c56591e41..1806d9725d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.net6.json +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.net6.json @@ -120,54 +120,20 @@ ], "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" + "AsyncStateMachineAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" + "AsyncStateMachineAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { - "Type": "Method", - "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" - ], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { - "Type": "Method", - "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" - ], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { - "Type": "Method", - "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" - ], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { - "Type": "Method", - "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" - ], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -1013,6 +979,18 @@ "Microsoft.Azure.Cosmos.Encryption.Custom.EncryptionOptions;System.Object;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { + "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor get_JsonProcessor()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor get_JsonProcessor();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor JsonProcessor": { + "Type": "Property", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor JsonProcessor;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor get_JsonProcessor();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_JsonProcessor(Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "System.Collections.Generic.IEnumerable`1[System.String] get_PathsToEncrypt()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -1068,6 +1046,13 @@ ], "MethodInfo": "Void set_EncryptionAlgorithm(System.String);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "Void set_JsonProcessor(Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Void set_JsonProcessor(Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void set_PathsToEncrypt(System.Collections.Generic.IEnumerable`1[System.String])[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -1132,54 +1117,20 @@ ], "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" + "AsyncStateMachineAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" + "AsyncStateMachineAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { - "Type": "Method", - "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" - ], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { - "Type": "Method", - "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" - ], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { - "Type": "Method", - "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" - ], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { - "Type": "Method", - "Attributes": [ - "AsyncStateMachineAttribute", - "ObsoleteAttribute" - ], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -1204,26 +1155,22 @@ "Type": "Method", "Attributes": [], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { - "Type": "Method", - "Attributes": [], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { - "Type": "Method", - "Attributes": [], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { - "Type": "Method", + } + }, + "NestedTypes": {} + }, + "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor;System.Enum;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:True;IsClass:False;IsValueType:True;IsNested:False;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "Int32 value__": { + "Type": "Field", "Attributes": [], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "Int32 value__;IsInitOnly:False;IsStatic:False;" }, - "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { - "Type": "Method", + "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor Newtonsoft": { + "Type": "Field", "Attributes": [], - "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor Newtonsoft;IsInitOnly:False;IsStatic:True;" } }, "NestedTypes": {} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 1c749f08b3..d0d0636170 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -24,6 +24,8 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests.CFP.AllVersionsAndDeletes [TestCategory("ChangeFeedProcessor")] public class BuilderWithCustomSerializerTests { + public TestContext TestContext { get; set; } + [TestMethod] [Owner("philipthomas")] [Description("Validating to deserization of ChangeFeedItem with a Delete payload with TimeToLiveExpired set to true.")] @@ -657,6 +659,8 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA } [TestMethod] + [TestCategory("MultiMaster")] + [TestCategory("ChangeFeed")] [Owner("philipthomas-MSFT")] [Description("Scenario: When a document is created, then updated, and finally deleted, there should be 3 changes that will appear for that " + "document when using ChangeFeedProcessor with AllVersionsAndDeletes set as the ChangeFeedMode.")] @@ -664,6 +668,9 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA [DataRow(false)] public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool propertyNameCaseInsensitive) { + bool isMultiMaster = this.TestContext.Properties.ContainsKey("TestCategory") && + this.TestContext.Properties["TestCategory"].ToString().Contains("MultiMaster"); + CosmosClient cosmosClient = TestCommon.CreateCosmosClient((cosmosClientBuilder) => cosmosClientBuilder.WithSystemTextJsonSerializerOptions( new JsonSerializerOptions() @@ -740,9 +747,19 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr Assert.IsNull(deleteChange.Current.id); Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); - Assert.IsNull(deleteChange.Previous); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.Values.FirstOrDefault().ToString()); + + if (isMultiMaster) + { + Assert.IsNull(deleteChange.Previous); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.Values.FirstOrDefault().ToString()); + } + else + { + Assert.IsNull(deleteChange.Previous); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.Values.FirstOrDefault().ToString()); + } Assert.IsTrue(condition: createChange.Metadata.ConflictResolutionTimestamp < replaceChange.Metadata.ConflictResolutionTimestamp, message: "The create operation must happen before the replace operation."); Assert.IsTrue(condition: replaceChange.Metadata.ConflictResolutionTimestamp < deleteChange.Metadata.ConflictResolutionTimestamp, message: "The replace operation must happen before the delete operation."); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.net6.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.net6.json index 01c5457e3d..228784345a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.net6.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.net6.json @@ -87,9 +87,13 @@ ], "MethodInfo": "Boolean get_IsTimeToLiveExpired();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Boolean IsTimeToLiveExpired[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = NullValueHandling.Ignore = 1, PropertyName = \"timeToLiveExpired\")]": { + "Boolean IsTimeToLiveExpired[System.Text.Json.Serialization.JsonIncludeAttribute()]-[System.Text.Json.Serialization.JsonPropertyNameAttribute(\"timeToLiveExpired\")]-[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = NullValueHandling.Ignore = 1, PropertyName = \"timeToLiveExpired\")]": { "Type": "Property", - "Attributes": [], + "Attributes": [ + "JsonIncludeAttribute", + "JsonPropertyAttribute", + "JsonPropertyNameAttribute" + ], "MethodInfo": "Boolean IsTimeToLiveExpired;CanRead:True;CanWrite:True;Boolean get_IsTimeToLiveExpired();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Int64 get_Lsn()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { @@ -106,14 +110,22 @@ ], "MethodInfo": "Int64 get_PreviousLsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Int64 Lsn[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = NullValueHandling.Ignore = 1, PropertyName = \"lsn\")]": { + "Int64 Lsn[System.Text.Json.Serialization.JsonIncludeAttribute()]-[System.Text.Json.Serialization.JsonPropertyNameAttribute(\"lsn\")]-[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = NullValueHandling.Ignore = 1, PropertyName = \"lsn\")]": { "Type": "Property", - "Attributes": [], + "Attributes": [ + "JsonIncludeAttribute", + "JsonPropertyAttribute", + "JsonPropertyNameAttribute" + ], "MethodInfo": "Int64 Lsn;CanRead:True;CanWrite:True;Int64 get_Lsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Int64 PreviousLsn[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = NullValueHandling.Ignore = 1, PropertyName = \"previousImageLSN\")]": { + "Int64 PreviousLsn[System.Text.Json.Serialization.JsonIncludeAttribute()]-[System.Text.Json.Serialization.JsonPropertyNameAttribute(\"previousImageLSN\")]-[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = NullValueHandling.Ignore = 1, PropertyName = \"previousImageLSN\")]": { "Type": "Property", - "Attributes": [], + "Attributes": [ + "JsonIncludeAttribute", + "JsonPropertyAttribute", + "JsonPropertyNameAttribute" + ], "MethodInfo": "Int64 PreviousLsn;CanRead:True;CanWrite:True;Int64 get_PreviousLsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { @@ -123,34 +135,45 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = NullValueHandling.Ignore = 1, PropertyName = \"operationType\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Newtonsoft.Json.Converters.StringEnumConverter))]": { + "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType[Newtonsoft.Json.JsonConverterAttribute(typeof(Newtonsoft.Json.Converters.StringEnumConverter))]-[System.Text.Json.Serialization.JsonIncludeAttribute()]-[System.Text.Json.Serialization.JsonPropertyNameAttribute(\"operationType\")]-[System.Text.Json.Serialization.JsonConverterAttribute(typeof(System.Text.Json.Serialization.JsonStringEnumConverter))]-[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = NullValueHandling.Ignore = 1, PropertyName = \"operationType\")]": { "Type": "Property", - "Attributes": [], + "Attributes": [ + "JsonConverterAttribute", + "JsonConverterAttribute", + "JsonIncludeAttribute", + "JsonPropertyAttribute", + "JsonPropertyNameAttribute" + ], "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.DateTime ConflictResolutionTimestamp[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = NullValueHandling.Ignore = 1, PropertyName = \"crts\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Microsoft.Azure.Documents.UnixDateTimeConverter))]": { - "Type": "Property", + "System.Collections.Generic.Dictionary`2[System.String,System.Object] get_PartitionKey()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", "Attributes": [ "CompilerGeneratedAttribute" ], - "MethodInfo": "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.Collections.Generic.Dictionary`2[System.String,System.Object] get_PartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.Dictionary`2[System.String,System.Object] PartitionKey": { + "System.Collections.Generic.Dictionary`2[System.String,System.Object] PartitionKey[System.Text.Json.Serialization.JsonIncludeAttribute()]-[System.Text.Json.Serialization.JsonPropertyNameAttribute(\"partitionKey\")]-[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = NullValueHandling.Ignore = 1, PropertyName = \"partitionKey\")]": { "Type": "Property", - "Attributes": [], + "Attributes": [ + "JsonIncludeAttribute", + "JsonPropertyAttribute", + "JsonPropertyNameAttribute" + ], "MethodInfo": "System.Collections.Generic.Dictionary`2[System.String,System.Object] PartitionKey;CanRead:True;CanWrite:True;System.Collections.Generic.Dictionary`2[System.String,System.Object] get_PartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.DateTime ConflictResolutionTimestamp": { + "System.Nullable`1[System.DateTime] ConflictResolutionTimestamp[System.Text.Json.Serialization.JsonIgnoreAttribute()]-[Newtonsoft.Json.JsonIgnoreAttribute()]": { "Type": "Property", - "Attributes": [], - "MethodInfo": "System.DateTime ConflictResolutionTimestamp;CanRead:True;CanWrite:True;System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.DateTime get_ConflictResolutionTimestamp()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", "Attributes": [ - "CompilerGeneratedAttribute" + "JsonIgnoreAttribute", + "JsonIgnoreAttribute" ], - "MethodInfo": "System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.Nullable`1[System.DateTime] ConflictResolutionTimestamp;CanRead:True;CanWrite:False;System.Nullable`1[System.DateTime] get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Nullable`1[System.DateTime] get_ConflictResolutionTimestamp()": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Nullable`1[System.DateTime] get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "System.String get_Id()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", @@ -159,9 +182,13 @@ ], "MethodInfo": "System.String get_Id();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.String Id": { + "System.String Id[System.Text.Json.Serialization.JsonIncludeAttribute()]-[System.Text.Json.Serialization.JsonPropertyNameAttribute(\"id\")]-[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = NullValueHandling.Ignore = 1, PropertyName = \"id\")]": { "Type": "Property", - "Attributes": [], + "Attributes": [ + "JsonIncludeAttribute", + "JsonPropertyAttribute", + "JsonPropertyNameAttribute" + ], "MethodInfo": "System.String Id;CanRead:True;CanWrite:True;System.String get_Id();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Void .ctor()": { From 342755916f53fbf71ef1d1c712ee5814c82bf7dd Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Mon, 10 Nov 2025 09:57:37 -0800 Subject: [PATCH 25/33] Update connection string for multi-master mode --- .../BuilderWithCustomSerializerTests.cs | 81 ++++++++++--------- 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index d0d0636170..2fadafc0a2 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -565,43 +565,43 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA foreach (ChangeFeedItem change in docs) { - if (change.Metadata.OperationType == ChangeFeedOperationType.Create) - { - // current - Assert.AreEqual(expected: "1", actual: change.Current.id.ToString()); - Assert.AreEqual(expected: "1", actual: change.Current.pk.ToString()); - Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Current.description.ToString()); - Assert.AreEqual(expected: ttlInSeconds, actual: change.Current.ttl); - - // metadata - Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); - Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); - Assert.IsFalse(change.Metadata.IsTimeToLiveExpired); - Assert.IsNull(change.Metadata.Id); - Assert.IsNull(change.Metadata.PartitionKey); - - // previous - Assert.IsNull(change.Previous); - } - else if (change.Metadata.OperationType == ChangeFeedOperationType.Delete) - { - // current - Assert.IsNull(change.Current.id); - - // metadata - Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); - Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); - Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); - Assert.AreEqual(expected: "1", actual: change.Metadata.Id.ToString()); - Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.Values.FirstOrDefault()); - - // previous - Assert.IsNull(change.Previous); - - // stop after reading delete since it is the last document in feed. - stopwatch.Stop(); - allDocsProcessed.Set(); - } + if (change.Metadata.OperationType == ChangeFeedOperationType.Create) + { + // current + Assert.AreEqual(expected: "1", actual: change.Current.id.ToString()); + Assert.AreEqual(expected: "1", actual: change.Current.pk.ToString()); + Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Current.description.ToString()); + Assert.AreEqual(expected: ttlInSeconds, actual: change.Current.ttl); + + // metadata + Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); + Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); + Assert.IsFalse(change.Metadata.IsTimeToLiveExpired); + Assert.IsNull(change.Metadata.Id); + Assert.IsNull(change.Metadata.PartitionKey); + + // previous + Assert.IsNull(change.Previous); + } + else if (change.Metadata.OperationType == ChangeFeedOperationType.Delete) + { + // current + Assert.IsNull(change.Current.id); + + // metadata + Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); + Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); + Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); + Assert.AreEqual(expected: "1", actual: change.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.Values.FirstOrDefault()); + + // previous + Assert.IsNull(change.Previous); + + // stop after reading delete since it is the last document in feed. + stopwatch.Stop(); + allDocsProcessed.Set(); + } else { Assert.Fail("Invalid operation."); @@ -671,13 +671,18 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr bool isMultiMaster = this.TestContext.Properties.ContainsKey("TestCategory") && this.TestContext.Properties["TestCategory"].ToString().Contains("MultiMaster"); + string accountEndpoint = isMultiMaster ? + TestCommon.GetMultiRegionConnectionString() : + null; + CosmosClient cosmosClient = TestCommon.CreateCosmosClient((cosmosClientBuilder) => cosmosClientBuilder.WithSystemTextJsonSerializerOptions( new JsonSerializerOptions() { PropertyNameCaseInsensitive = propertyNameCaseInsensitive }), - useCustomSeralizer: false); + useCustomSeralizer: false, + accountEndpointOverride: accountEndpoint); Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(id: Guid.NewGuid().ToString()); Container leaseContainer = await database.CreateContainerIfNotExistsAsync(containerProperties: new ContainerProperties(id: "leases", partitionKeyPath: "/id")); From 95a0f3e609e04cae643d0785eb09160364794093 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Mon, 10 Nov 2025 10:53:13 -0800 Subject: [PATCH 26/33] Refactor test --- .../BuilderWithCustomSerializerTests.cs | 115 ++++++++++-------- 1 file changed, 64 insertions(+), 51 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 2fadafc0a2..4f2ca52772 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -24,8 +24,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests.CFP.AllVersionsAndDeletes [TestCategory("ChangeFeedProcessor")] public class BuilderWithCustomSerializerTests { - public TestContext TestContext { get; set; } - [TestMethod] [Owner("philipthomas")] [Description("Validating to deserization of ChangeFeedItem with a Delete payload with TimeToLiveExpired set to true.")] @@ -559,54 +557,54 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA ChangeFeedProcessor processor = monitoredContainer .GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(processorName: "processor", onChangesDelegate: (ChangeFeedProcessorContext context, IReadOnlyCollection> docs, CancellationToken token) => { - // NOTE(philipthomas-MSFT): Please allow these Logger.LogLine because TTL on items will purge at random times so I am using this to test when ran locally using emulator. + // NOTE(philipthomas-MSFT): Please allow these Logger.LogLine because TTL on items will purge at random times so I am using this to test when ran locally using emulator. - Logger.LogLine($"@ {DateTime.Now}, {nameof(stopwatch)} -> CFP AVAD took '{stopwatch.ElapsedMilliseconds}' to read document CRUD in feed."); + Logger.LogLine($"@ {DateTime.Now}, {nameof(stopwatch)} -> CFP AVAD took '{stopwatch.ElapsedMilliseconds}' to read document CRUD in feed."); - foreach (ChangeFeedItem change in docs) + foreach (ChangeFeedItem change in docs) + { + if (change.Metadata.OperationType == ChangeFeedOperationType.Create) { - if (change.Metadata.OperationType == ChangeFeedOperationType.Create) - { - // current - Assert.AreEqual(expected: "1", actual: change.Current.id.ToString()); - Assert.AreEqual(expected: "1", actual: change.Current.pk.ToString()); - Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Current.description.ToString()); - Assert.AreEqual(expected: ttlInSeconds, actual: change.Current.ttl); - - // metadata - Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); - Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); - Assert.IsFalse(change.Metadata.IsTimeToLiveExpired); - Assert.IsNull(change.Metadata.Id); - Assert.IsNull(change.Metadata.PartitionKey); - - // previous - Assert.IsNull(change.Previous); - } - else if (change.Metadata.OperationType == ChangeFeedOperationType.Delete) - { - // current - Assert.IsNull(change.Current.id); - - // metadata - Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); - Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); - Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); - Assert.AreEqual(expected: "1", actual: change.Metadata.Id.ToString()); - Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.Values.FirstOrDefault()); - - // previous - Assert.IsNull(change.Previous); - - // stop after reading delete since it is the last document in feed. - stopwatch.Stop(); - allDocsProcessed.Set(); - } - else - { - Assert.Fail("Invalid operation."); - } + // current + Assert.AreEqual(expected: "1", actual: change.Current.id.ToString()); + Assert.AreEqual(expected: "1", actual: change.Current.pk.ToString()); + Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Current.description.ToString()); + Assert.AreEqual(expected: ttlInSeconds, actual: change.Current.ttl); + + // metadata + Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); + Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); + Assert.IsFalse(change.Metadata.IsTimeToLiveExpired); + Assert.IsNull(change.Metadata.Id); + Assert.IsNull(change.Metadata.PartitionKey); + + // previous + Assert.IsNull(change.Previous); } + else if (change.Metadata.OperationType == ChangeFeedOperationType.Delete) + { + // current + Assert.IsNull(change.Current.id); + + // metadata + Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); + Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); + Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); + Assert.AreEqual(expected: "1", actual: change.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.Values.FirstOrDefault()); + + // previous + Assert.IsNull(change.Previous); + + // stop after reading delete since it is the last document in feed. + stopwatch.Stop(); + allDocsProcessed.Set(); + } + else + { + Assert.Fail("Invalid operation."); + } + } return Task.CompletedTask; }) @@ -659,18 +657,33 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA } [TestMethod] - [TestCategory("MultiMaster")] [TestCategory("ChangeFeed")] [Owner("philipthomas-MSFT")] [Description("Scenario: When a document is created, then updated, and finally deleted, there should be 3 changes that will appear for that " + - "document when using ChangeFeedProcessor with AllVersionsAndDeletes set as the ChangeFeedMode.")] + "document when using ChangeFeedProcessor with AllVersionsAndDeletes set as the ChangeFeedMode. This test runs against the Cosmos DB Emulator" + + " which has enablePreviousImageForDeleteInFFCF set to true.")] [DataRow(true)] [DataRow(false)] - public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool propertyNameCaseInsensitive) + public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsyncEmulator(bool propertyNameCaseInsensitive) { - bool isMultiMaster = this.TestContext.Properties.ContainsKey("TestCategory") && - this.TestContext.Properties["TestCategory"].ToString().Contains("MultiMaster"); + await this.WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(propertyNameCaseInsensitive); + } + + [TestMethod] + [TestCategory("MultiMaster")] + [Owner("philipthomas-MSFT")] + [Description("Scenario: When a document is created, then updated, and finally deleted, there should be 3 changes that will appear for that " + + "document when using ChangeFeedProcessor with AllVersionsAndDeletes set as the ChangeFeedMode. This test runs against a live multi-region" + + " Cosmos DB account which has does not have enablePreviousImageForDeleteInFFCF set.")] + [DataRow(true)] + [DataRow(false)] + public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsyncLiveAccount(bool propertyNameCaseInsensitive) + { + await this.WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(propertyNameCaseInsensitive, true); + } + private async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool propertyNameCaseInsensitive, bool isMultiMaster = false) + { string accountEndpoint = isMultiMaster ? TestCommon.GetMultiRegionConnectionString() : null; From ce56009e83713805e2fbd41a4c062232c48f2358 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Mon, 10 Nov 2025 12:18:46 -0800 Subject: [PATCH 27/33] Update cosmosclient createion --- .../BuilderWithCustomSerializerTests.cs | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 4f2ca52772..566952e995 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -684,19 +684,21 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsyncLiveAcco private async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool propertyNameCaseInsensitive, bool isMultiMaster = false) { - string accountEndpoint = isMultiMaster ? - TestCommon.GetMultiRegionConnectionString() : - null; + (string defaultEndpoint, string authKey) = TestCommon.GetAccountInfo(); + string accountEndpoint = TestCommon.GetMultiRegionConnectionString(); - CosmosClient cosmosClient = TestCommon.CreateCosmosClient((cosmosClientBuilder) => - cosmosClientBuilder.WithSystemTextJsonSerializerOptions( - new JsonSerializerOptions() - { - PropertyNameCaseInsensitive = propertyNameCaseInsensitive - }), - useCustomSeralizer: false, - accountEndpointOverride: accountEndpoint); + CosmosClientOptions options = new CosmosClientOptions() + { + UseSystemTextJsonSerializerWithOptions = new JsonSerializerOptions() + { + PropertyNameCaseInsensitive = propertyNameCaseInsensitive + } + }; + CosmosClient cosmosClient = isMultiMaster + ? new CosmosClient(accountEndpoint, options) + : new CosmosClient(accountEndpoint, authKey, options); + Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(id: Guid.NewGuid().ToString()); Container leaseContainer = await database.CreateContainerIfNotExistsAsync(containerProperties: new ContainerProperties(id: "leases", partitionKeyPath: "/id")); ContainerInternal monitoredContainer = await this.CreateMonitoredContainer(ChangeFeedMode.AllVersionsAndDeletes, database); From 3a2a6a31ab06cc42b16041d116b3fea17206fd47 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Mon, 10 Nov 2025 14:58:47 -0800 Subject: [PATCH 28/33] Fix endpoint for emulator --- .../AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 566952e995..2120303259 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -697,7 +697,7 @@ private async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool p CosmosClient cosmosClient = isMultiMaster ? new CosmosClient(accountEndpoint, options) - : new CosmosClient(accountEndpoint, authKey, options); + : new CosmosClient(defaultEndpoint, authKey, options); Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(id: Guid.NewGuid().ToString()); Container leaseContainer = await database.CreateContainerIfNotExistsAsync(containerProperties: new ContainerProperties(id: "leases", partitionKeyPath: "/id")); From 8f90e979cf1fc19d04314175e96f2d5348175f95 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Mon, 10 Nov 2025 16:18:24 -0800 Subject: [PATCH 29/33] Fix test for emulator --- .../AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 2120303259..55a20ffbd2 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -776,7 +776,7 @@ private async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool p } else { - Assert.IsNull(deleteChange.Previous); + Assert.IsNotNull(deleteChange.Previous); Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.Values.FirstOrDefault().ToString()); } From 605c37514cbab3d256bc4c3257ab71685b71d8d3 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Tue, 11 Nov 2025 14:52:24 -0800 Subject: [PATCH 30/33] Update builder test --- .../CFP/AllVersionsAndDeletes/BuilderTests.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs index 90b7d8c357..fe549fa145 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs @@ -145,7 +145,7 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA [TestMethod] [Owner("philipthomas-MSFT")] [Description("Scenario: When a document is created, then updated, and finally deleted, there should be 3 changes that will appear for that " + - "document when using ChangeFeedProcessor with AllVersionsAndDeletes set as the ChangeFeedMode.")] + "document when using ChangeFeedProcessor with AllVersionsAndDeletes set as the ChangeFeedMode and enablePreviousImageForDeleteInFFCF true")] public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() { ContainerInternal monitoredContainer = await this.CreateMonitoredContainer(ChangeFeedMode.AllVersionsAndDeletes); @@ -215,7 +215,10 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.Values.FirstOrDefault()); Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); - Assert.IsNull(deleteChange.Previous); + Assert.IsNotNull(deleteChange.Previous); + Assert.AreEqual(expected: "1", actual: deleteChange.Previous.id.ToString()); + Assert.AreEqual(expected: "1", actual: deleteChange.Previous.pk.ToString()); + Assert.AreEqual(expected: "test after replace", actual: deleteChange.Previous.description.ToString()); Assert.IsTrue(condition: createChange.Metadata.ConflictResolutionTimestamp < replaceChange.Metadata.ConflictResolutionTimestamp, message: "The create operation must happen before the replace operation."); Assert.IsTrue(condition: replaceChange.Metadata.ConflictResolutionTimestamp < deleteChange.Metadata.ConflictResolutionTimestamp, message: "The replace operation must happen before the delete operation."); From 9e580cdf149b167bfa8ffb3e386c4f31e18828be Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Wed, 12 Nov 2025 16:50:07 -0800 Subject: [PATCH 31/33] Update builder test --- .../BuilderWithCustomSerializerTests.cs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 55a20ffbd2..cef25fd9fc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -695,9 +695,9 @@ private async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool p } }; - CosmosClient cosmosClient = isMultiMaster - ? new CosmosClient(accountEndpoint, options) - : new CosmosClient(defaultEndpoint, authKey, options); + CosmosClient cosmosClient = //isMultiMaster + //? new CosmosClient(accountEndpoint, options) + new CosmosClient(defaultEndpoint, authKey, options); Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(id: Guid.NewGuid().ToString()); Container leaseContainer = await database.CreateContainerIfNotExistsAsync(containerProperties: new ContainerProperties(id: "leases", partitionKeyPath: "/id")); @@ -767,20 +767,19 @@ private async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool p Assert.IsNull(deleteChange.Current.id); Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); - + Assert.Fail(); if (isMultiMaster) { Assert.IsNull(deleteChange.Previous); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.Values.FirstOrDefault().ToString()); } else { Assert.IsNotNull(deleteChange.Previous); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.Values.FirstOrDefault().ToString()); + Assert.AreEqual(expected: "1", actual: deleteChange.Previous.id.ToString()); } - + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.Values.FirstOrDefault().ToString()); + Assert.IsTrue(condition: createChange.Metadata.ConflictResolutionTimestamp < replaceChange.Metadata.ConflictResolutionTimestamp, message: "The create operation must happen before the replace operation."); Assert.IsTrue(condition: replaceChange.Metadata.ConflictResolutionTimestamp < deleteChange.Metadata.ConflictResolutionTimestamp, message: "The replace operation must happen before the delete operation."); Assert.IsTrue(condition: createChange.Metadata.Lsn < replaceChange.Metadata.Lsn, message: "The create operation must happen before the replace operation."); @@ -948,7 +947,7 @@ private async Task CreateMonitoredContainer( if (changeFeedMode == ChangeFeedMode.AllVersionsAndDeletes) { - properties.ChangeFeedPolicy.FullFidelityRetention = TimeSpan.FromMinutes(5); + //properties.ChangeFeedPolicy.FullFidelityRetention = TimeSpan.FromMinutes(5); properties.DefaultTimeToLive = -1; } From 16f47523cd82f41bf11f47e6a17e7c2be3dbb2c9 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Wed, 12 Nov 2025 17:47:57 -0800 Subject: [PATCH 32/33] Update builder test --- .../BuilderWithCustomSerializerTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index cef25fd9fc..f1d57b460f 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -695,9 +695,9 @@ private async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool p } }; - CosmosClient cosmosClient = //isMultiMaster - //? new CosmosClient(accountEndpoint, options) - new CosmosClient(defaultEndpoint, authKey, options); + CosmosClient cosmosClient = isMultiMaster + ? new CosmosClient(accountEndpoint, options) + : new CosmosClient(defaultEndpoint, authKey, options); Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(id: Guid.NewGuid().ToString()); Container leaseContainer = await database.CreateContainerIfNotExistsAsync(containerProperties: new ContainerProperties(id: "leases", partitionKeyPath: "/id")); From 3a945f637dbd13aa957f775cb5a021ad1a90ba05 Mon Sep 17 00:00:00 2001 From: Yash Trivedi Date: Wed, 12 Nov 2025 18:07:14 -0800 Subject: [PATCH 33/33] Update builder test --- .../AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index f1d57b460f..242784b4cc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -767,7 +767,7 @@ private async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool p Assert.IsNull(deleteChange.Current.id); Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); - Assert.Fail(); + if (isMultiMaster) { Assert.IsNull(deleteChange.Previous);