From f629e508b536d6362790671581de366d87329dc2 Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Tue, 3 Dec 2024 19:43:00 -0800 Subject: [PATCH 01/18] Update ChangeFeedItem to include id and pk metadata --- .../Resource/FullFidelity/ChangeFeedItem.cs | 6 ++--- .../FullFidelity/ChangeFeedMetadata.cs | 17 ++++++++++++-- .../FullFidelity/ChangeFeedMetadataFields.cs | 2 ++ .../Converters/ChangeFeedMetadataConverter.cs | 22 +++++++++++++++++++ 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs index 417cc3b1b6..dc856ef553 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs @@ -57,21 +57,21 @@ namespace Microsoft.Azure.Cosmos class ChangeFeedItem { /// - /// The full fidelity change feed current item. + /// The current version of the item for all versions and deletes change feed mode. /// [JsonProperty(PropertyName = "current")] [JsonPropertyName("current")] public T Current { get; set; } /// - /// The full fidelity change feed metadata. + /// The item metadata for all versions and deletes change feed mode. /// [JsonProperty(PropertyName = "metadata", NullValueHandling = NullValueHandling.Ignore)] [JsonPropertyName("metadata")] public ChangeFeedMetadata Metadata { get; set; } /// - /// For delete operations, previous image is always going to be provided. The previous image on replace operations is not going to be exposed by default and requires account-level or container-level opt-in. + /// The previous version of the item for all versions and deletes change feed mode. The previous version on delete and replace operations is not exposed by default and requires container-level opt-in. Refer to https://aka.ms/cosmosdb-change-feed-deletes for more information. /// [JsonProperty(PropertyName = "previous", NullValueHandling = NullValueHandling.Ignore)] [JsonPropertyName("previous")] diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index 1dae4f1e1b..b507644d9f 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos { using System; + using System.Collections.Generic; using System.Text.Json; using Microsoft.Azure.Cosmos.Resource.FullFidelity; using Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters; @@ -21,7 +22,7 @@ namespace Microsoft.Azure.Cosmos #else internal #endif - class ChangeFeedMetadata + class ChangeFeedMetadata { /// /// The change's conflict resolution timestamp. @@ -50,9 +51,21 @@ class ChangeFeedMetadata public long PreviousLsn { get; internal set; } /// - /// Used to distinquish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents). + /// Used to distinguish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents). /// [JsonProperty(PropertyName = ChangeFeedMetadataFields.TimeToLiveExpired, NullValueHandling = NullValueHandling.Ignore)] public bool IsTimeToLiveExpired { get; internal set; } + + /// + /// The id of the previous item version. Used for delete operations only. + /// + [JsonProperty(PropertyName = ChangeFeedMetadataFields.DeletedItemId, NullValueHandling = NullValueHandling.Ignore)] + public string DeletedItemId { get; internal set; } + + /// + /// The partition key of the previous item version. Dictionary Key is the partition key property name and Dictionary Value is the partition key property value. Used for delete operations only. + /// + [JsonProperty(PropertyName = ChangeFeedMetadataFields.DeletedItemPartitionKey, NullValueHandling = NullValueHandling.Ignore)] + public Dictionary DeletedItemPartitionKey { get; internal set; } } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs index db39a386a9..d45bf1bd44 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs @@ -11,5 +11,7 @@ internal class ChangeFeedMetadataFields public const string OperationType = "operationType"; public const string PreviousImageLSN = "previousImageLSN"; public const string TimeToLiveExpired = "timeToLiveExpired"; + public const string DeletedItemId = "id"; + public const string DeletedItemPartitionKey = "partitionKey"; } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs index 0b5056051a..f448772ee2 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters { using System; + using System.Collections.Generic; using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; @@ -56,6 +57,19 @@ public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToCo { metadata.PreviousLsn = property.Value.GetInt64(); } + else if (property.NameEquals(ChangeFeedMetadataFields.DeletedItemId)) + { + metadata.DeletedItemId = property.Value.GetString(); + } + else if (property.NameEquals(ChangeFeedMetadataFields.DeletedItemPartitionKey)) + { + Dictionary partitionKey = new Dictionary(); + foreach (JsonProperty pk in property.Value.EnumerateObject()) + { + partitionKey.Add(pk.Name, pk.Value.GetString()); + } + metadata.DeletedItemPartitionKey = partitionKey; + } } return metadata; @@ -75,6 +89,14 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json writer.WriteNumber(ChangeFeedMetadataFields.Lsn, value.Lsn); writer.WriteString(ChangeFeedMetadataFields.OperationType, value.OperationType.ToString()); writer.WriteNumber(ChangeFeedMetadataFields.PreviousImageLSN, value.PreviousLsn); + writer.WriteString(ChangeFeedMetadataFields.DeletedItemId, value.DeletedItemId); + + writer.WriteStartObject("partitionKey"); + foreach (KeyValuePair kvp in value.DeletedItemPartitionKey) + { + writer.WriteString(kvp.Key, kvp.Value); + } + writer.WriteEndObject(); writer.WriteEndObject(); } From 970b689ef79490295cdd0635488da2e4d9945277 Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Thu, 5 Dec 2024 14:55:58 -0800 Subject: [PATCH 02/18] update emulator tests for avad delete operations --- .../CFP/AllVersionsAndDeletes/BuilderTests.cs | 10 ++++++++++ .../BuilderWithCustomSerializerTests.cs | 16 ++++++++++++++++ .../FeedToken/ChangeFeedIteratorCoreTests.cs | 2 ++ 3 files changed, 28 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs index 669c6bd194..c5c96b0a22 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs @@ -84,6 +84,9 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); // previous + Assert.AreEqual(expected: "1", actual: change.Metadata.DeletedItemId.ToString()); + change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString()); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString()); @@ -155,6 +158,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() ChangeFeedProcessor processor = monitoredContainer .GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(processorName: "processor", onChangesDelegate: (ChangeFeedProcessorContext context, IReadOnlyCollection> docs, CancellationToken token) => { + string metadataId = default; + string metadataPk = default; string id = default; string pk = default; string description = default; @@ -171,6 +176,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() } else { + metadataId = change.Metadata.DeletedItemId.ToString(); + change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out metadataPk).ToString(); id = change.Previous.id.ToString(); pk = change.Previous.pk.ToString(); description = change.Previous.description.ToString(); @@ -211,6 +218,9 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() ChangeFeedItem deleteChange = docs.ElementAt(2); Assert.IsNull(deleteChange.Current.id); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.DeletedItemId.ToString()); + deleteChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); Assert.IsNotNull(deleteChange.Previous); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index a6780e4409..08cf70a150 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -92,6 +92,9 @@ static void ValidateDeserialization(List> activitie Assert.IsTrue(deletedChange.Metadata.IsTimeToLiveExpired); Assert.IsNotNull(deletedChange.Previous); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: deletedChange.Previous.description); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.DeletedItemId.ToString()); + deletedChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: "1", actual: deletedChange.Previous.id); Assert.AreEqual(expected: 5, actual: deletedChange.Previous.ttl); } @@ -295,6 +298,9 @@ static void ValidateDeserialization(List> activitie Assert.IsFalse(deletedChange.Metadata.IsTimeToLiveExpired); Assert.IsNotNull(deletedChange.Previous); Assert.AreEqual(expected: "test after replace", actual: deletedChange.Previous.description); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.DeletedItemId.ToString()); + deletedChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: "1", actual: deletedChange.Previous.id); Assert.AreEqual(expected: 0, actual: deletedChange.Previous.ttl); } @@ -416,6 +422,9 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); // previous + Assert.AreEqual(expected: "1", actual: change.Metadata.DeletedItemId.ToString()); + change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString()); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString()); @@ -508,6 +517,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr { Logger.LogLine($"@ {DateTime.Now}, {nameof(docs)} -> {System.Text.Json.JsonSerializer.Serialize(docs)}"); + string metadataId = default; + string metadataPk = default; string id = default; string pk = default; string description = default; @@ -522,6 +533,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr } else { + metadataId = change.Metadata.DeletedItemId.ToString(); + change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out metadataPk).ToString(); id = change.Previous.id.ToString(); pk = change.Previous.pk.ToString(); description = change.Previous.description.ToString(); @@ -565,6 +578,9 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); Assert.IsNotNull(deleteChange.Previous); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.DeletedItemId.ToString()); + deleteChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); + Assert.AreEqual(expected: "1", actual: partitionKey); Assert.AreEqual(expected: "1", actual: deleteChange.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: deleteChange.Previous.pk.ToString()); Assert.AreEqual(expected: "test after replace", actual: deleteChange.Previous.description.ToString()); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs index a43d86faf5..f4cc9c2ab5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs @@ -896,6 +896,7 @@ private async Task ValidateChangeFeedIteratorCore_WithQuery( foreach (ChangeFeedItem item in feedResponse) { + Assert.AreEqual(expected: "id3", actual: item.Metadata.DeletedItemId.ToString()); Assert.AreEqual("id3", item.Previous.Id); Assert.AreEqual(ChangeFeedOperationType.Delete, item.Metadata.OperationType); } @@ -1094,6 +1095,7 @@ public async Task ChangeFeedIteratorCore_FeedRange_VerifyingWireFormatTests() Assert.AreNotEqual(notExpected: default, actual: deleteOperation.Metadata.Lsn); Assert.AreNotEqual(notExpected: default, actual: deleteOperation.Metadata.PreviousLsn); Assert.IsNotNull(deleteOperation.Previous); + Assert.AreEqual(expected: id, actual: deleteOperation.Metadata.DeletedItemId.ToString()); Assert.AreEqual(expected: id, actual: deleteOperation.Previous.Id); Assert.AreEqual(expected: "205 16th St NW", actual: deleteOperation.Previous.Line1); Assert.AreEqual(expected: "Atlanta", actual: deleteOperation.Previous.City); From 1823f65a7446e63e98f78cc8ecf1e22a63bcae87 Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Tue, 10 Dec 2024 18:00:14 -0800 Subject: [PATCH 03/18] add output of UpdateContracts script --- .../Contracts/DotNetPreviewSDKAPI.json | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index f5ed83ba12..2a48f3ea62 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -137,6 +137,20 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Collections.Generic.Dictionary`2[System.String,System.String] DeletedItemPartitionKey[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"partitionKey\")]": { + "Type": "Property", + "Attributes": [ + "JsonPropertyAttribute" + ], + "MethodInfo": "System.Collections.Generic.Dictionary`2[System.String,System.String] DeletedItemPartitionKey;CanRead:True;CanWrite:True;System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "System.DateTime ConflictResolutionTimestamp[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"crts\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Microsoft.Azure.Documents.UnixDateTimeConverter))]": { "Type": "Property", "Attributes": [ @@ -152,6 +166,20 @@ ], "MethodInfo": "System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.String DeletedItemId[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"id\")]": { + "Type": "Property", + "Attributes": [ + "JsonPropertyAttribute" + ], + "MethodInfo": "System.String DeletedItemId;CanRead:True;CanWrite:True;System.String get_DeletedItemId();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.String get_DeletedItemId()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "System.String get_DeletedItemId();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor()": { "Type": "Constructor", "Attributes": [], From 1e66f3b27ba129793976c2e4411d69da8b65c8e3 Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Wed, 11 Dec 2024 17:16:25 -0800 Subject: [PATCH 04/18] undo encryption contracts changes --- .../DotNetSDKEncryptionCustomAPI.json | 258 +++++++----------- 1 file changed, 93 insertions(+), 165 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json index 35cef3bfdc..ccb4b56fff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json @@ -1,111 +1,5 @@ { "Subclasses": { - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions;System.Object;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { - "Subclasses": {}, - "Members": { - "CompressionAlgorithm Algorithm": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "CompressionAlgorithm Algorithm;CanRead:True;CanWrite:True;CompressionAlgorithm get_Algorithm();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_Algorithm(CompressionAlgorithm);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "CompressionAlgorithm get_Algorithm()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "CompressionAlgorithm get_Algorithm();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Int32 get_MinimalCompressedLength()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Int32 get_MinimalCompressedLength();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Int32 MinimalCompressedLength": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "Int32 MinimalCompressedLength;CanRead:True;CanWrite:True;Int32 get_MinimalCompressedLength();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_MinimalCompressedLength(Int32);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions+CompressionAlgorithm": { - "Type": "NestedType", - "Attributes": [], - "MethodInfo": null - }, - "System.IO.Compression.CompressionLevel CompressionLevel": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "System.IO.Compression.CompressionLevel CompressionLevel;CanRead:True;CanWrite:True;System.IO.Compression.CompressionLevel get_CompressionLevel();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_CompressionLevel(System.IO.Compression.CompressionLevel);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "System.IO.Compression.CompressionLevel get_CompressionLevel()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "System.IO.Compression.CompressionLevel get_CompressionLevel();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Void .ctor()": { - "Type": "Constructor", - "Attributes": [], - "MethodInfo": "[Void .ctor(), Void .ctor()]" - }, - "Void set_Algorithm(CompressionAlgorithm)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_Algorithm(CompressionAlgorithm);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Void set_CompressionLevel(System.IO.Compression.CompressionLevel)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_CompressionLevel(System.IO.Compression.CompressionLevel);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Void set_MinimalCompressedLength(Int32)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_MinimalCompressedLength(Int32);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - } - }, - "NestedTypes": { - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions+CompressionAlgorithm;System.Enum;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:True;IsClass:False;IsValueType:True;IsNested:True;IsGenericType:False;IsSerializable:True": { - "Subclasses": {}, - "Members": { - "CompressionAlgorithm None": { - "Type": "Field", - "Attributes": [], - "MethodInfo": "CompressionAlgorithm None;IsInitOnly:False;IsStatic:True;" - }, - "Int32 value__": { - "Type": "Field", - "Attributes": [], - "MethodInfo": "Int32 value__;IsInitOnly:False;IsStatic:False;" - } - }, - "NestedTypes": {} - } - } - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions+CompressionAlgorithm;System.Enum;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:True;IsClass:False;IsValueType:True;IsNested:True;IsGenericType:False;IsSerializable:True": { - "Subclasses": {}, - "Members": { - "CompressionAlgorithm None": { - "Type": "Field", - "Attributes": [], - "MethodInfo": "CompressionAlgorithm None;IsInitOnly:False;IsStatic:True;" - }, - "Int32 value__": { - "Type": "Field", - "Attributes": [], - "MethodInfo": "Int32 value__;IsInitOnly:False;IsStatic:False;" - } - }, - "NestedTypes": {} - }, "Microsoft.Azure.Cosmos.Encryption.Custom.CosmosDataEncryptionKeyProvider;Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { @@ -226,20 +120,54 @@ ], "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -1085,30 +1013,6 @@ "Microsoft.Azure.Cosmos.Encryption.Custom.EncryptionOptions;System.Object;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions CompressionOptions": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions CompressionOptions;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions get_CompressionOptions();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_CompressionOptions(Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions get_CompressionOptions()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions get_CompressionOptions();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor get_JsonProcessor()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor get_JsonProcessor();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor JsonProcessor": { - "Type": "Property", - "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor JsonProcessor;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor get_JsonProcessor();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;Void set_JsonProcessor(Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "System.Collections.Generic.IEnumerable`1[System.String] get_PathsToEncrypt()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -1150,13 +1054,6 @@ "Attributes": [], "MethodInfo": "[Void .ctor(), Void .ctor()]" }, - "Void set_CompressionOptions(Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_CompressionOptions(Microsoft.Azure.Cosmos.Encryption.Custom.CompressionOptions);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "Void set_DataEncryptionKeyId(System.String)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -1171,13 +1068,6 @@ ], "MethodInfo": "Void set_EncryptionAlgorithm(System.String);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Void set_JsonProcessor(Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor)[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", - "Attributes": [ - "CompilerGeneratedAttribute" - ], - "MethodInfo": "Void set_JsonProcessor(Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - }, "Void set_PathsToEncrypt(System.Collections.Generic.IEnumerable`1[System.String])[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ @@ -1242,20 +1132,54 @@ ], "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -1280,22 +1204,26 @@ "Type": "Method", "Attributes": [], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" - } - }, - "NestedTypes": {} - }, - "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor;System.Enum;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:True;IsClass:False;IsValueType:True;IsNested:False;IsGenericType:False;IsSerializable:True": { - "Subclasses": {}, - "Members": { - "Int32 value__": { - "Type": "Field", + }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", "Attributes": [], - "MethodInfo": "Int32 value__;IsInitOnly:False;IsStatic:False;" + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor Newtonsoft": { - "Type": "Field", + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.JsonProcessor Newtonsoft;IsInitOnly:False;IsStatic:True;" + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" } }, "NestedTypes": {} From 60223b4a487b741ad3d314443247b63ff814b262 Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Wed, 11 Dec 2024 17:17:54 -0800 Subject: [PATCH 05/18] update changefeed metadata serialization --- .../Converters/ChangeFeedMetadataConverter.cs | 17 ++++++++++++----- .../BuilderWithCustomSerializerTests.cs | 12 ++++++++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs index f448772ee2..6112c7c22a 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs @@ -89,14 +89,21 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json writer.WriteNumber(ChangeFeedMetadataFields.Lsn, value.Lsn); writer.WriteString(ChangeFeedMetadataFields.OperationType, value.OperationType.ToString()); writer.WriteNumber(ChangeFeedMetadataFields.PreviousImageLSN, value.PreviousLsn); - writer.WriteString(ChangeFeedMetadataFields.DeletedItemId, value.DeletedItemId); - writer.WriteStartObject("partitionKey"); - foreach (KeyValuePair kvp in value.DeletedItemPartitionKey) + if (value.DeletedItemId != null) { - writer.WriteString(kvp.Key, kvp.Value); + writer.WriteString(ChangeFeedMetadataFields.DeletedItemId, value.DeletedItemId); + } + + if (value.DeletedItemPartitionKey != null) + { + writer.WriteStartObject(ChangeFeedMetadataFields.DeletedItemPartitionKey); + foreach (KeyValuePair kvp in value.DeletedItemPartitionKey) + { + writer.WriteString(kvp.Key, kvp.Value); + } + writer.WriteEndObject(); } - writer.WriteEndObject(); writer.WriteEndObject(); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 08cf70a150..c8c35b069b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -39,7 +39,11 @@ public void ValidateNSJAndSTJSerializationOfChangeFeedItemDeleteTimeToLiveExpire ""crts"": 1722511591, ""operationType"": ""delete"", ""timeToLiveExpired"": true, - ""previousImageLSN"": 16 + ""previousImageLSN"": 16, + ""id"": ""1"", + ""partitionKey"": { + ""pk"": ""1"" + } }, ""previous"": { ""id"": ""1"", @@ -219,7 +223,11 @@ public void ValidateNSJAndSTJSerializationOfChangeFeedItemTest(bool propertyName ""lsn"": 376, ""operationType"": ""delete"", ""previousImageLSN"": 375, - ""timeToLiveExpired"": false + ""timeToLiveExpired"": false, + ""id"": ""1"", + ""partitionKey"": { + ""pk"": ""1"" + } }, ""previous"": { ""id"": ""1"", From aacd447996607222d31c0b4056549a584386d87c Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Wed, 11 Dec 2024 17:34:13 -0800 Subject: [PATCH 06/18] update ttl delete test --- .../BuilderWithCustomSerializerTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index c8c35b069b..fcbabdc291 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -390,7 +390,7 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Container leaseContainer = await database.CreateContainerIfNotExistsAsync(containerProperties: new ContainerProperties(id: "leases", partitionKeyPath: "/id")); ContainerInternal monitoredContainer = await this.CreateMonitoredContainer(ChangeFeedMode.AllVersionsAndDeletes, database); Exception exception = default; - int ttlInSeconds = 5; + int ttlInSeconds = 1; Stopwatch stopwatch = new(); ManualResetEvent allDocsProcessed = new ManualResetEvent(false); @@ -428,11 +428,11 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); - - // previous Assert.AreEqual(expected: "1", actual: change.Metadata.DeletedItemId.ToString()); change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); Assert.AreEqual(expected: "1", actual: partitionKey); + + // previous Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString()); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString()); From b2f0e2a47075891d22c791b90ab71c59add34fbe Mon Sep 17 00:00:00 2001 From: Justine Cocchi Date: Mon, 13 Jan 2025 20:12:50 -0800 Subject: [PATCH 07/18] update change feed metadata contract --- .../FullFidelity/ChangeFeedMetadata.cs | 8 +-- .../FullFidelity/ChangeFeedMetadataFields.cs | 4 +- .../Converters/ChangeFeedMetadataConverter.cs | 52 ++++++++++++++----- .../CFP/AllVersionsAndDeletes/BuilderTests.cs | 14 +++-- .../BuilderWithCustomSerializerTests.cs | 24 ++++----- .../FeedToken/ChangeFeedIteratorCoreTests.cs | 4 +- 6 files changed, 64 insertions(+), 42 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index b507644d9f..5909fee2c2 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -59,13 +59,13 @@ class ChangeFeedMetadata /// /// The id of the previous item version. Used for delete operations only. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.DeletedItemId, NullValueHandling = NullValueHandling.Ignore)] - public string DeletedItemId { get; internal set; } + [JsonProperty(PropertyName = ChangeFeedMetadataFields.Id, NullValueHandling = NullValueHandling.Ignore)] + public string Id { get; internal set; } /// /// The partition key of the previous item version. Dictionary Key is the partition key property name and Dictionary Value is the partition key property value. Used for delete operations only. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.DeletedItemPartitionKey, NullValueHandling = NullValueHandling.Ignore)] - public Dictionary DeletedItemPartitionKey { get; internal set; } + [JsonProperty(PropertyName = ChangeFeedMetadataFields.PartitionKey, NullValueHandling = NullValueHandling.Ignore)] + public List<(string, object)> PartitionKey { get; internal set; } } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs index d45bf1bd44..e0005b8c05 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs @@ -11,7 +11,7 @@ internal class ChangeFeedMetadataFields public const string OperationType = "operationType"; public const string PreviousImageLSN = "previousImageLSN"; public const string TimeToLiveExpired = "timeToLiveExpired"; - public const string DeletedItemId = "id"; - public const string DeletedItemPartitionKey = "partitionKey"; + public const string Id = "id"; + public const string PartitionKey = "partitionKey"; } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs index 6112c7c22a..ac70541357 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs @@ -57,18 +57,18 @@ public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToCo { metadata.PreviousLsn = property.Value.GetInt64(); } - else if (property.NameEquals(ChangeFeedMetadataFields.DeletedItemId)) + else if (property.NameEquals(ChangeFeedMetadataFields.Id)) { - metadata.DeletedItemId = property.Value.GetString(); + metadata.Id = property.Value.GetString(); } - else if (property.NameEquals(ChangeFeedMetadataFields.DeletedItemPartitionKey)) + else if (property.NameEquals(ChangeFeedMetadataFields.PartitionKey)) { - Dictionary partitionKey = new Dictionary(); + List<(string, object)> partitionKey = new List<(string, object)>(); foreach (JsonProperty pk in property.Value.EnumerateObject()) { - partitionKey.Add(pk.Name, pk.Value.GetString()); + partitionKey.Add((pk.Name, pk.Value)); } - metadata.DeletedItemPartitionKey = partitionKey; + metadata.PartitionKey = partitionKey; } } @@ -90,18 +90,46 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json writer.WriteString(ChangeFeedMetadataFields.OperationType, value.OperationType.ToString()); writer.WriteNumber(ChangeFeedMetadataFields.PreviousImageLSN, value.PreviousLsn); - if (value.DeletedItemId != null) + if (value.Id != null) { - writer.WriteString(ChangeFeedMetadataFields.DeletedItemId, value.DeletedItemId); + writer.WriteString(ChangeFeedMetadataFields.Id, value.Id); } - if (value.DeletedItemPartitionKey != null) + if (value.PartitionKey != null) { - writer.WriteStartObject(ChangeFeedMetadataFields.DeletedItemPartitionKey); - foreach (KeyValuePair kvp in value.DeletedItemPartitionKey) + writer.WriteStartObject(ChangeFeedMetadataFields.PartitionKey); + + foreach ((string, object) pk in value.PartitionKey) { - writer.WriteString(kvp.Key, kvp.Value); + JsonElement pkValue = (JsonElement)pk.Item2; + + switch (pkValue.ValueKind) + { + case JsonValueKind.String: + writer.WriteString(pk.Item1, pkValue.GetString()); + break; + + case JsonValueKind.Number: + writer.WriteNumber(pk.Item1, pkValue.GetDouble()); + break; + + case JsonValueKind.True: + case JsonValueKind.False: + writer.WriteBoolean(pk.Item1, pkValue.GetBoolean()); + break; + + case JsonValueKind.Null: + writer.WriteNull(pk.Item1); + break; + + case JsonValueKind.Undefined: + break; + + default: + throw new JsonException(string.Format(CultureInfo.CurrentCulture, RMResources.JsonUnexpectedToken)); + } } + writer.WriteEndObject(); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs index c5c96b0a22..c1c03f1abf 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs @@ -84,9 +84,8 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); // previous - Assert.AreEqual(expected: "1", actual: change.Metadata.DeletedItemId.ToString()); - change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: change.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.FirstOrDefault().Item2); Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: change.Previous.pk.ToString()); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: change.Previous.description.ToString()); @@ -176,8 +175,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() } else { - metadataId = change.Metadata.DeletedItemId.ToString(); - change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out metadataPk).ToString(); + metadataId = change.Metadata.Id.ToString(); + metadataPk = change.Metadata.PartitionKey.FirstOrDefault().Item2.ToString(); id = change.Previous.id.ToString(); pk = change.Previous.pk.ToString(); description = change.Previous.description.ToString(); @@ -218,9 +217,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync() ChangeFeedItem deleteChange = docs.ElementAt(2); Assert.IsNull(deleteChange.Current.id); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.DeletedItemId.ToString()); - deleteChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.FirstOrDefault().Item2); Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); Assert.IsNotNull(deleteChange.Previous); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index fcbabdc291..5f3c7169cb 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -96,9 +96,8 @@ static void ValidateDeserialization(List> activitie Assert.IsTrue(deletedChange.Metadata.IsTimeToLiveExpired); Assert.IsNotNull(deletedChange.Previous); Assert.AreEqual(expected: "Testing TTL on CFP.", actual: deletedChange.Previous.description); - Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.DeletedItemId.ToString()); - deletedChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.PartitionKey.FirstOrDefault().Item2.ToString()); Assert.AreEqual(expected: "1", actual: deletedChange.Previous.id); Assert.AreEqual(expected: 5, actual: deletedChange.Previous.ttl); } @@ -306,9 +305,8 @@ static void ValidateDeserialization(List> activitie Assert.IsFalse(deletedChange.Metadata.IsTimeToLiveExpired); Assert.IsNotNull(deletedChange.Previous); Assert.AreEqual(expected: "test after replace", actual: deletedChange.Previous.description); - Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.DeletedItemId.ToString()); - deletedChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: deletedChange.Metadata.PartitionKey.FirstOrDefault().Item2.ToString()); Assert.AreEqual(expected: "1", actual: deletedChange.Previous.id); Assert.AreEqual(expected: 0, actual: deletedChange.Previous.ttl); } @@ -428,9 +426,8 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); Assert.IsTrue(change.Metadata.IsTimeToLiveExpired); - Assert.AreEqual(expected: "1", actual: change.Metadata.DeletedItemId.ToString()); - change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: change.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: change.Metadata.PartitionKey.FirstOrDefault().Item2); // previous Assert.AreEqual(expected: "1", actual: change.Previous.id.ToString()); @@ -541,8 +538,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr } else { - metadataId = change.Metadata.DeletedItemId.ToString(); - change.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out metadataPk).ToString(); + metadataId = change.Metadata.Id.ToString(); + metadataPk = change.Metadata.PartitionKey.FirstOrDefault().Item2.ToString(); id = change.Previous.id.ToString(); pk = change.Previous.pk.ToString(); description = change.Previous.description.ToString(); @@ -586,9 +583,8 @@ public async Task WhenADocumentIsCreatedThenUpdatedThenDeletedTestsAsync(bool pr Assert.AreEqual(expected: deleteChange.Metadata.OperationType, actual: ChangeFeedOperationType.Delete); Assert.AreEqual(expected: replaceChange.Metadata.Lsn, actual: deleteChange.Metadata.PreviousLsn); Assert.IsNotNull(deleteChange.Previous); - Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.DeletedItemId.ToString()); - deleteChange.Metadata.DeletedItemPartitionKey.TryGetValue("pk", out string partitionKey).ToString(); - Assert.AreEqual(expected: "1", actual: partitionKey); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.Id.ToString()); + Assert.AreEqual(expected: "1", actual: deleteChange.Metadata.PartitionKey.FirstOrDefault().Item2); Assert.AreEqual(expected: "1", actual: deleteChange.Previous.id.ToString()); Assert.AreEqual(expected: "1", actual: deleteChange.Previous.pk.ToString()); Assert.AreEqual(expected: "test after replace", actual: deleteChange.Previous.description.ToString()); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs index f4cc9c2ab5..ca679ab6a8 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/FeedToken/ChangeFeedIteratorCoreTests.cs @@ -896,7 +896,7 @@ private async Task ValidateChangeFeedIteratorCore_WithQuery( foreach (ChangeFeedItem item in feedResponse) { - Assert.AreEqual(expected: "id3", actual: item.Metadata.DeletedItemId.ToString()); + Assert.AreEqual(expected: "id3", actual: item.Metadata.Id.ToString()); Assert.AreEqual("id3", item.Previous.Id); Assert.AreEqual(ChangeFeedOperationType.Delete, item.Metadata.OperationType); } @@ -1095,7 +1095,7 @@ public async Task ChangeFeedIteratorCore_FeedRange_VerifyingWireFormatTests() Assert.AreNotEqual(notExpected: default, actual: deleteOperation.Metadata.Lsn); Assert.AreNotEqual(notExpected: default, actual: deleteOperation.Metadata.PreviousLsn); Assert.IsNotNull(deleteOperation.Previous); - Assert.AreEqual(expected: id, actual: deleteOperation.Metadata.DeletedItemId.ToString()); + Assert.AreEqual(expected: id, actual: deleteOperation.Metadata.Id.ToString()); Assert.AreEqual(expected: id, actual: deleteOperation.Previous.Id); Assert.AreEqual(expected: "205 16th St NW", actual: deleteOperation.Previous.Line1); Assert.AreEqual(expected: "Atlanta", actual: deleteOperation.Previous.City); From 4129a953991f351c35c313255ace85867561cfc8 Mon Sep 17 00:00:00 2001 From: Dikshi Bahl Date: Fri, 9 May 2025 13:05:10 -0500 Subject: [PATCH 08/18] fix: adding newsoft converter to handle partitionKey --- .../FullFidelity/ChangeFeedMetadata.cs | 2 +- .../Converters/ChangeFeedMetadataConverter.cs | 39 ++-- .../ChangeFeedMetadataNewtonSoftConverter.cs | 207 ++++++++++++++++++ 3 files changed, 230 insertions(+), 18 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index 5909fee2c2..7a4184f5e8 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -6,7 +6,6 @@ namespace Microsoft.Azure.Cosmos { using System; using System.Collections.Generic; - using System.Text.Json; using Microsoft.Azure.Cosmos.Resource.FullFidelity; using Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters; using Microsoft.Azure.Documents; @@ -17,6 +16,7 @@ namespace Microsoft.Azure.Cosmos /// The metadata of a change feed resource with is initialized to . /// [System.Text.Json.Serialization.JsonConverter(typeof(ChangeFeedMetadataConverter))] + [JsonConverter(typeof(ChangeFeedMetadataNewtonSoftConverter))] #if PREVIEW public #else diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs index ac70541357..6975c65c97 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs @@ -66,7 +66,15 @@ public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToCo List<(string, object)> partitionKey = new List<(string, object)>(); foreach (JsonProperty pk in property.Value.EnumerateObject()) { - partitionKey.Add((pk.Name, pk.Value)); + object actualValue = pk.Value.ValueKind switch + { + JsonValueKind.String => pk.Value.GetString(), + JsonValueKind.Number => pk.Value.TryGetInt64(out long longValue) ? longValue : (object)pk.Value.GetDouble(), + JsonValueKind.True or JsonValueKind.False => pk.Value.GetBoolean(), + JsonValueKind.Null => null, + _ => throw new JsonException($"Unexpected JsonValueKind '{pk.Value.ValueKind}' for PartitionKey property."), + }; + partitionKey.Add((pk.Name, actualValue)); } metadata.PartitionKey = partitionKey; } @@ -99,37 +107,34 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json { writer.WriteStartObject(ChangeFeedMetadataFields.PartitionKey); - foreach ((string, object) pk in value.PartitionKey) + foreach ((string key, object objectValue) in value.PartitionKey) { - JsonElement pkValue = (JsonElement)pk.Item2; - - switch (pkValue.ValueKind) + switch (objectValue) { - case JsonValueKind.String: - writer.WriteString(pk.Item1, pkValue.GetString()); + case string stringValue: + writer.WriteString(key, stringValue); break; - case JsonValueKind.Number: - writer.WriteNumber(pk.Item1, pkValue.GetDouble()); + case long longValue: + writer.WriteNumber(key, longValue); break; - case JsonValueKind.True: - case JsonValueKind.False: - writer.WriteBoolean(pk.Item1, pkValue.GetBoolean()); + case double doubleValue: + writer.WriteNumber(key, doubleValue); break; - case JsonValueKind.Null: - writer.WriteNull(pk.Item1); + case bool boolValue: + writer.WriteBoolean(key, boolValue); break; - case JsonValueKind.Undefined: + case null: + writer.WriteNull(key); break; default: - throw new JsonException(string.Format(CultureInfo.CurrentCulture, RMResources.JsonUnexpectedToken)); + throw new JsonException($"Unexpected value type '{value.GetType()}' for PartitionKey property."); } } - writer.WriteEndObject(); } diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs new file mode 100644 index 0000000000..b9c84da4c5 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs @@ -0,0 +1,207 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters +{ + using System; + using System.Collections.Generic; + using Microsoft.Azure.Cosmos.Spatial; + using Newtonsoft.Json; + + internal class ChangeFeedMetadataNewtonSoftConverter : JsonConverter + { + private readonly static DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + + /// + /// Writes the JSON representation of the object. + /// + /// The to write to. + /// The object value to write. + /// The calling serializer. + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + if (value is ChangeFeedMetadata metadata) + { + writer.WriteStartObject(); + + writer.WritePropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp); + serializer.Serialize(writer, metadata.ConflictResolutionTimestamp); + + writer.WritePropertyName(ChangeFeedMetadataFields.Lsn); + writer.WriteValue(metadata.Lsn); + + writer.WritePropertyName(ChangeFeedMetadataFields.OperationType); + serializer.Serialize(writer, metadata.OperationType); + + writer.WritePropertyName(ChangeFeedMetadataFields.PreviousImageLSN); + writer.WriteValue(metadata.PreviousLsn); + + writer.WritePropertyName(ChangeFeedMetadataFields.TimeToLiveExpired); + writer.WriteValue(metadata.IsTimeToLiveExpired); + + writer.WritePropertyName(ChangeFeedMetadataFields.Id); + writer.WriteValue(metadata.Id); + + writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); + writer.WriteStartArray(); + if (metadata.PartitionKey != null) + { + writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); + writer.WriteStartObject(); + + foreach ((string key, object objectValue) in metadata.PartitionKey) + { + writer.WritePropertyName(key); + + if (objectValue == null) + { + writer.WriteNull(); + } + else + { + switch (objectValue) + { + case string stringValue: + writer.WriteValue(stringValue); + break; + + case long longValue: + writer.WriteValue(longValue); + break; + + case double doubleValue: + writer.WriteValue(doubleValue); + break; + + case bool boolValue: + writer.WriteValue(boolValue); + break; + + default: + throw new JsonSerializationException($"Unexpected value type: {objectValue.GetType()} for PartitionKey property."); + } + } + } + + writer.WriteEndObject(); + } + + writer.WriteEndObject(); + + } + else + { + throw new JsonSerializationException($"Unexpected value when converting {nameof(ChangeFeedMetadata)}."); + } + } + + /// + /// Reads the JSON representation of the object. + /// + /// The to read from. + /// Type of the object. + /// The existing value of object being read. + /// The calling serializer. + /// The deserialized object. + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null) + { + return null; + } + + ChangeFeedMetadata metadata = new ChangeFeedMetadata(); + List<(string, object)> partitionKey = new List<(string, object)>(); + + reader.Read(); // StartObject + + while (reader.TokenType == JsonToken.PropertyName) + { + string propertyName = reader.Value.ToString(); + reader.Read(); // Move to property value + + switch (propertyName) + { + case ChangeFeedMetadataFields.ConflictResolutionTimestamp: + metadata.ConflictResolutionTimestamp = ChangeFeedMetadataNewtonSoftConverter.ToDateTimeFromUnixTimeInSeconds(Convert.ToInt64(reader.Value)); + break; + + case ChangeFeedMetadataFields.Lsn: + metadata.Lsn = reader.Value != null ? Convert.ToInt64(reader.Value) : 0; + break; + + case ChangeFeedMetadataFields.OperationType: + metadata.OperationType = serializer.Deserialize(reader); + break; + + case ChangeFeedMetadataFields.PreviousImageLSN: + metadata.PreviousLsn = reader.Value != null ? Convert.ToInt64(reader.Value) : 0; + break; + + case ChangeFeedMetadataFields.TimeToLiveExpired: + metadata.IsTimeToLiveExpired = reader.Value != null && Convert.ToBoolean(reader.Value); + break; + + case ChangeFeedMetadataFields.Id: + metadata.Id = reader.Value?.ToString(); + break; + + case ChangeFeedMetadataFields.PartitionKey: + if (reader.TokenType == JsonToken.StartObject) + { + reader.Read(); // Move to the first property in the object + + while (reader.TokenType == JsonToken.PropertyName) + { + string key = reader.Value.ToString(); + reader.Read(); // Move to the value of the property + + object value = reader.TokenType switch + { + JsonToken.String => reader.Value.ToString(), + JsonToken.Integer => Convert.ToInt64(reader.Value), + JsonToken.Float => Convert.ToDouble(reader.Value), + JsonToken.Boolean => Convert.ToBoolean(reader.Value), + JsonToken.Null => null, + _ => throw new JsonSerializationException($"Unexpected token type: {reader.TokenType} for PartitionKey property.") + }; + + partitionKey.Add((key, value)); + reader.Read(); // Move to the next property or EndObject + } + } + break; + + default: + reader.Skip(); + break; + } + + reader.Read(); // Move to next property or EndObject + } + + metadata.PartitionKey = partitionKey; + return metadata; + } + /// + /// Determines whether this instance can convert the specified object type. + /// + /// Type of the object. + /// true if this instance can convert the specified object type; otherwise, false. + public override bool CanConvert(Type objectType) + { + return objectType == typeof(ChangeFeedMetadata); + } + + private static long ToUnixTimeInSecondsFromDateTime(DateTime date) + { + return (long)(date - ChangeFeedMetadataNewtonSoftConverter.UnixEpoch).TotalSeconds; + } + + private static DateTime ToDateTimeFromUnixTimeInSeconds(long unixTimeInSeconds) + { + return ChangeFeedMetadataNewtonSoftConverter.UnixEpoch.AddSeconds(unixTimeInSeconds); + } + } +} From c41d59c0adf1876158aff36e361611e0902de794 Mon Sep 17 00:00:00 2001 From: Dikshi Bahl Date: Fri, 9 May 2025 15:00:57 -0500 Subject: [PATCH 09/18] fix:updated contract --- .../Contracts/DotNetPreviewSDKAPI.json | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index 2a48f3ea62..ae48b42248 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -137,19 +137,19 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.Dictionary`2[System.String,System.String] DeletedItemPartitionKey[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"partitionKey\")]": { - "Type": "Property", + "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", "Attributes": [ - "JsonPropertyAttribute" + "CompilerGeneratedAttribute" ], - "MethodInfo": "System.Collections.Generic.Dictionary`2[System.String,System.String] DeletedItemPartitionKey;CanRead:True;CanWrite:True;System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", + "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] PartitionKey[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"partitionKey\")]": { + "Type": "Property", "Attributes": [ - "CompilerGeneratedAttribute" + "JsonPropertyAttribute" ], - "MethodInfo": "System.Collections.Generic.Dictionary`2[System.String,System.String] get_DeletedItemPartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] PartitionKey;CanRead:True;CanWrite:True;System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "System.DateTime ConflictResolutionTimestamp[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"crts\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Microsoft.Azure.Documents.UnixDateTimeConverter))]": { "Type": "Property", @@ -166,19 +166,19 @@ ], "MethodInfo": "System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.String DeletedItemId[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"id\")]": { - "Type": "Property", + "System.String get_Id()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", "Attributes": [ - "JsonPropertyAttribute" + "CompilerGeneratedAttribute" ], - "MethodInfo": "System.String DeletedItemId;CanRead:True;CanWrite:True;System.String get_DeletedItemId();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.String get_Id();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.String get_DeletedItemId()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { - "Type": "Method", + "System.String Id[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"id\")]": { + "Type": "Property", "Attributes": [ - "CompilerGeneratedAttribute" + "JsonPropertyAttribute" ], - "MethodInfo": "System.String get_DeletedItemId();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "System.String Id;CanRead:True;CanWrite:True;System.String get_Id();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Void .ctor()": { "Type": "Constructor", From 5ddaec18588c67683b51ed322c33038fa9652a3b Mon Sep 17 00:00:00 2001 From: Dikshi Bahl Date: Sat, 10 May 2025 12:02:31 -0500 Subject: [PATCH 10/18] fix:refactoring --- .../ChangeFeedMetadataNewtonSoftConverter.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs index b9c84da4c5..e685099a9e 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs @@ -24,9 +24,8 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s if (value is ChangeFeedMetadata metadata) { writer.WriteStartObject(); - writer.WritePropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp); - serializer.Serialize(writer, metadata.ConflictResolutionTimestamp); + serializer.Serialize(writer, ChangeFeedMetadataNewtonSoftConverter.ToUnixTimeInSecondsFromDateTime(metadata.ConflictResolutionTimestamp)); writer.WritePropertyName(ChangeFeedMetadataFields.Lsn); writer.WriteValue(metadata.Lsn); @@ -42,13 +41,10 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s writer.WritePropertyName(ChangeFeedMetadataFields.Id); writer.WriteValue(metadata.Id); - - writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); - writer.WriteStartArray(); if (metadata.PartitionKey != null) { writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey); - writer.WriteStartObject(); + writer.WriteStartObject(); foreach ((string key, object objectValue) in metadata.PartitionKey) { @@ -84,11 +80,10 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s } } - writer.WriteEndObject(); + writer.WriteEndObject(); // End PartitionKey object } writer.WriteEndObject(); - } else { From 31bc8b82c29e7c19af6512b5a17f8370d9e51636 Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Tue, 27 May 2025 13:25:16 -0500 Subject: [PATCH 11/18] fix:addressing comments --- .../src/Resource/FullFidelity/ChangeFeedMetadata.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index 7a4184f5e8..a6de520a25 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -57,13 +57,15 @@ class ChangeFeedMetadata public bool IsTimeToLiveExpired { get; internal set; } /// - /// The id of the previous item version. Used for delete operations only. + /// Used for delete operations only. + /// The id of the previous item version. /// [JsonProperty(PropertyName = ChangeFeedMetadataFields.Id, NullValueHandling = NullValueHandling.Ignore)] public string Id { get; internal set; } /// - /// The partition key of the previous item version. Dictionary Key is the partition key property name and Dictionary Value is the partition key property value. Used for delete operations only. + /// Used for delete operations only. + /// The partition key of the previous item version. string is the partition key property name and object is the partition key property value. All levels of hierarchy will be represented in order if a HPK is used. /// [JsonProperty(PropertyName = ChangeFeedMetadataFields.PartitionKey, NullValueHandling = NullValueHandling.Ignore)] public List<(string, object)> PartitionKey { get; internal set; } From d0d775dc62abfd91c4f74e1c0a7ae8e5c612700b Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 28 May 2025 11:23:33 -0500 Subject: [PATCH 12/18] Update Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs Co-authored-by: Kiran Kumar Kolli --- .../src/Resource/FullFidelity/ChangeFeedMetadata.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index a6de520a25..f27f08a405 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -57,7 +57,7 @@ class ChangeFeedMetadata public bool IsTimeToLiveExpired { get; internal set; } /// - /// Used for delete operations only. + /// Applicable for delete operations only, otherwise null. /// The id of the previous item version. /// [JsonProperty(PropertyName = ChangeFeedMetadataFields.Id, NullValueHandling = NullValueHandling.Ignore)] From 59f53a5c3b303b5ef4296e5e042c0ac35981e8dc Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 28 May 2025 11:23:46 -0500 Subject: [PATCH 13/18] Update Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs Co-authored-by: Kiran Kumar Kolli --- .../src/Resource/FullFidelity/ChangeFeedMetadata.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index f27f08a405..3b1ab146d0 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -64,7 +64,7 @@ class ChangeFeedMetadata public string Id { get; internal set; } /// - /// Used for delete operations only. + /// Applicable for delete operations only, otherwise null. /// The partition key of the previous item version. string is the partition key property name and object is the partition key property value. All levels of hierarchy will be represented in order if a HPK is used. /// [JsonProperty(PropertyName = ChangeFeedMetadataFields.PartitionKey, NullValueHandling = NullValueHandling.Ignore)] From 2870d206aa158f62edbb5848b6ddbdd4ca21eba5 Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 28 May 2025 15:56:34 -0500 Subject: [PATCH 14/18] fix: addressing comments --- .../src/Resource/FullFidelity/ChangeFeedMetadata.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs index 3b1ab146d0..7c6d982475 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs @@ -27,47 +27,38 @@ class ChangeFeedMetadata /// /// The change's conflict resolution timestamp. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.ConflictResolutionTimestamp, NullValueHandling = NullValueHandling.Ignore)] - [JsonConverter(typeof(UnixDateTimeConverter))] public DateTime ConflictResolutionTimestamp { get; internal set; } /// /// The current change's logical sequence number. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.Lsn, NullValueHandling = NullValueHandling.Ignore)] public long Lsn { get; internal set; } /// /// The change's feed operation type . /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.OperationType, NullValueHandling = NullValueHandling.Ignore)] - [JsonConverter(typeof(StringEnumConverter))] public ChangeFeedOperationType OperationType { get; internal set; } /// /// The previous change's logical sequence number. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.PreviousImageLSN, NullValueHandling = NullValueHandling.Ignore)] public long PreviousLsn { get; internal set; } /// /// Used to distinguish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents). /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.TimeToLiveExpired, NullValueHandling = NullValueHandling.Ignore)] public bool IsTimeToLiveExpired { get; internal set; } /// /// Applicable for delete operations only, otherwise null. /// The id of the previous item version. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.Id, NullValueHandling = NullValueHandling.Ignore)] public string Id { get; internal set; } /// /// Applicable for delete operations only, otherwise null. /// The partition key of the previous item version. string is the partition key property name and object is the partition key property value. All levels of hierarchy will be represented in order if a HPK is used. /// - [JsonProperty(PropertyName = ChangeFeedMetadataFields.PartitionKey, NullValueHandling = NullValueHandling.Ignore)] public List<(string, object)> PartitionKey { get; internal set; } } } From 079e9aa0ee91e3b18671cdad12df4c8cb7b58bc3 Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Sun, 1 Jun 2025 13:11:35 -0500 Subject: [PATCH 15/18] fix: addressing comments --- .../Converters/ChangeFeedMetadataNewtonSoftConverter.cs | 4 ++-- .../CFP/AllVersionsAndDeletes/BuilderTests.cs | 3 ++- .../AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs index e685099a9e..ac5e8f2a7a 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs @@ -107,7 +107,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist } ChangeFeedMetadata metadata = new ChangeFeedMetadata(); - List<(string, object)> partitionKey = new List<(string, object)>(); + List<(string, object)> partitionKey = null; reader.Read(); // StartObject @@ -145,8 +145,8 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist case ChangeFeedMetadataFields.PartitionKey: if (reader.TokenType == JsonToken.StartObject) { + partitionKey ??= new List<(string, object)>(); reader.Read(); // Move to the first property in the object - while (reader.TokenType == JsonToken.PropertyName) { string key = reader.Value.ToString(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs index c1c03f1abf..914e256220 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderTests.cs @@ -69,7 +69,8 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); Assert.IsFalse(change.Metadata.IsTimeToLiveExpired); - + Assert.IsNull(change.Metadata.Id); + Assert.IsNull(change.Metadata.PartitionKey); // previous Assert.IsNull(change.Previous); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs index 5f3c7169cb..061ffbfcd0 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CFP/AllVersionsAndDeletes/BuilderWithCustomSerializerTests.cs @@ -413,6 +413,8 @@ public async Task WhenADocumentIsCreatedWithTtlSetThenTheDocumentIsDeletedTestsA Assert.IsTrue(DateTime.TryParse(s: change.Metadata.ConflictResolutionTimestamp.ToString(), out _), message: "Invalid csrt must be a datetime value."); Assert.IsTrue(change.Metadata.Lsn > 0, message: "Invalid lsn must be a long value."); Assert.IsFalse(change.Metadata.IsTimeToLiveExpired); + Assert.IsNull(change.Metadata.Id); + Assert.IsNull(change.Metadata.PartitionKey); // previous Assert.IsNull(change.Previous); From aa60a72fa69c9ba815f30250485a6472e66eb52b Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 4 Jun 2025 12:34:55 -0500 Subject: [PATCH 16/18] fix: refactoring --- .../Converters/ChangeFeedMetadataNewtonSoftConverter.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs index ac5e8f2a7a..911cd98589 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs @@ -165,6 +165,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist partitionKey.Add((key, value)); reader.Read(); // Move to the next property or EndObject } + metadata.PartitionKey = partitionKey; } break; @@ -175,8 +176,6 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist reader.Read(); // Move to next property or EndObject } - - metadata.PartitionKey = partitionKey; return metadata; } /// From 2658b33f134ee9eff006883cd80ed85de567967b Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 17 Sep 2025 16:51:45 -0500 Subject: [PATCH 17/18] updating contract file --- .../Contracts/DotNetPreviewSDKAPI.json | 44 ++++++------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index ae48b42248..6ae5f8fa96 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -87,11 +87,9 @@ ], "MethodInfo": "Boolean get_IsTimeToLiveExpired();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Boolean IsTimeToLiveExpired[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"timeToLiveExpired\")]": { + "Boolean IsTimeToLiveExpired": { "Type": "Property", - "Attributes": [ - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "Boolean IsTimeToLiveExpired;CanRead:True;CanWrite:True;Boolean get_IsTimeToLiveExpired();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Int64 get_Lsn()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { @@ -108,18 +106,14 @@ ], "MethodInfo": "Int64 get_PreviousLsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Int64 Lsn[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"lsn\")]": { + "Int64 Lsn": { "Type": "Property", - "Attributes": [ - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "Int64 Lsn;CanRead:True;CanWrite:True;Int64 get_Lsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Int64 PreviousLsn[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"previousImageLSN\")]": { + "Int64 PreviousLsn": { "Type": "Property", - "Attributes": [ - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "Int64 PreviousLsn;CanRead:True;CanWrite:True;Int64 get_PreviousLsn();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { @@ -129,12 +123,9 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"operationType\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Newtonsoft.Json.Converters.StringEnumConverter))]": { + "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType": { "Type": "Property", - "Attributes": [ - "JsonConverterAttribute", - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedOperationType OperationType;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedOperationType get_OperationType();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { @@ -144,19 +135,14 @@ ], "MethodInfo": "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] PartitionKey[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"partitionKey\")]": { + "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] PartitionKey": { "Type": "Property", - "Attributes": [ - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] PartitionKey;CanRead:True;CanWrite:True;System.Collections.Generic.List`1[System.ValueTuple`2[System.String,System.Object]] get_PartitionKey();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.DateTime ConflictResolutionTimestamp[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"crts\")]-[Newtonsoft.Json.JsonConverterAttribute(typeof(Microsoft.Azure.Documents.UnixDateTimeConverter))]": { + "System.DateTime ConflictResolutionTimestamp": { "Type": "Property", - "Attributes": [ - "JsonConverterAttribute", - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "System.DateTime ConflictResolutionTimestamp;CanRead:True;CanWrite:True;System.DateTime get_ConflictResolutionTimestamp();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "System.DateTime get_ConflictResolutionTimestamp()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { @@ -173,11 +159,9 @@ ], "MethodInfo": "System.String get_Id();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.String Id[Newtonsoft.Json.JsonPropertyAttribute(NullValueHandling = 1, PropertyName = \"id\")]": { + "System.String Id": { "Type": "Property", - "Attributes": [ - "JsonPropertyAttribute" - ], + "Attributes": [], "MethodInfo": "System.String Id;CanRead:True;CanWrite:True;System.String get_Id();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Void .ctor()": { From 8694811cc668898d252019daa4ed117f6194442d Mon Sep 17 00:00:00 2001 From: dibahlfi <106994927+dibahlfi@users.noreply.github.com> Date: Wed, 29 Oct 2025 14:34:41 -0500 Subject: [PATCH 18/18] enhanced logging --- .../Converters/ChangeFeedMetadataConverter.cs | 13 +++++++++++-- .../ChangeFeedMetadataNewtonSoftConverter.cs | 4 ++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs index 6975c65c97..3ef24a7213 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs @@ -72,13 +72,22 @@ public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToCo JsonValueKind.Number => pk.Value.TryGetInt64(out long longValue) ? longValue : (object)pk.Value.GetDouble(), JsonValueKind.True or JsonValueKind.False => pk.Value.GetBoolean(), JsonValueKind.Null => null, - _ => throw new JsonException($"Unexpected JsonValueKind '{pk.Value.ValueKind}' for PartitionKey property."), + _ => throw new JsonException($"Unexpected JsonValueKind '{pk.Value.ValueKind}' for PartitionKey property '{pk.Name}'."), }; partitionKey.Add((pk.Name, actualValue)); } metadata.PartitionKey = partitionKey; } } + + // validate delete operation requirements + if (metadata.OperationType == ChangeFeedOperationType.Delete) + { + if (metadata.Id == null || metadata.PartitionKey == null) + { + throw new JsonException("Delete operations require both 'id' and 'partitionKey' to be present."); + } + } return metadata; } @@ -132,7 +141,7 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json break; default: - throw new JsonException($"Unexpected value type '{value.GetType()}' for PartitionKey property."); + throw new JsonException($"Unexpected value type '{objectValue.GetType()}' for PartitionKey property '{key}'."); } } writer.WriteEndObject(); diff --git a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs index 911cd98589..5ab7070192 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs @@ -75,7 +75,7 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s break; default: - throw new JsonSerializationException($"Unexpected value type: {objectValue.GetType()} for PartitionKey property."); + throw new JsonSerializationException($"Unexpected value type '{objectValue.GetType()}' for PartitionKey property '{key}'."); } } } @@ -87,7 +87,7 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s } else { - throw new JsonSerializationException($"Unexpected value when converting {nameof(ChangeFeedMetadata)}."); + throw new JsonSerializationException($"Unexpected value '{value}' of type '{value?.GetType()}' when converting {nameof(ChangeFeedMetadata)}."); } }