Change feed: Adds id and pk to ChangeFeedMetadata for delete operations

Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedItem.cs

@@ -57,21 +57,21 @@ namespace Microsoft.Azure.Cosmos
         class ChangeFeedItem<T>
     {
         /// <summary>
-        /// The full fidelity change feed current item.
+        /// The current version of the item for all versions and deletes change feed mode.
         /// </summary>
         [JsonProperty(PropertyName = "current")]
         [JsonPropertyName("current")]
         public T Current { get; set; }
 
         /// <summary>
-        /// The full fidelity change feed metadata.
+        /// The item metadata for all versions and deletes change feed mode.
         /// </summary>
         [JsonProperty(PropertyName = "metadata", NullValueHandling = NullValueHandling.Ignore)]
         [JsonPropertyName("metadata")]
         public ChangeFeedMetadata Metadata { get; set; }
 
         /// <summary>
-        /// For delete operations, previous image is always going to be provided. The previous image on replace operations is not going to be exposed by default and requires account-level or container-level opt-in.
+        /// The previous version of the item for all versions and deletes change feed mode. The previous version on delete and replace operations is not exposed by default and requires container-level opt-in. Refer to https://aka.ms/cosmosdb-change-feed-deletes for more information.
         /// </summary>
         [JsonProperty(PropertyName = "previous", NullValueHandling = NullValueHandling.Ignore)]
         [JsonPropertyName("previous")]

Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadata.cs

@@ -5,7 +5,7 @@
 namespace Microsoft.Azure.Cosmos
 {
     using System;
-    using System.Text.Json;
+    using System.Collections.Generic;
     using Microsoft.Azure.Cosmos.Resource.FullFidelity;
     using Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters;
     using Microsoft.Azure.Documents;
@@ -16,43 +16,49 @@ namespace Microsoft.Azure.Cosmos
     /// The metadata of a change feed resource with <see cref="ChangeFeedMode"/> is initialized to <see cref="ChangeFeedMode.AllVersionsAndDeletes"/>.
     /// </summary>
     [System.Text.Json.Serialization.JsonConverter(typeof(ChangeFeedMetadataConverter))]
+    [JsonConverter(typeof(ChangeFeedMetadataNewtonSoftConverter))]
 #if PREVIEW
     public
 #else
     internal
 #endif
-        class ChangeFeedMetadata
+    class ChangeFeedMetadata
     {
         /// <summary>
         /// The change's conflict resolution timestamp.
         /// </summary>
-        [JsonProperty(PropertyName = ChangeFeedMetadataFields.ConflictResolutionTimestamp, NullValueHandling = NullValueHandling.Ignore)]
-        [JsonConverter(typeof(UnixDateTimeConverter))]
         public DateTime ConflictResolutionTimestamp { get; internal set; }
 
         /// <summary>
         /// The current change's logical sequence number.
         /// </summary>
-        [JsonProperty(PropertyName = ChangeFeedMetadataFields.Lsn, NullValueHandling = NullValueHandling.Ignore)]
         public long Lsn { get; internal set; }
 
         /// <summary>
         /// The change's feed operation type <see cref="ChangeFeedOperationType"/>.
         /// </summary>
-        [JsonProperty(PropertyName = ChangeFeedMetadataFields.OperationType, NullValueHandling = NullValueHandling.Ignore)]
-        [JsonConverter(typeof(StringEnumConverter))]
         public ChangeFeedOperationType OperationType { get; internal set; }
 
         /// <summary>
         /// The previous change's logical sequence number.
         /// </summary>
-        [JsonProperty(PropertyName = ChangeFeedMetadataFields.PreviousImageLSN, NullValueHandling = NullValueHandling.Ignore)]
         public long PreviousLsn { get; internal set; }
 
         /// <summary>
-        /// Used to distinquish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents).
+        /// Used to distinguish explicit deletes (e.g. via DeleteItem) from deletes caused by TTL expiration (a collection may define time-to-live policy for documents).
         /// </summary>
-        [JsonProperty(PropertyName = ChangeFeedMetadataFields.TimeToLiveExpired, NullValueHandling = NullValueHandling.Ignore)]
         public bool IsTimeToLiveExpired { get; internal set; }
+
+        /// <summary>
+        /// Applicable for delete operations only, otherwise null.
+        /// The id of the previous item version. 
+        /// </summary>
+        public string Id { get; internal set; }
+
+        /// <summary>
+        ///  Applicable for delete operations only, otherwise null.
+        /// The partition key of the previous item version. string  is the partition key property name and object is the partition key property value. All levels of hierarchy will be represented in order if a HPK is used.
+        /// </summary>
+        public List<(string, object)> PartitionKey { get; internal set; }
     }
 }

Microsoft.Azure.Cosmos/src/Resource/FullFidelity/ChangeFeedMetadataFields.cs

@@ -11,5 +11,7 @@ internal class ChangeFeedMetadataFields
         public const string OperationType = "operationType";
         public const string PreviousImageLSN = "previousImageLSN";
         public const string TimeToLiveExpired = "timeToLiveExpired";
+        public const string Id = "id";
+        public const string PartitionKey = "partitionKey";
     }
 }

Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataConverter.cs

@@ -5,6 +5,7 @@
 namespace Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters
 {
     using System;
+    using System.Collections.Generic;
     using System.Globalization;
     using System.Text.Json;
     using System.Text.Json.Serialization;
@@ -56,6 +57,36 @@ public override ChangeFeedMetadata Read(ref Utf8JsonReader reader, Type typeToCo
                 {
                     metadata.PreviousLsn = property.Value.GetInt64();
                 }
+                else if (property.NameEquals(ChangeFeedMetadataFields.Id))
+                {
+                    metadata.Id = property.Value.GetString();
+                }
+                else if (property.NameEquals(ChangeFeedMetadataFields.PartitionKey))
+                {
+                    List<(string, object)> partitionKey = new List<(string, object)>();
+                    foreach (JsonProperty pk in property.Value.EnumerateObject())
+                    {
+                        object actualValue = pk.Value.ValueKind switch
+                        {
+                            JsonValueKind.String => pk.Value.GetString(),
+                            JsonValueKind.Number => pk.Value.TryGetInt64(out long longValue) ? longValue : (object)pk.Value.GetDouble(),
+                            JsonValueKind.True or JsonValueKind.False => pk.Value.GetBoolean(),
+                            JsonValueKind.Null => null,
+                            _ => throw new JsonException($"Unexpected JsonValueKind '{pk.Value.ValueKind}' for PartitionKey property '{pk.Name}'."),
+                        };
+                        partitionKey.Add((pk.Name, actualValue));
+                    }
+                    metadata.PartitionKey = partitionKey;
+                }
+            }
+
+            // validate delete operation requirements
+            if (metadata.OperationType == ChangeFeedOperationType.Delete)
+            {
+                if (metadata.Id == null || metadata.PartitionKey == null)
+                {
+                    throw new JsonException("Delete operations require both 'id' and 'partitionKey' to be present.");
+                }
             }
 
             return metadata;
@@ -76,6 +107,46 @@ public override void Write(Utf8JsonWriter writer, ChangeFeedMetadata value, Json
             writer.WriteString(ChangeFeedMetadataFields.OperationType, value.OperationType.ToString());
             writer.WriteNumber(ChangeFeedMetadataFields.PreviousImageLSN, value.PreviousLsn);
 
+            if (value.Id != null)
+            {
+                writer.WriteString(ChangeFeedMetadataFields.Id, value.Id);
+            }
+
+            if (value.PartitionKey != null)
+            {
+                writer.WriteStartObject(ChangeFeedMetadataFields.PartitionKey);
+
+                foreach ((string key, object objectValue) in value.PartitionKey)
+                {
+                    switch (objectValue)
+                    {
+                        case string stringValue:
+                            writer.WriteString(key, stringValue);
+                            break;
+
+                        case long longValue:
+                            writer.WriteNumber(key, longValue);
+                            break;
+
+                        case double doubleValue:
+                            writer.WriteNumber(key, doubleValue);
+                            break;
+
+                        case bool boolValue:
+                            writer.WriteBoolean(key, boolValue);
+                            break;
+
+                        case null:
+                            writer.WriteNull(key);
+                            break;
+
+                        default:
+                           throw new JsonException($"Unexpected value type '{objectValue.GetType()}' for PartitionKey property '{key}'.");
+                    }
+                }
+                writer.WriteEndObject();
+            }
+
             writer.WriteEndObject();
         }
 

Microsoft.Azure.Cosmos/src/Resource/FullFidelity/Converters/ChangeFeedMetadataNewtonSoftConverter.cs

@@ -0,0 +1,201 @@
+// ------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+// ------------------------------------------------------------
+
+namespace Microsoft.Azure.Cosmos.Resource.FullFidelity.Converters
+{
+    using System;
+    using System.Collections.Generic;
+    using Microsoft.Azure.Cosmos.Spatial;
+    using Newtonsoft.Json;
+
+    internal class ChangeFeedMetadataNewtonSoftConverter : JsonConverter
+    {
+        private readonly static DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
+
+        /// <summary>
+        /// Writes the JSON representation of the object.
+        /// </summary>
+        /// <param name="writer">The <see cref="JsonWriter"/> to write to.</param>
+        /// <param name="value">The object value to write.</param>
+        /// <param name="serializer">The calling serializer.</param>
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            if (value is ChangeFeedMetadata metadata)
+            {
+                writer.WriteStartObject();
+                writer.WritePropertyName(ChangeFeedMetadataFields.ConflictResolutionTimestamp);
+                serializer.Serialize(writer, ChangeFeedMetadataNewtonSoftConverter.ToUnixTimeInSecondsFromDateTime(metadata.ConflictResolutionTimestamp));
+
+                writer.WritePropertyName(ChangeFeedMetadataFields.Lsn);
+                writer.WriteValue(metadata.Lsn);
+
+                writer.WritePropertyName(ChangeFeedMetadataFields.OperationType);
+                serializer.Serialize(writer, metadata.OperationType);
+
+                writer.WritePropertyName(ChangeFeedMetadataFields.PreviousImageLSN);
+                writer.WriteValue(metadata.PreviousLsn);
+
+                writer.WritePropertyName(ChangeFeedMetadataFields.TimeToLiveExpired);
+                writer.WriteValue(metadata.IsTimeToLiveExpired);
+
+                writer.WritePropertyName(ChangeFeedMetadataFields.Id);
+                writer.WriteValue(metadata.Id);
+                if (metadata.PartitionKey != null)
+                {
+                    writer.WritePropertyName(ChangeFeedMetadataFields.PartitionKey);
+                    writer.WriteStartObject(); 
+
+                    foreach ((string key, object objectValue) in metadata.PartitionKey)
+                    {
+                        writer.WritePropertyName(key);
+
+                        if (objectValue == null)
+                        {
+                            writer.WriteNull();
+                        }
+                        else
+                        {
+                            switch (objectValue)
+                            {
+                                case string stringValue:
+                                    writer.WriteValue(stringValue);
+                                    break;
+
+                                case long longValue:
+                                    writer.WriteValue(longValue);
+                                    break;
+
+                                case double doubleValue:
+                                    writer.WriteValue(doubleValue);
+                                    break;
+
+                                case bool boolValue:
+                                    writer.WriteValue(boolValue);
+                                    break;
+
+                                default:
+                                    throw new JsonSerializationException($"Unexpected value type '{objectValue.GetType()}' for PartitionKey property '{key}'.");
+                            }
+                        }
+                    }
+
+                    writer.WriteEndObject(); // End PartitionKey object
+                }
+
+                writer.WriteEndObject();
+            }
+            else
+            {
+                throw new JsonSerializationException($"Unexpected value '{value}' of type '{value?.GetType()}' when converting {nameof(ChangeFeedMetadata)}.");
+            }
+        }
+
+        /// <summary>
+        /// Reads the JSON representation of the object.
+        /// </summary>
+        /// <param name="reader">The <see cref="JsonReader"/> to read from.</param>
+        /// <param name="objectType">Type of the object.</param>
+        /// <param name="existingValue">The existing value of object being read.</param>
+        /// <param name="serializer">The calling serializer.</param>
+        /// <returns>The deserialized object.</returns>
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            if (reader.TokenType == JsonToken.Null)
+            {
+                return null;
+            }
+
+            ChangeFeedMetadata metadata = new ChangeFeedMetadata();
+            List<(string, object)> partitionKey = null;
+
+            reader.Read(); // StartObject
+
+            while (reader.TokenType == JsonToken.PropertyName)
+            {
+                string propertyName = reader.Value.ToString();
+                reader.Read(); // Move to property value
+
+                switch (propertyName)
+                {
+                    case ChangeFeedMetadataFields.ConflictResolutionTimestamp:
+                        metadata.ConflictResolutionTimestamp = ChangeFeedMetadataNewtonSoftConverter.ToDateTimeFromUnixTimeInSeconds(Convert.ToInt64(reader.Value));
+                        break;
+
+                    case ChangeFeedMetadataFields.Lsn:
+                        metadata.Lsn = reader.Value != null ? Convert.ToInt64(reader.Value) : 0;
+                        break;
+
+                    case ChangeFeedMetadataFields.OperationType:
+                        metadata.OperationType = serializer.Deserialize<ChangeFeedOperationType>(reader);
+                        break;
+
+                    case ChangeFeedMetadataFields.PreviousImageLSN:
+                        metadata.PreviousLsn = reader.Value != null ? Convert.ToInt64(reader.Value) : 0;
+                        break;
+
+                    case ChangeFeedMetadataFields.TimeToLiveExpired:
+                        metadata.IsTimeToLiveExpired = reader.Value != null && Convert.ToBoolean(reader.Value);
+                        break;
+
+                    case ChangeFeedMetadataFields.Id:
+                        metadata.Id = reader.Value?.ToString();
+                        break;
+
+                    case ChangeFeedMetadataFields.PartitionKey:
+                        if (reader.TokenType == JsonToken.StartObject)
+                        {
+                            partitionKey ??= new List<(string, object)>();
+                            reader.Read(); // Move to the first property in the object
+                            while (reader.TokenType == JsonToken.PropertyName)
+                            {
+                                string key = reader.Value.ToString();
+                                reader.Read(); // Move to the value of the property
+
+                                object value = reader.TokenType switch
+                                {
+                                    JsonToken.String => reader.Value.ToString(),
+                                    JsonToken.Integer => Convert.ToInt64(reader.Value),
+                                    JsonToken.Float => Convert.ToDouble(reader.Value),
+                                    JsonToken.Boolean => Convert.ToBoolean(reader.Value),
+                                    JsonToken.Null => null,
+                                    _ => throw new JsonSerializationException($"Unexpected token type: {reader.TokenType} for PartitionKey property.")
+                                };
+
+                                partitionKey.Add((key, value));
+                                reader.Read(); // Move to the next property or EndObject
+                            }
+                            metadata.PartitionKey = partitionKey;
+                        }
+                        break;
+
+                    default:
+                        reader.Skip();
+                        break;
+                }
+
+                reader.Read(); // Move to next property or EndObject
+            }
+            return metadata;
+        }
+        /// <summary>
+        /// Determines whether this instance can convert the specified object type.
+        /// </summary>
+        /// <param name="objectType">Type of the object.</param>
+        /// <returns><c>true</c> if this instance can convert the specified object type; otherwise, <c>false</c>.</returns>
+        public override bool CanConvert(Type objectType)
+        {
+            return objectType == typeof(ChangeFeedMetadata);
+        }
+
+        private static long ToUnixTimeInSecondsFromDateTime(DateTime date)
+        {
+            return (long)(date - ChangeFeedMetadataNewtonSoftConverter.UnixEpoch).TotalSeconds;
+        }
+
+        private static DateTime ToDateTimeFromUnixTimeInSeconds(long unixTimeInSeconds)
+        {
+            return ChangeFeedMetadataNewtonSoftConverter.UnixEpoch.AddSeconds(unixTimeInSeconds);
+        }
+    }
+}