[Internal] Binary Encoding: Fixes DateTime Parsing Issue with Trailing Zeros in the Milli-Seconds Precision.

Microsoft.Azure.Cosmos/src/Json/Interop/CosmosDBToNewtonsoftWriter.cs

@@ -3,7 +3,7 @@
 //------------------------------------------------------------
 namespace Microsoft.Azure.Cosmos.Json.Interop
 {
-    using System;
+    using System;
     using System.IO;
     using System.Text;
 
@@ -17,7 +17,13 @@ namespace Microsoft.Azure.Cosmos.Json.Interop
     internal
 #endif
     sealed class CosmosDBToNewtonsoftWriter : Newtonsoft.Json.JsonWriter
-    {
+    {
+        /// <summary>
+        /// The built-in DateTime format "o"/ "O" is comparable to the custom format of: "yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fffffffK".
+        /// In order to remove the trailing zeros from the milli-second precision, we replace the lower-case f's with upper case ones.
+        /// </summary>
+        private const string RoundTripFormatWithoutTrailingZeros = "yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'FFFFFFFK";
+
         /// <summary>
         /// A CosmosDB JSON writer used for the actual implementation.
         /// </summary>
@@ -314,17 +320,21 @@ public override void WriteValue(sbyte value)
         public override void WriteValue(decimal value)
         {
             this.WriteValue((double)value);
-        }
-
+        }
+
         /// <summary>
         /// Writes a <see cref="DateTime"/> value.
         /// </summary>
-        /// <param name="value">The <see cref="DateTime"/> value to write.</param>
-        public override void WriteValue(DateTime value)
-        {
-            this.WriteValue(value.ToString("O"));
-        }
-
+        /// <param name="value">The <see cref="DateTime"/> value to write.</param>
+        public override void WriteValue(DateTime value)
+        {
+            // We use rount trip format for datetime parsing and trim the additional trailing zeros using a custom "O" format
+            // to maintain milliseconds precision.
+            this.WriteValue(
+                value.ToString(
+                    format: CosmosDBToNewtonsoftWriter.RoundTripFormatWithoutTrailingZeros));
+        }
+
         /// <summary>
         /// Writes a <see cref="byte"/>[] value.
         /// </summary>

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemIntegrationTests.cs

@@ -1,9 +1,11 @@
 namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
 {
     using System;
-    using System.Collections.Generic;
+    using System.Collections.Generic;
+    using System.IO;
     using System.Linq;
-    using System.Net;
+    using System.Net;
+    using System.Text;
     using System.Text.Json;
     using System.Text.Json.Serialization;
     using System.Threading;
@@ -31,8 +33,8 @@ public class CosmosItemIntegrationTests
 
         [TestInitialize]
         public async Task TestInitAsync()
-        {
-            this.connectionString = ConfigurationManager.GetEnvironmentVariable<string>("COSMOSDB_MULTI_REGION", null);
+        {
+            this.connectionString = ConfigurationManager.GetEnvironmentVariable<string>("COSMOSDB_MULTI_REGION", null);
 
             JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions()
             {
@@ -148,7 +150,93 @@ public async Task ReadMany2UnreachablePartitionsTest()
                 rule.Disable();
                 fiClient.Dispose();
             }
-        }
+        }
+
+        [TestMethod]
+        [Timeout(70000)]
+        [TestCategory("MultiRegion")]
+        public async Task DateTimeArrayRoundtrip_BinaryEncoding_CompareExtraDates_IntegrationTest()
+        {
+            string binaryEncodingEnabled = "binaryEncodingEnabled" + Guid.NewGuid().ToString("N");
+            string binaryEncodingDisabled = "binaryEncodingDisabled" + Guid.NewGuid().ToString("N");
+            string pk = "pk";
+            string testId = Guid.NewGuid().ToString();
+
+            string[] dateStrings =
+            {
+                "12/25/2023","2023-12-25","12-25-2023","25.12.2023","25/12/2023",
+                "Dec 25, 2023","Dec 25 2023","2023-12-25T10:00:00","2023-12-25T10:00:00.123",
+                "12/25/2023 10:00 AM","12/25/2023 10:00:00 AM","12/25/2023 10:00:00.123 AM","9999-12-31T23:59:59",
+                "2023-12-25T10:00:00.1","2023-12-25T10:00:00.12",
+                "2023-12-25T10:00:00.1234","2023-12-25T10:00:00.1234567"
+            };
+            string[] formats =
+            {
+                "MM/dd/yyyy","yyyy-MM-dd","MM-dd-yyyy","dd.MM.yyyy","dd/MM/yyyy",
+                "MMM dd, yyyy","MMM dd yyyy","yyyy-MM-ddTHH:mm:ss","yyyy-MM-ddTHH:mm:ss.fff",
+                "yyyy-MM-ddTHH:mm:ss.f","yyyy-MM-ddTHH:mm:ss.ff","yyyy-MM-ddTHH:mm:ss.ffff",
+                "yyyy-MM-ddTHH:mm:ss.fffffff","MM/dd/yyyy hh:mm tt","MM/dd/yyyy hh:mm:ss tt",
+                "MM/dd/yyyy hh:mm:ss.fff tt"
+            };
+            DateTime[] parsedDates = dateStrings
+                .Select(s => DateTime.ParseExact(s, formats, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.None))
+                .ToArray();
+
+            TestCosmosItem testItem = new TestCosmosItem(
+                id: testId,
+                pk: pk,
+                title: "title",
+                email: "[email protected]",
+                body: "Binary encoding test document.",
+                createdUtc: DateTime.UtcNow,
+                modifiedUtc: DateTime.Parse("2025-03-26T20:22:20Z", null, System.Globalization.DateTimeStyles.AdjustToUniversal),
+                extraDates: parsedDates);
+
+            Database db = this.database;
+            ContainerResponse containerBEEnabledResponse = await db.CreateContainerAsync(binaryEncodingEnabled, "/pk");
+            ContainerResponse containerBEDisabledResponse = await db.CreateContainerAsync(binaryEncodingDisabled, "/pk");
+
+            try
+            {
+                // BinaryEncodingEnabled = True
+                Environment.SetEnvironmentVariable(ConfigurationManager.BinaryEncodingEnabled, "True");
+                string rawJsonBEEnabled;
+                string rawJsonBEDisabled;
+                using (CosmosClient clientBinaryEncodingEnabled = new CosmosClient(this.connectionString))
+                {
+                    Container containerBinaryEncodingEnabled = clientBinaryEncodingEnabled.GetDatabase(db.Id).GetContainer(binaryEncodingEnabled);
+                    await containerBinaryEncodingEnabled.CreateItemAsync(testItem, new Microsoft.Azure.Cosmos.PartitionKey(pk));
+                    using ResponseMessage response = await containerBinaryEncodingEnabled.ReadItemStreamAsync(testId, new Microsoft.Azure.Cosmos.PartitionKey(pk));
+                    using StreamReader reader = new StreamReader(response.Content, Encoding.UTF8);
+                    rawJsonBEEnabled = await reader.ReadToEndAsync();
+
+                }
+
+                // BinaryEncodingEnabled = False
+                Environment.SetEnvironmentVariable(ConfigurationManager.BinaryEncodingEnabled, "False");
+                using (CosmosClient clientBinaryEncodingDisabled = new CosmosClient(this.connectionString))
+                {
+                    Container containerBinaryEncodingDisabled = clientBinaryEncodingDisabled.GetDatabase(db.Id).GetContainer(binaryEncodingDisabled);
+                    await containerBinaryEncodingDisabled.CreateItemAsync(testItem, new Microsoft.Azure.Cosmos.PartitionKey(pk));
+                    using ResponseMessage response = await containerBinaryEncodingDisabled.ReadItemStreamAsync(testId, new Microsoft.Azure.Cosmos.PartitionKey(pk));
+                    using StreamReader reader = new StreamReader(response.Content, Encoding.UTF8);
+                    rawJsonBEDisabled = await reader.ReadToEndAsync();
+                }
+
+                using JsonDocument docTrue = JsonDocument.Parse(rawJsonBEEnabled);
+                using JsonDocument docFalse = JsonDocument.Parse(rawJsonBEDisabled);
+
+                string extraDatesTrue = docTrue.RootElement.GetProperty("ExtraDates").GetRawText();
+                string extraDatesFalse = docFalse.RootElement.GetProperty("ExtraDates").GetRawText();
+
+                Assert.AreEqual(extraDatesTrue, extraDatesFalse, $"ExtraDates JSON mismatch:\nTrue: {extraDatesTrue}\nFalse: {extraDatesFalse}");
+            }
+            finally
+            {
+                await containerBEEnabledResponse.Container.DeleteContainerAsync();
+                await containerBEDisabledResponse.Container.DeleteContainerAsync();
+            }
+        }
 
         [TestMethod]
         [TestCategory("MultiRegion")]
@@ -1376,6 +1464,41 @@ await this.container.DeleteItemAsync<CosmosIntegrationTestObject>(
             {
                 // Ignore
             }
+        }
+
+        public sealed class TestCosmosItem
+        {
+            [JsonConstructor]
+            public TestCosmosItem(
+                string id,
+                string pk,
+                string title,
+                string email,
+                string body,
+                DateTime createdUtc,
+                DateTime modifiedUtc,
+                DateTime[] extraDates)
+            {
+                this.id = id;
+                this.pk = pk;
+                this.title = title;
+                this.email = email;
+                this.body = body;
+                this.CreatedUtc = createdUtc;
+                this.ModifiedUtc = modifiedUtc;
+                this.ExtraDates = extraDates;
+            }
+
+#pragma warning disable IDE1006
+            public string id { get; }
+            public string pk { get; }
+            public string title { get; }
+            public string email { get; }
+            public string body { get; }
+#pragma warning restore IDE1006 // Naming Styles
+            public DateTime CreatedUtc { get; }
+            public DateTime ModifiedUtc { get; }
+            public DateTime[] ExtraDates { get; }
         }
     }
 }

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/NewtonsoftInteropTests.cs

@@ -210,6 +210,68 @@ public void AllPrimitivesObjectTest()
             NewtonsoftInteropTests.VerifyNewtonsoftInterop<JObject>(value);
         }
 
+        [TestMethod]
+        [Owner("dkunda")]
+        public void AllDatetimeVariationsTest()
+        {
+            // Arrange
+            JObject jsonObject = new JObject();
+
+            // 1. Current UTC DateTime using JProperty
+            DateTime utcNow = DateTime.UtcNow;
+            JProperty utcNowProperty = new JProperty("currentUtcDateTime", utcNow);
+            jsonObject.Add(utcNowProperty);
+
+            // 2. Current Local DateTime using JProperty
+            DateTime localNow = DateTime.Now;
+            JProperty localNowProperty = new JProperty("currentLocalDateTime", localNow);
+            jsonObject.Add(localNowProperty);
+
+            // 3. Date Only (formatted as string) using JProperty
+            DateTime dateOnly = new DateTime(2023, 10, 26, 0, 0, 0, DateTimeKind.Unspecified);
+            JProperty dateOnlyProperty = new JProperty("specificDateOnly", dateOnly.ToString("yyyy-MM-dd"));
+            jsonObject.Add(dateOnlyProperty);
+
+            // 4. Time Only (formatted as string) using JProperty
+            DateTime timeOnly = new DateTime(1, 1, 1, 14, 30, 0, DateTimeKind.Unspecified);
+            JProperty timeOnlyProperty = new JProperty("specificTimeOnly", timeOnly.ToString("HH:mm:ss"));
+            jsonObject.Add(timeOnlyProperty);
+
+            // 5. DateTime with milliseconds using JProperty
+            DateTime preciseDateTime = new DateTime(2024, 5, 29, 10, 15, 30, 123, DateTimeKind.Local);
+            JProperty preciseDateTimeProperty = new JProperty("preciseDateTime", preciseDateTime);
+            jsonObject.Add(preciseDateTimeProperty);
+
+            // 6. DateTime in ISO 8601 format (UTC) using JProperty
+            DateTime isoUtcDateTime = new DateTime(2025, 1, 15, 8, 0, 0, DateTimeKind.Utc);
+            JProperty isoUtcDateTimeProperty = new JProperty("isoUtcDateTime", isoUtcDateTime.ToString("o", CultureInfo.InvariantCulture));
+            jsonObject.Add(isoUtcDateTimeProperty);
+
+            // 7. DateTime in custom format using JProperty
+            DateTime customFormattedDateTime = new DateTime(2022, 7, 1, 9, 45, 10, DateTimeKind.Local);
+            JProperty customFormattedDateTimeProperty = new JProperty("customFormattedDateTime", customFormattedDateTime.ToString("MM/dd/yyyy HH:mm:ss"));
+            jsonObject.Add(customFormattedDateTimeProperty);
+
+            // 8. Nullable DateTime (representing a missing or optional date) using JProperty
+            DateTime? nullableDateTime = null;
+            JProperty nullableDateTimeProperty = new JProperty("nullableDateTime", nullableDateTime);
+            jsonObject.Add(nullableDateTimeProperty);
+
+            // 9. MinValue DateTime using JProperty
+            JProperty minDateTimeProperty = new JProperty("minDateTime", DateTime.MinValue);
+            jsonObject.Add(minDateTimeProperty);
+
+            // 10. MaxValue DateTime using JProperty
+            JProperty maxDateTimeProperty = new JProperty("maxDateTime", DateTime.MaxValue);
+            jsonObject.Add(maxDateTimeProperty);
+
+            // 11. MaxValue DateTime using JProperty
+            JProperty exactDateTimeProperty = new JProperty("exactDateTime", DateTime.Parse("2025-03-26T20:22:20Z"));
+            jsonObject.Add(exactDateTimeProperty);
+
+            NewtonsoftInteropTests.VerifyNewtonsoftInterop<JObject>(jsonObject);
+        }
+
         public enum Day { Sun, Mon, Tue, Wed, Thu, Fri, Sat };
 
         public sealed class ObjectWithAttributes