-
Notifications
You must be signed in to change notification settings - Fork 498
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Client encryption]: Add JsonNodeSqlSerializer (#4779)
# Pull Request Template ## Description Initial commit for JsonNode on Encryption path. This is currently not executed on any production/preview path. Depends on JsonNode features available from System.Text.Json 8.0+. - Adds JsonNodeSqlSerializer - JObjectSqlSerializer now doesn't format inner serialized JArrays/JObjects (with line breaks/indentations) To be processed after #4766 ## Type of change Please delete options that are not relevant. - [] New feature (non-breaking change which adds functionality) ## Closing issues Contributes to #4678 --------- Co-authored-by: Juraj Blazek <[email protected]> Co-authored-by: juraj-blazek <[email protected]> Co-authored-by: Santosh Kulkarni <[email protected]>
- Loading branch information
4 people
authored
Oct 10, 2024
1 parent
b5d7da0
commit 4f515bb
Showing
5 changed files
with
190 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
93 changes: 93 additions & 0 deletions
93
Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| // ------------------------------------------------------------ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // ------------------------------------------------------------ | ||
|
|
||
| #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER | ||
| namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation | ||
| { | ||
| using System; | ||
| using System.Diagnostics; | ||
| using System.Text.Json; | ||
| using System.Text.Json.Nodes; | ||
| using Microsoft.Data.Encryption.Cryptography.Serializers; | ||
|
|
||
| internal class JsonNodeSqlSerializer | ||
| { | ||
| private static readonly SqlBitSerializer SqlBoolSerializer = new (); | ||
| private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); | ||
| private static readonly SqlBigIntSerializer SqlLongSerializer = new (); | ||
|
|
||
| // UTF-8 encoding. | ||
| private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); | ||
|
|
||
| #pragma warning disable SA1101 // Prefix local calls with this - false positive on SerializeFixed | ||
| internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JsonNode propertyValue, ArrayPoolManager arrayPoolManager) | ||
| { | ||
| byte[] buffer; | ||
| int length; | ||
|
|
||
| if (propertyValue == null) | ||
| { | ||
| return (TypeMarker.Null, null, -1); | ||
| } | ||
|
|
||
| switch (propertyValue.GetValueKind()) | ||
| { | ||
| case JsonValueKind.Undefined: | ||
| Debug.Assert(false, "Undefined value cannot be in the JSON"); | ||
| return (default, null, -1); | ||
| case JsonValueKind.Null: | ||
| Debug.Assert(false, "Null type should have been handled by caller"); | ||
| return (TypeMarker.Null, null, -1); | ||
| case JsonValueKind.True: | ||
| (buffer, length) = SerializeFixed(SqlBoolSerializer, true); | ||
| return (TypeMarker.Boolean, buffer, length); | ||
| case JsonValueKind.False: | ||
| (buffer, length) = SerializeFixed(SqlBoolSerializer, false); | ||
| return (TypeMarker.Boolean, buffer, length); | ||
| case JsonValueKind.Number: | ||
| if (long.TryParse(propertyValue.ToJsonString(), out long longValue)) | ||
| { | ||
| (buffer, length) = SerializeFixed(SqlLongSerializer, longValue); | ||
| return (TypeMarker.Long, buffer, length); | ||
| } | ||
| else if (double.TryParse(propertyValue.ToJsonString(), out double doubleValue)) | ||
| { | ||
| (buffer, length) = SerializeFixed(SqlDoubleSerializer, doubleValue); | ||
| return (TypeMarker.Double, buffer, length); | ||
| } | ||
| else | ||
| { | ||
| throw new InvalidOperationException("Unsupported Number type"); | ||
| } | ||
|
|
||
| case JsonValueKind.String: | ||
| (buffer, length) = SerializeString(propertyValue.GetValue<string>()); | ||
| return (TypeMarker.String, buffer, length); | ||
| case JsonValueKind.Array: | ||
| (buffer, length) = SerializeString(propertyValue.ToJsonString()); | ||
| return (TypeMarker.Array, buffer, length); | ||
| case JsonValueKind.Object: | ||
| (buffer, length) = SerializeString(propertyValue.ToJsonString()); | ||
| return (TypeMarker.Object, buffer, length); | ||
| default: | ||
| throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.GetValueKind()}"); | ||
| } | ||
|
|
||
| (byte[], int) SerializeFixed<T>(IFixedSizeSerializer<T> serializer, T value) | ||
| { | ||
| byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); | ||
| int length = serializer.Serialize(value, buffer); | ||
| return (buffer, length); | ||
| } | ||
|
|
||
| (byte[], int) SerializeString(string value) | ||
| { | ||
| byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); | ||
| int length = SqlVarCharSerializer.Serialize(value, buffer); | ||
| return (buffer, length); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| #endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
92 changes: 92 additions & 0 deletions
92
...crosoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER | ||
|
|
||
| namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation | ||
| { | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Text.Json.Nodes; | ||
| using Microsoft.Azure.Cosmos.Encryption.Custom; | ||
| using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; | ||
| using Microsoft.VisualStudio.TestTools.UnitTesting; | ||
| using Newtonsoft.Json.Linq; | ||
|
|
||
| [TestClass] | ||
| public class JsonNodeSqlSerializerTests | ||
| { | ||
| private static ArrayPoolManager _poolManager; | ||
|
|
||
| [ClassInitialize] | ||
| public static void ClassInitialize(TestContext context) | ||
| { | ||
| _ = context; | ||
| _poolManager = new ArrayPoolManager(); | ||
| } | ||
|
|
||
| [TestMethod] | ||
| [DynamicData(nameof(SerializationSamples))] | ||
| public void Serialize_SupportedValue(JsonNode testNode, byte expectedType, byte[] expectedBytes, int expectedLength) | ||
| { | ||
| JsonNodeSqlSerializer serializer = new(); | ||
|
|
||
| (TypeMarker serializedType, byte[] serializedBytes, int serializedBytesCount) = serializer.Serialize(testNode, _poolManager); | ||
|
|
||
| Assert.AreEqual((TypeMarker)expectedType, serializedType); | ||
| Assert.AreEqual(expectedLength, serializedBytesCount); | ||
| if (expectedLength == -1) | ||
| { | ||
| Assert.IsTrue(serializedBytes == null); | ||
| } | ||
| else | ||
| { | ||
| Assert.IsTrue(expectedBytes.SequenceEqual(serializedBytes.AsSpan(0, serializedBytesCount).ToArray())); | ||
| } | ||
| } | ||
|
|
||
| public static IEnumerable<object[]> SerializationSamples | ||
| { | ||
| get | ||
| { | ||
| List<object[]> values = new() | ||
| { | ||
| new object[] {JsonValue.Create((string)null), (byte)TypeMarker.Null, null, -1 }, | ||
| new object[] {JsonValue.Create(true), (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(true), 8}, | ||
| new object[] {JsonValue.Create(false), (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(false), 8}, | ||
| new object[] {JsonValue.Create(192), (byte)TypeMarker.Long, GetNewtonsoftValueEquivalent(192), 8}, | ||
| new object[] {JsonValue.Create(192.5), (byte)TypeMarker.Double, GetNewtonsoftValueEquivalent(192.5), 8}, | ||
| new object[] {JsonValue.Create(testString), (byte)TypeMarker.String, GetNewtonsoftValueEquivalent(testString), 11}, | ||
| new object[] {JsonValue.Create(testArray), (byte)TypeMarker.Array, GetNewtonsoftValueEquivalent(testArray), 10}, | ||
| new object[] {JsonValue.Create(testClass), (byte)TypeMarker.Object, GetNewtonsoftValueEquivalent(testClass), 33} | ||
| }; | ||
|
|
||
| return values; | ||
| } | ||
| } | ||
|
|
||
| private static readonly string testString = "Hello world"; | ||
| private static readonly int[] testArray = new[] {10, 18, 19}; | ||
| private static readonly TestClass testClass = new() { SomeInt = 1, SomeString = "asdf" }; | ||
|
|
||
| private class TestClass | ||
| { | ||
| public int SomeInt { get; set; } | ||
| public string SomeString { get; set; } | ||
| } | ||
|
|
||
| private static byte[] GetNewtonsoftValueEquivalent<T>(T value) | ||
| { | ||
| JObjectSqlSerializer serializer = new (); | ||
| JToken token = value switch | ||
| { | ||
| int[] => new JArray(value), | ||
| TestClass => JObject.FromObject(value), | ||
| _ => new JValue(value), | ||
| }; | ||
| (TypeMarker _, byte[] bytes, int lenght) = serializer.Serialize(token, _poolManager); | ||
| return bytes.AsSpan(0, lenght).ToArray(); | ||
| } | ||
|
|
||
| } | ||
| } | ||
|
|
||
| #endif |