From d6f89ea6f298e6b7459fa8246a22d8efe1aeb045 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 12 Sep 2024 13:41:09 +0200 Subject: [PATCH 01/49] Add baseline benchmarks for `Microsoft.Azure.Cosmos.Encryption.Custom` --- .../src/AssemblyInfo.cs | 3 +- .../EncryptionBenchmark.cs | 102 ++++++++++++++++++ ...Encryption.Custom.Performance.Tests.csproj | 26 +++++ .../Program.cs | 21 ++++ .../Readme.md | 19 ++++ .../TestDoc.cs | 62 +++++++++++ Microsoft.Azure.Cosmos.sln | 31 +++++- 7 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs index c9e211085f..aca9899cc7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs @@ -5,4 +5,5 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.Tests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] -[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] \ No newline at end of file +[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] +[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs new file mode 100644 index 0000000000..2e92274e6c --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -0,0 +1,102 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests +{ + using System.IO; + using BenchmarkDotNet.Attributes; + using Microsoft.Data.Encryption.Cryptography; + using Moq; + + [RPlotExporter] + public partial class EncryptionBenchmark + { + private static readonly byte[] DekData = Enumerable.Repeat((byte)0, 32).ToArray(); + private static readonly DataEncryptionKeyProperties DekProperties = new( + "id", + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + DekData, + new EncryptionKeyWrapMetadata("name", "value"), DateTime.UtcNow); + private static readonly Mock StoreProvider = new(); + + private TestDoc? testDoc; + private CosmosEncryptor? encryptor; + private Custom.DataEncryptionKey? dek; + + private EncryptionOptions? encryptionOptions; + private byte[]? encryptedData; + private byte[]? plaintext; + + [Params(1, 10, 100)] + public int DocumentSizeInKb { get; set; } + + [GlobalSetup] + public async Task Setup() + { + StoreProvider + .Setup(x => x.UnwrapKey(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(DekData); + + this.dek = this.CreateMdeDek(); + + Mock keyProvider = new(); + keyProvider + .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(() => this.dek); + + this.encryptor = new(keyProvider.Object); + this.testDoc = TestDoc.Create(approximateSize: this.DocumentSizeInKb * 1024); + + this.encryptionOptions = CreateEncryptionOptions(); + + this.plaintext = EncryptionProcessor.BaseSerializer.ToStream(this.testDoc).ToArray(); + + Stream encryptedStream = await EncryptionProcessor.EncryptAsync( + new MemoryStream(this.plaintext), + this.encryptor, + this.encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + using MemoryStream memoryStream = new MemoryStream(); + + encryptedStream.CopyTo(memoryStream); + this.encryptedData = memoryStream.ToArray(); + } + + [Benchmark] + public async Task Encrypt() + { + await EncryptionProcessor.EncryptAsync( + new MemoryStream(this.plaintext!), + this.encryptor, + this.encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + } + + [Benchmark] + public async Task Decrypt() + { + await EncryptionProcessor.DecryptAsync( + new MemoryStream(this.encryptedData!), + this.encryptor, + new CosmosDiagnosticsContext(), + CancellationToken.None); + } + + private Custom.DataEncryptionKey CreateMdeDek() + { + return new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue); + } + + private static EncryptionOptions CreateEncryptionOptions() + { + EncryptionOptions options = new() + { + DataEncryptionKeyId = "dekId", + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt + }; + + return options; + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj new file mode 100644 index 0000000000..3599027dc2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -0,0 +1,26 @@ + + + + Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests + Exe + net6 + enable + enable + + + + + + + + + + + + + true + true + ..\..\..\testkey.snk + + + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs new file mode 100644 index 0000000000..5304787e46 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs @@ -0,0 +1,21 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests +{ + using BenchmarkDotNet.Configs; + using BenchmarkDotNet.Diagnosers; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Running; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + internal class Program + { + public static void Main(string[] args) + { + ManualConfig dontRequireSlnToRunBenchmarks = ManualConfig + .Create(DefaultConfig.Instance) + .AddJob(Job.MediumRun.WithToolchain(InProcessEmitToolchain.Instance)) + .AddDiagnoser(MemoryDiagnoser.Default); + + BenchmarkRunner.Run(dontRequireSlnToRunBenchmarks, args); + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md new file mode 100644 index 0000000000..967eed7430 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -0,0 +1,19 @@ +``` ini + +BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) +11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores +.NET SDK=9.0.100-preview.7.24407.12 + [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 + +Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 +LaunchCount=2 WarmupCount=10 + +``` +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **47.90 μs** | **1.284 μs** | **1.842 μs** | **4.5776** | **1.1597** | **-** | **56.22 KB** | +| Decrypt | 1 | 59.67 μs | 1.041 μs | 1.558 μs | 5.2490 | 1.3428 | - | 64.79 KB | +| **Encrypt** | **10** | **154.57 μs** | **2.728 μs** | **3.998 μs** | **20.7520** | **4.1504** | **-** | **255.95 KB** | +| Decrypt | 10 | 220.03 μs | 6.124 μs | 8.585 μs | 29.0527 | 5.8594 | - | 357.41 KB | +| **Encrypt** | **100** | **2,761.51 μs** | **213.677 μs** | **319.822 μs** | **218.7500** | **173.8281** | **142.5781** | **2459.89 KB** | +| Decrypt | 100 | 2,445.99 μs | 136.839 μs | 200.577 μs | 347.6563 | 300.7813 | 253.9063 | 3406.33 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs new file mode 100644 index 0000000000..824045c489 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs @@ -0,0 +1,62 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests +{ + using System.Text; + using Newtonsoft.Json; + + public partial class EncryptionBenchmark + { + internal class TestDoc + { + public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt", "/SensitiveDict" }; + + [JsonProperty("id")] + public string Id { get; set; } + + public string NonSensitive { get; set; } + + public string SensitiveStr { get; set; } + + public int SensitiveInt { get; set; } + + public Dictionary SensitiveDict { get; set; } + + public TestDoc() + { + } + + public static TestDoc Create(int approximateSize = -1) + { + return new TestDoc() + { + Id = Guid.NewGuid().ToString(), + NonSensitive = Guid.NewGuid().ToString(), + SensitiveStr = Guid.NewGuid().ToString(), + SensitiveInt = new Random().Next(), + SensitiveDict = GenerateBigDictionary(approximateSize), + }; + } + + private static Dictionary GenerateBigDictionary(int approximateSize) + { + const int stringSize = 100; + int items = Math.Max(1, approximateSize / stringSize); + + return Enumerable.Range(1, items).ToDictionary(x => x.ToString(), y => GenerateRandomString(stringSize)); + } + + private static string GenerateRandomString(int size) + { + Random rnd = new Random(); + const string characters = "abcdefghijklmnopqrstuvwxyz0123456789"; + + StringBuilder sb = new(); + for (int i = 0; i < size; i++) + { + sb.Append(characters[rnd.Next(0, characters.Length)]); + } + return sb.ToString(); + } + } + + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index d412905195..6fa5e4c3f2 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29123.88 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35209.166 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject @@ -34,6 +34,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos.Encr EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FaultInjection", "Microsoft.Azure.Cosmos\FaultInjection\src\FaultInjection.csproj", "{021DDC27-02EF-42C4-9A9E-AA600833C2EE}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests", "Microsoft.Azure.Cosmos.Encryption.Custom\tests\Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests\Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj", "{CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Cover|Any CPU = Cover|Any CPU @@ -164,6 +166,30 @@ Global {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|Any CPU.Build.0 = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.ActiveCfg = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.Build.0 = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.Build.0 = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.ActiveCfg = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.Build.0 = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|Any CPU.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|Any CPU.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|x64.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|x64.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|x64.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|x64.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|Any CPU.Build.0 = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|x64.ActiveCfg = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -175,6 +201,7 @@ Global {D7C78D76-A740-4129-BAAE-894640F95D74} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} {F87719DB-BB52-4B12-9D9C-F6AE30BAB3D7} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} {B5B3631D-AC2F-4257-855D-D6FE12F20B60} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {C6A1D820-CB03-4DE6-87D1-46EF476F0040} From 8928b880d43b58aea3fe3bc78ff6ba0dc40198a2 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 12 Sep 2024 14:05:57 +0200 Subject: [PATCH 02/49] Cleanup --- .../EncryptionBenchmark.cs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 2e92274e6c..7a1bb21549 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -18,7 +18,6 @@ public partial class EncryptionBenchmark private TestDoc? testDoc; private CosmosEncryptor? encryptor; - private Custom.DataEncryptionKey? dek; private EncryptionOptions? encryptionOptions; private byte[]? encryptedData; @@ -34,12 +33,10 @@ public async Task Setup() .Setup(x => x.UnwrapKey(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(DekData); - this.dek = this.CreateMdeDek(); - Mock keyProvider = new(); keyProvider .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(() => this.dek); + .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); this.testDoc = TestDoc.Create(approximateSize: this.DocumentSizeInKb * 1024); @@ -55,8 +52,7 @@ public async Task Setup() new CosmosDiagnosticsContext(), CancellationToken.None); - using MemoryStream memoryStream = new MemoryStream(); - + using MemoryStream memoryStream = new MemoryStream(); encryptedStream.CopyTo(memoryStream); this.encryptedData = memoryStream.ToArray(); } @@ -82,11 +78,6 @@ await EncryptionProcessor.DecryptAsync( CancellationToken.None); } - private Custom.DataEncryptionKey CreateMdeDek() - { - return new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue); - } - private static EncryptionOptions CreateEncryptionOptions() { EncryptionOptions options = new() From 9dc7d5416803b524a86374e21c14aa9bf75abe02 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 12 Sep 2024 15:02:31 +0200 Subject: [PATCH 03/49] Use set of static test data for benchmarks --- .../EncryptionBenchmark.cs | 17 ++++++++++++----- ...s.Encryption.Custom.Performance.Tests.csproj | 12 ++++++++++++ .../Readme.md | 12 ++++++------ .../sampledata/testdoc-100kb.json | 1 + .../sampledata/testdoc-10kb.json | 1 + .../sampledata/testdoc-1kb.json | 1 + 6 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 7a1bb21549..d8e4c7d8a2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -16,7 +16,6 @@ public partial class EncryptionBenchmark new EncryptionKeyWrapMetadata("name", "value"), DateTime.UtcNow); private static readonly Mock StoreProvider = new(); - private TestDoc? testDoc; private CosmosEncryptor? encryptor; private EncryptionOptions? encryptionOptions; @@ -39,11 +38,8 @@ public async Task Setup() .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); - this.testDoc = TestDoc.Create(approximateSize: this.DocumentSizeInKb * 1024); - this.encryptionOptions = CreateEncryptionOptions(); - - this.plaintext = EncryptionProcessor.BaseSerializer.ToStream(this.testDoc).ToArray(); + this.plaintext = this.LoadTestDoc(); Stream encryptedStream = await EncryptionProcessor.EncryptAsync( new MemoryStream(this.plaintext), @@ -89,5 +85,16 @@ private static EncryptionOptions CreateEncryptionOptions() return options; } + + private byte[] LoadTestDoc() + { + string name = $"Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.sampledata.testdoc-{this.DocumentSizeInKb}kb.json"; + using Stream resourceStream = typeof(EncryptionBenchmark).Assembly.GetManifestResourceStream(name)!; + + byte[] buffer = new byte[resourceStream!.Length]; + resourceStream.Read(buffer, 0, buffer.Length); + + return buffer; + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 3599027dc2..e32a42c36c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -8,6 +8,18 @@ enable + + + + + + + + + + + + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 967eed7430..bb4836273b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **47.90 μs** | **1.284 μs** | **1.842 μs** | **4.5776** | **1.1597** | **-** | **56.22 KB** | -| Decrypt | 1 | 59.67 μs | 1.041 μs | 1.558 μs | 5.2490 | 1.3428 | - | 64.79 KB | -| **Encrypt** | **10** | **154.57 μs** | **2.728 μs** | **3.998 μs** | **20.7520** | **4.1504** | **-** | **255.95 KB** | -| Decrypt | 10 | 220.03 μs | 6.124 μs | 8.585 μs | 29.0527 | 5.8594 | - | 357.41 KB | -| **Encrypt** | **100** | **2,761.51 μs** | **213.677 μs** | **319.822 μs** | **218.7500** | **173.8281** | **142.5781** | **2459.89 KB** | -| Decrypt | 100 | 2,445.99 μs | 136.839 μs | 200.577 μs | 347.6563 | 300.7813 | 253.9063 | 3406.33 KB | +| **Encrypt** | **1** | **60.05 μs** | **1.537 μs** | **2.300 μs** | **5.0659** | **1.2817** | **-** | **62.65 KB** | +| Decrypt | 1 | 70.76 μs | 0.812 μs | 1.164 μs | 5.7373 | 1.4648 | - | 71.22 KB | +| **Encrypt** | **10** | **165.23 μs** | **3.741 μs** | **5.365 μs** | **21.2402** | **3.6621** | **-** | **262.38 KB** | +| Decrypt | 10 | 231.32 μs | 4.627 μs | 6.635 μs | 29.5410 | 3.4180 | - | 363.84 KB | +| **Encrypt** | **100** | **2,572.40 μs** | **242.163 μs** | **362.458 μs** | **201.1719** | **126.9531** | **125.0000** | **2466.27 KB** | +| Decrypt | 100 | 2,952.48 μs | 397.387 μs | 557.081 μs | 255.8594 | 210.9375 | 160.1563 | 3412.88 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json new file mode 100644 index 0000000000..21d40c5f05 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json @@ -0,0 +1 @@ +{"id":"e010f76d-17ce-4165-8225-35ce05333ee3","NonSensitive":"bbdb1cda-3444-4cf1-9a0f-063ce785476e","SensitiveStr":"b4f1b63c-96f3-4739-a5c1-ef1ca06f458e","SensitiveInt":770489940,"SensitiveDict":{"1":"qwt2ya60f3t7vst0j3cs9fqdiv658do98f5dfmh55yq5kjk3zu7p9ogbr8ph8c7h0iuttixja7b30vq52z9wqujjkf43utdsirsg","2":"vjvr3z8ns12xrhyxfnh5aptwqb3ejtkoql1od8jdxurgfxk2xwavu88de98b5he7v08d77guujejjrui08czkn4k7arrr1ijaqy6","3":"7kg7uasrf2i10b3itl13lxpzlt2tefzh7tk5r2eyxeu98b39l0zm7p2rk6ivvwyeti6q8wlkwebemw5i6nrnq1uog5j0kzlvnem9","4":"qasrvpil3s88t8f3fmr9i8jie3tqji5pxo0iaui21v97ay7o9myfs00w56euwd3pz3y5311rul9g42wuro9rb5eal42k6zo5m897","5":"sgu94rueqoqrpiljdekiiezjp4s1ds6xsd37yvcm7ulqoeq9h1xasaqgv4u067bbzi2uhd2ixbqh2ol1v3y6ndwhznxcs1l80ynz","6":"8z9uw1q9lobl3vqzj9n9lolcjtgdmyhg6riiabrrn1cj7fozwgmtetvv5xfnh8mg5avdib5csgl6hrrpvs8xgrqfv9atz1rqfrfn","7":"y7mo0hhc12sr7unppcwkfkfkavu47ym1c4ogwx0em6t3ept6ksfhxgn80645cy4q0njaisjc76o0l74ek1wyu83n55zkycuc909d","8":"ml3gxghqanig46gb0t5x0eowc8x6lwkosh054jcypjkj0sdykaync2o4853k1cm1ttod7a5htzyw9tnl3ch6iypopdh3271a5b7i","9":"jd3pcvu99al4qt476yjfauw31la95wjm07vxv4g5b9y32i3holfe38921chp54h3qx21bb7h42ho51caslx4lvsq4a4uv0r6chhi","10":"8mfl983nto693ihpwy9y3mo3c6lba8f4epr2anq7r26g5loauf5jt204wpmid3vcw7wq4vxfss5pgdco9ipgp5pcpivlrn0f0e22","11":"3zf5k550nhtqah3lmwj0qn6z940np7jpu3wmra6i22sf6wce1tftz7u9uur5vjts9qvixj41f19f0afi39inw2icke3pz1zj33lf","12":"xiog07lz2g0fvctld0l8ltfmamd1jgd7mmxh3z1xmshkrhkwhc0joa3g2zj7mwn2s1dbeo9tfpcwa32cqutxp34924i5pxy80tjv","13":"fq7tmr6pofoppj4xloubu493zlldoyqreciapdsohlyp49731bp6kw7nk3hvsd5m90nyqvddgmkt7ku61yulpdzy7xn1tqq3bnba","14":"5p03agq9dpmtmtrdeojlkq1llb9oxxrbg07yhc4dt34po7nf0hf89epk38pfxzgl6l8ru3g9tavh346dnfmlagfgumaqvevy526x","15":"edi9zs677mgmnhzjvjmc7ormtzl4oa1yt97ei7n441to010w3hbrk0uj4rzulb7f39w8a20cbqcxrhzrhsta4bz1bnnv8rgyo4x5","16":"1z986f9aidg55rii7volmthfr48beb29l5ttabi684pgwlxnveyhq82v0zuv7i2mcx6mo8r17qh6wuqqasw4cyu7kmgumly2ckzd","17":"rlz20xqzaqdec6ec4hqaswz55f1ckop30vpgpm6g18ub3ksakdjg8azt2rgpnwg6ruyp28x2sniqt3zctasrc69p7txmt97s340k","18":"v80p03aa9x9sqz5qcl1iljzy3um0kxxwa1z1nfna60f6ma48c023y7c0833th99bgta4qjrwir0d1c6pud8gspz63yrn8av8fdct","19":"ijrq4hmjsv4binbc03anhfbnl418rfvcfgzyce08g5hhudpwtj7ii05ri64kcife7uh9jdl1gz9bzyzjpr2st0qbnjomihd1yg7w","20":"j3ipcp1gyzqizwbtumcv430jek7siwi978lgxij5yiru5q98xo3q5oiqx87v7d05bctwchk7oj8xfyj7194jsbz5z6kr6bvpi4hk","21":"5lwdli6md1iacxxbqp9zc12tp26w0qw4b5dg87aif54mb66111nip0jyherf5obz91n6v8bdnchiqfulwsdsu4kwfeud4z0uvaog","22":"zl6ztc3ncnc61kwd0crhk7fpwignjggdivhje1yrvgtv31ksk63ttxe9001fa03jxzmwogh2ombrgkvdokihndkcjhhxhrn2ruon","23":"p4uemv64y8497cnexs05ih3i3of1k9112cedrrvz1j7zzv7tg3i4a5avbcdvw23vjs8ivnec7zdjxzihnpm7d6k6m5fqt09fbnrx","24":"gtgky4vi7x6oskpnqlv8eymkru8j19fy9rubxspr6zkjpm1zin34c1hacu21cp4xfq5hsclhmf443avr8g9hn51cr3r9h55et3se","25":"iqp21mfuklwrix35ixwuwxqxwmt4jhywfb5h3t11q4e5p1p1zqptjbipvrbhilc5icbvltfcv7vfcnsncjqfz8yut4h9gaymj06e","26":"4x7gkhll44182spczbl4bmtyhzs9mmefwxrfx6v72owz16ug59nrgch8o8wle0x8fm0090uyck8bkgxxl8fssnrftaypf5uwx84f","27":"cdwh6okkrd4f3pgyvessf2i9hl6bq4vhzvwucr2eyu0l53vq86pdwr6qxoe1vq8iwtzbfu2nwe5eg23zeyas10c2p9z5o9pjpl4p","28":"k20tzkucfijvtux1087365s69lrsg6axm2f7v03g4riygpzxj4e54rxfvvcw49xbyrq9oaofrroxw5owpbnihv3d35b72xolycpd","29":"fjy7dhkw5w06fm7vzlpjg29vg03605p2hivrrndrs4su432heklth18ns080z9ek8ihr6bssxc4mn5lzrmim4077qo7i1njqxyjx","30":"ld3s0rpja4naxd72v9u7op8rfmojlrv13z15tg1vmquwo4dvm2z1qpr6qn0hm4npva740kj582ltfzl66qnx35oec4fkul7kdftw","31":"fxro95cche8o7lprcf1sbl9u0biasa2nj9xqgvzaogep46rx6mbxtjs99lnrzgqson4nxz8x2rpegb2l9g0k0kepz4s9lbkr2sn3","32":"8cqqhyjskso2lh5xnjzx5qpei5149hfb9951wqwojdnbn86gaq1i2rpof4k2rzqfib5qqisfahg3awi1ug6owmezzwh3jsb9k7di","33":"mt8alt08khh3yitrp47npz6bcnbu6oqr9aiwyocww87k0wwtyqtwz2hyan7vkmnyz8ldnbsxinxbx9kep8ajofladz5ni84l5xrw","34":"zvm7or3haiy5bv03njeo03vs0f5gk6c4nnloq5vmoaq3v89zgm94350n7o3t6p7cv64i3igklsylnd9beiqp9dfwzgphffsi6pnm","35":"f0wzkqyacsnlft7g8ytcz3v6itbvrssn08pq0jkiufoyarteuemej4qgfmad88h1pq1daph09ovmm5dgzotrti0lcj8zf64sgb3p","36":"cubf6nafgxt71f6lr6d1b887i4r6pwscllaslo7pfznjtz1ijwf1e3hl4idg1d27k1s0k4hy5k1snztzkqn535735e66ecbhxq8l","37":"exyo0bmxxrd7rbiq7wd4wrta6rlc9x4oyxa09ctq259acpauqa413cpmz28usjg2hijcvarug64vccyvg8h6o6fznrcex4mmu95z","38":"3xlf2u73l8yt2r2v1qpavh2e81yyejgahelmeje1ckowmtlf1jqxiddkpj120cw54yax77jdk724qyfluvjqdkbinjd5zn019x53","39":"nqle32172bknqrze6zwy4t4jaft6z14kmnufopk1dxygrrw248joqpd2mzcgb3wt82g7mn8kmfvy8qzdfz0pjf4eych9jyjeiau4","40":"aq32vdqky0tkvfii1vyxsh862czpdwi8g12j0vs8ssdqrqbyc535h356uuipp0sljwkn92qjqcj3g1miz6tnymm952uiy3yz01wa","41":"dmwfuq186j6dnedbp6rsj0ge3g07xg66g12udj3unku6va3h40c6bpmueq5cx00q0ysal22q0pjzia5pujokm83xs209eorz7tk4","42":"s4rv0mwelpvtg1o53vr9jlwkqgnb9cb0omfor8qxc77vgjlfluedwzbnzd3cgbgbbkvtyjfs1y2gkjafwq18mejsfvhu3kxt9kj9","43":"khxgqvw2q6wxpcokidcg88gpm610ig4cs6dephbeutyzg020jzabmnuitizz7unv3c8d0hbwe535y8flmargdvfs3r9az2r5myg9","44":"roc8qqxdmavb744qd6pe3d2cv2dti8og2wcr1yhxkyynsbld0kscrcc82cgsuay7x5xnu4qwni1774dymzxjvxwhtw99d61bngdo","45":"58l4eccws8nldagemuu0hcp4b1tftohobh07agj4yxk7a2czjycnu41ohxtn1mjl2aeht7u99cwo4qqp7wvxzlcmhrj84amu1l1u","46":"nash8eow6tb49k1m1nhzodmnvhmlz5af70n77yefa0g9xmhlh840g877ts6kyahedhjvugf3joui44ntpia9gzxt00z6jipfantt","47":"d7nhmhwgj7vghf6qggkxw2zev1xs2f95l7h990lxoyc6ktdvfe4gsqx0iscqa0t2mtclhjps7uhz1an4gnpe6a0ba0u527dgfawi","48":"vb6tbv7cazdydqvi8qcnj1fadfbqcic9s8a8p674oo5jjbe3wc05rlovz6covhac3qm3jxonbycv31wlktx9cspetuqoe93k6hcn","49":"vnrsuz93okofehv60xlv0f4wht3whoz4asenw87yakja2luioht825b9rg2ry64vvsyuzyt2bhoeqnz1wabyhytaqujbgjhr3sog","50":"fmckh5p1ku854hz1rah2poma0rw6i7y2ylyboskao82pd19h3vvrsd19xlw4rsgiubzfa8q6j6nskiznxomilsn82lsl394prxrx","51":"6lxfcvmismjli2x3tbte38gq0ee81zpen4lthbmgh5htvccyz8x82tjogmb1xcivmon4udo4a9uwr535ltxhobojjtej9yjv543a","52":"1s21v23ajpgy5ff51tie14oz0zvhrrvu19dmpfo6ycb6m4p478ihs1onut69ab6959bn1mmonmb5vqw71stp70vcsc5fo037xgbp","53":"wfwla1k144s3s2xxecdb7cs2ndm3t5w6y4zhem3mwv89qypwzi7z0tkcacx6jj73721jr8sqx6uplhwojooqv20162pw39k8y6i2","54":"cek6rjkudrcdbwkti0dd37a40d4t44gxz2kf096vr6ff3nyyh8h6mrh1fcfplkrck7u7c0pzasr4f17tx0zwrs4w9ca0hizxfumr","55":"oqt2q6qp31qrcl21qqfxq0j9v05885c3f7u0y2voyg2kotoyuhjzaj6xlj5kuyj7aclvyne8jirgwlghplpkb38tacyv1yqxhyda","56":"erc8ma2ql6nc8lxs9cezywd515dayhxujopnt8oj385qp9shxwca8zqqin88jabsiql1ceejufknu1gmzgp6h1cv4r0ukj7lw1m5","57":"gwwulnmocf1kcz35xho022tsy9ec1j3wa1sa1m9rlnwt6z7rq79pe1zehwk4v2f6p6gizlbyeaskbgufqps6lelubczgbv7haczy","58":"nawt8ac0cli65jfc44itfm6xw8rl9y3tvc4dg7rir67imzn0reli2lzl7rt0wqb2hx6bsj0fi6fdmisiume4kp66e3tymbkv5qvb","59":"dy80gxpm9sooohykpx8cejnvrn1bz5vqrpzwp3zqmuvj5lepavnx68j12v8ly4hygnoe31aqp16mu5r5tmvksum2cpl0afc0wyw2","60":"h1s7tg8733ikglzojmq4lla16g5mfixje4vxq0ajp7sik6wuw1ctpzm01xwa584is8bsrrmv5s2pvy3xnyrslnh2jtd09uqsp2xs","61":"qwn8ksa9vwucdfgl71dicvrsv4sgh4olbfvnigd62jolal1lu22ttjebweg3nabs0u3uzhbv5bdwmxmtmu5dcjmiymb197n5wurq","62":"xbg0byt7xmtw6gjcbsyo9lvzg7amfl46reuzpzpokrt5vo1gfle9q42m6vw4vaoqu1b0h9fuupwnebkl6ish7a901nnwh0b890ly","63":"lj0xzdhvtnii10ikqxwd7vtd1cz41z1cx9fcpbv494zltl88dj0g7mzhlbygt44bwqj4xoxt22e8txvrh3ocp18rodwznbgyt63g","64":"lvvnygyywhqqcspmbrn1eas0uqfta08pbjhqhmnxywwr10kyqb5tv654k9h6jdhvwehbdimqndb7e65roppp88vah2u9zqf1eqwe","65":"iwzxqnfa0we8s333i0qo1csyptswuzx8zstm93bi8u4trl2mme5rsxdogfsmoez7fpqr8zbka3pnet1a0z8eqiltfb55ag2wrdyc","66":"2bfivp8qdt3s58wtykddfys4j3ypossy3arlvjwbp2sj9gd4wctbjd8xmr8um3z2xihw9l5c6w0jlj373ks9wdjtb1wh8a7yhjb6","67":"nufqjuqrnzeyx4s16jv4kwavw4micmfevqvmi9uffqt4ft960pgibsscpmg5k410jmnv6eehflc4jqow87asxll4f4wi8h6x9lar","68":"kfb02ufcimu9k7x9h5u7t2t08kn65l7huhe4n25ukuzz50ik11xn3qk159hzfccfde86cc0j1gnv7w4mxokynv5o1augohcykogp","69":"2pq2flugd0gfs7mlwwkeikvwoegy3a1i2rqqmhae3z5g8plwonu25q9xm0bgnumfq8o2w1hkz39nd9zi0wqlx72fj8i8ykr3wfa0","70":"khgghtv7w7uw00l73xes0hf0auabq192tjpr0vjhkwrthexbgnoektv4yklvitb306wk0rs6hqr56sy6rc8c5wonfa7x5e0dksle","71":"67w6l85ln872nsleeet2vht06cq9b94zfoq44ilr48nqzef1c8j54xwtygwebwmwcn2yi4cxpazm7v0lyjs7ldvuc6kuvl7kb2ks","72":"we7ylh7k9hy1vx0v72ljolms34vetgzqotsuwv163viy2k5jf12tx1jb9lt5qhzyp48pa2gtolyck4b5k4v7vb5xf0ohbtk9a5r4","73":"ndg6ufbv635xi7ow63jqlgr4rqqdvh6h082rkz8ui911q8nlfxh1vwc1zat3hy642hepuqdfxzfe96bcvd2aano4c7izytayd17f","74":"71ploor208vumkdgt2xshtljz961tjep6248xrc5efpm5krijwqtisrxj9ms50n8uemiau26fhp3rdyopgzjneat11y02cnobrzh","75":"z9ebfmy95tikhengt79kztvht0n156dev010qzvw201vs9lg6f5vqrn3sbv7upl6l750a8co385xpfblwdlzpndigh0tl2005tvv","76":"t6rozbasnvi30kk9za0jieyy5ff60jdv80w7a3nn8m0mr3y8vc1hqu8rs1s302iwq5qn3am5sdmny0tg4uhrsc47chhfbk4s2xft","77":"kqpowkkv2pdq4c7y4kpqz3parbgv9v4pk2wesfzxr6ssnlhpdpdrh1zayke7gswtu7lgmd17tum34vazwgy6p8c2evwhzwtxdkix","78":"hw570z97ayivd746etnt9aatnne9g9cnfbtrn7qlfvwns6pfx5lhx811aslmjxy3z7ht3rivut96qb1rx0n0qvminwrbj8wgc4wt","79":"b33snb68uejym9bvckbto56vnyqvgss6axbfi619x948hn7nxvurltj4zf7iqub59585p5sasw3kno5t58ot2rm3qewgpfivr37g","80":"h20v5q35etwdsw4z05j8abjx5qny6uk79h241u0mmg2s1xdtyvtbpcd6i9w3b4napggmwhzla921i7m5ki05s7zftaeoo2vqba4g","81":"d9qiueg2lv78brwmnci1lloon61j307fhcsrnh4s1artjok4q8iat05foh0pczziuabx1c0g7sjzmh152n5ojbwgh1iz1z1xbf60","82":"ie2y7y8i2xshl1u72l9jex8ql2n56i5i27cogue4lcmlgjo06roxu7dbg8b2zw07bpptms7pq0ktkz09vyxdru6gdluk4225luga","83":"2mx1x5jc2eyvx9o9de4izikzz5cliihjpntdd1wmemzkhkufzieri9fvgjmuur9ky6vszw8j5z1pvq5vuqi2yjm1a4u7tb1l8efp","84":"5w2ufrqbi9pahyby6w1xqqsgtea53yyg4tdrnmo56fcb275gzrt3s8f9sctfrovjcg4exdodf29ksvyqnoh5je289jl92vcuc2qv","85":"7jd2nqzrong3f1xei3g0ytl3qap9j53ov3nn7hkvmyvrb2wnbrifmqrw3him28l4rfs2kr43r7kf9cwg7xacecgdhx5aw5to6c1w","86":"91y3qy89mitteonfotvif1vsa9lul5d0l7n4h5bm2rcu4lf7mnnt3tctt08y50412g88scm2n5rch377qg1rksjjus0atbbxefph","87":"lywssvp6ojt7pdrrqp0cqpph6d1qc0l4oopstqvgq3n520b9gq1ovlk15m298lizfont3m7g08o7w1zxq17uu1q97i9fo1bfhcfg","88":"w8ritbk5qlnv93f5kjaoojrlgyo0vkfluomx4qgqipznczn5lfy8bjqavclww6861d1on3ymn23x7j03kp0pgjrtxk5mii5iifft","89":"9q1rbjidwgzwtm46wd1ngnxjaq0jeolqnmb68hmotj3u3oui4fzmfmsj89c31tgdfsys9uf18p4nq9yvpyaynlzurn5zsbakua5m","90":"5aa97rpo3qs2nwsdzieiqzwtz0cheiqnwapivhq6t6slqwaki5e8utqc2lp91a0l3og7rgq48xvbrrynj1von1aa7l6cmfxu9lju","91":"if2mnsvvb5h2w0t1r4zmgx1ngph1n72drqp8evn7g40lo45zxsgszdvf50t8kk6rzd3ayge3i143rcmx43v7etbrx7lhsr5xvbug","92":"ev81wrtb95922kbs1ffgolvkmklz9na3ka9nz91chasajeuj38c23l02kpelh3bsuk9b53siffw2hw5d83n5cuoeugyrr6a6pu9a","93":"t5m0u61zhkmqp26ongg41g85wpir0o3868lk5gmh1uk9ijp6lfjf7v1k747ivtpelgayprg2ah6ud0zxq7421728apgezzbf7war","94":"bsxlwh2eikokss3yc8t9rpmyf37bt1ojvbsj0d1op3qavu73pekx3k3k0x3xl2prh35f25tk7j5vxoo66f1oq7cyulwp640xxvd0","95":"ur09m5a9sliiz6f8zjhbzl4r97xma384mzd0x2n54pn90s5er7higdxgohqkmi2964bw95gl3n7xijli27j8bn4rntuz9xufjs9g","96":"qdn4wd4qyh6s61gkhk3kas52dynw0hvggo202hsfw7f29k3t0xxse1f52cs8drd7dhxa28szrw42v5eg6f6sw9eiinu6wu9n0rau","97":"g7jnpbm5k3pfbxgft6wzh9onf7temjqamstaf8nuwmmk4zm5os5yru1511oavg2grch5n8p4mzdmxs62qmbeijabazn3ud8tsgfw","98":"yfmvs91kw3brgcm2tido1kmzvh8jx1s28yggdb0k8pv7ehxjsf2ymfphtlzvnknd8b00ano24n94luqd6tkuzlhbseld0wfwdg6z","99":"50bvqawehrwh7tp0brwrkb7bjql7rx1y53r96pv4dg23k2dfcaxytx1uhhcl7cfhucys7epmshumhtr3f5rx8twuf8blgqnsavzv","100":"n4vgzqcbik4a57o99qeydl2i8l47h1fleqalqzbvz1atnzs1kr4taud7uim88fjwnyljz9vajufdomjn83o3xrk7t697ys8rz64l","101":"rshrxc3kemqvxosz9pxi0finflmlubuyfovl02bdj04a4ur960542wovv86ay39y6g9i22yem77uefnl4le3jpsyo0yicpony6ga","102":"y1cdt4miizwzyztn5hrzdkc3qyqe955wbe87l2modj7955vdir34ikk12euhyf5iwkalojo5kq1l0wz2kvpd6vnibsr0fpsym00x","103":"p1a143tn8st9brom7f0o5iuvzn85mn57fgk4f7r7f94mdsxia232zk69cxprm874cqyijqn6vtwbuhtk17zs1ajy2pb8c6pgfl0v","104":"mtksiir2wnkvu0sytgkxn8fk5maso8z44g1s1qwsns7v9s2y5hnaq7k5uen9tvf24ear0j9hg3lnv9xirbn5s9d4cgsy5b0fgsrk","105":"26pk3x1pgyrh876mrm9ur68mwm2g8pupacp75bmzw6s4td6tkkbxp18q74hp5385mb2tls59yjb1cea5xeqpvoyful54peaffn9k","106":"wdc33hhknm0s6xa2ms3c933315fu9tlnbftvqsmlnb7cgj5sozj88cafkggxzcv4a68uw1c2y3h8ju4j2g1tl5y96b78nl5n2dzp","107":"dq76qb4ddn89kbqobm3jdnze3u55yyv3b3ob08hvl9wzq0fter0szmsbi35mhc9b7uy8qqikrn75vmt1m7s4hmfsz7q81s8s1j2f","108":"ku66tgz9onoezgl93yzlnvc2ogsncj6mf5ppm2lpsft8qnnp3hdelpqdbm780mnoyhvgd3a37i0pkfspcqmgr6a4zdqqhc0kn5tz","109":"981llvf8f6sy848kxqvj93ynyewu0t5yqsfzj34avl401dp1cb8ynld2s7l2twa7v811aiwfmjwj1l7xs6bhv3posj03ygk34m7c","110":"pkiaie7k5fj1qbt6p8cxdfloi5n6l8z1x2i9zr1jrz4smorokglakqggt2zbc0wtig1e76vuq3l09rt3g93nobfes5odek9nzv0n","111":"lwdw7f1shu67x92hqo54tmgt7m5jvni7nopejdajnznqdmx2ano6rsiq9q1ojf6z2xkg167x68tqjh15xx0hnb26ai87vri90oap","112":"fuclp0s7g0jlsxligsjfqmvnik7f3dx7r2g1mue6d9vz82djb1520a2smurfziw1xxgeg6pfp3rm9gt3x98kmfm9jfu6k0ep1mot","113":"5oa4993yc7wrvk4bqxs0evalp7iviqtjx8lpdy5wksqi4iw4elytm3dglazy5rz57mykho6n11jovhhsus2sjskaa4y427emej2n","114":"95iupp7zqdyyj1d961pf4kyks478456ubu2elf6wei7ttky3ll4aokxwhn2xq6dusu4kntesdvdth73w5fvf6urvvra6wtc9wxzy","115":"aefa23tkcavwvcrjyir21kcpzg00fvs2mn5nin95gf3zp71k2zlbzp0y11bqfsm6bgc8dya160dg48oiccm2goi1ieb2tduwn0kx","116":"vqpqd9adk380670hj75oga83mz2dcd8682gbemlk4ds1gxwfmeym3ioz7qnyrmp1spf3w6ieshl6vy90aehuw5t1qu2iz2chz6md","117":"yu6we6kux1bh6rgl025lg307uewyhxcl62hvf2di7dgofvgm2h36iloz29tk5vgalpxgivgutqcfpvi868hl8o7azwydx1n8s1ui","118":"gg39a0q2apf08ay0kc7ol72r65d6d0fz8yd4xojqvybdm5jn4cfklwgpq0mzf8yve7kupwa7ibfvyx2u29gaf7u30345mwgx9t4n","119":"vc1wo5p00i0yonwqyy22o025zuwa20xa0z4sdnf7rw3ados9agi28a4nwanyymrofdq150hdxdqvih1ejdyp3hnu87nbp7rp4vs3","120":"wfc40963gjei71n0out369i47nb8txlde4ltkgpp80dj1rdczyjqbxzylg3up27wukyu5kz75hmd9vp77l9wgb7tu9qac9otfr9y","121":"vvthm368h5pj1lry2g6fplllw138p4ttfkhw6jspwvganr29yya2cunhzdgcm5y3w5br78nji97flhdjze17rrcdpi4srkbrewve","122":"6t066ihtfxpby0myjh5lydg29xwei2flyu104frvdng2h0bj7dtztwjuy9iefnmksd2ah1hqh7w9fnohibrtytluw4qgbuhoy737","123":"20vxvfteyc9sj621383iu0mdnxk6wuztxacy9fpwu3fxs8h6p279zoerz5avg4mw00znjpb8zopg242yotiidkbf68ugonqn8v1v","124":"vypofg2ckee2wg1781esdljnby494qd3ppwm31g8vewl2nivswnjdo2p06ztwefbii0cn3avrzfkmlyj2h5o35jzl5hfvl81crnl","125":"254ljzbhdykiqi9lpk75q33my0mbbkfd1l3ampk9ftdmhlj9tbgdn89sohxwavk6ivl5wq2oxevgxq141854zihvqxn5vrxs6lus","126":"zbvlq8z3t55glru9z9msmqo2r3velahmzvkuwlry08js98xuf4ed1bbbt18l1s10mugbxhzyxxcv1qdrk7n52witpco392skvmnj","127":"ira6h7258jnob5mpulirv9tc4qxetqlspx86uq3k2f77g6zid0e1wnmzjn1q5ddavil1hgcloqvmuk71eudlb4e7syd939uxkt77","128":"lyvzv39pxfadodp6jqohpt43oppxrjxiusiwkms4lsbc03y5p1jkg9qk4em9j1g1vp17w4gwaqt13xfqlsbstjl1zdtfi7h92o1b","129":"ob6lxyotqm5l592nu6syzsdbyi0ulcxaxpa7mf4p1fwxoc9oo4duur26xcfvckljj3jjwfzmrn7z9ts2r09o7x7av5ty1adovayk","130":"7qqf40tfz035w3rne9yh71pxais7vxq3k30j75v67ibcxdy4eri3y8ui63knejx5jh7631s9y79yqvp6atmmchayolm5zxaoxl8t","131":"qdwx94xdcs8tjae3gl2ckknwgb6fcquj353tf37zxeosjw2wzhzlllgrbfso7udfvxw409a78f1l9phu3segx6kuyybzuj5lmwzh","132":"eipoy5qe89eumkzj2us7f91it29rpbdzbog676cgk6681wxy64qnzkwl7abwh9flom04ocrqvmte7il6sdxz9t46lanffbp0wz98","133":"auh4qxynzq00kvt82hfyrdf2c3i9tqu1r0e6wqob17039zah0hc8q21u9g9klo4wwi2vxeen4jx1m50vo08j56pndmuphc49mhcr","134":"p2ioa4o4m7y0dv27q87co1a8ziknj8zwta7u7r3kztb3r6ajtcu79l8f81fg43orh4q5ygm4jee93h9bdmjg03b7686ghov87rj3","135":"ryunmeyyfsnczf5lu5hs5hvle33o59oknko7yunfcyif55nsp6mn591ilgro2ka27hmgu33n6fnextreyomlz3cwz3egl30gbhg2","136":"gnlj5uhkg8ihh4e8n1y2xj5ia00w4cyvtcsimc9c7j57it0pql5o2f10yiqxtbt8uvb32r3a22n9uib4m79tczfmlgvlzj1fvorz","137":"i0jgb8b4e183if1el77bt6830qr1nu6hgpfis2c0oc55ktn1i76gsf8bu9ef5n7xvrqwgbr0klp4tq2x8v493f6rddj2mpl99ffg","138":"llaop10i5w2l32rb6x1bg5s6nyk0604idvhbvd9kxpyoubwvx897vsw127wdqnbesnrz96xx2q9xo8zips73om4lg9t4d2o8r4nj","139":"ipsivxvo4za2bdlamkoi1wqyksc9xfhicm4xkfyqv5b0ts1lrr678pbof1sfgodneuno4psl2rllxucoi5def8cgpvoo29hvrpfe","140":"bzqlwpqexie0lvmnpurps5byzt8nl7t1ktvx9piivnfsb4pggaxtzz6qiu4znbwm8d2ruw26b2xvuswjvveo8gg2y7ca29xlhnd5","141":"dysj4859se2zciqa2tru6coy3shpadzhlzdwzdmb9neuflj606z47n0vg6fsie0iwy6ecdz2kon1p05pg5ymf1roc9wrvktyxyhp","142":"14dgb7adghlrfh8j5mqe65d61ezpdnnah1in87eq1itqk9uunbw64xjvtnmcf5k6b7jaewkxl2u40pnjdstcouf2sjvjfrb9n5x8","143":"uyeyp1wrv550kwhzcjgjvse9b10hprmxepe41qdvnd1ijyqz1kz69bbrin84ppwk77r8qgbkwte51bbjj8ohvrgqksm0ziceah17","144":"rxp6nwflgt46peyn17ly1n6istpdi5l8gmxnfmet408ssyumkp7vfmgeymni6560qtq7c5qdtdvsems2llhx5uaijjhp59ip73s8","145":"lscf1gesmh0n7hu33e0qqjwse50ff4t305ia9n74yjjrexmcyndhd61a6acv8xzq2ula47bnjns81gn8uvf2p1upqkg0z03n8qj5","146":"tq3s0jkfqz3102szln1k53oknok6s1o8w4mv3t8lify1kjmiviu4wl7szk1i0t92a07fz4zix8x7am0jiahzcjndiuton0mgk6jr","147":"zdf00kis6um4ul985uf7uiw1qd0whtztt0wdzzwxla6fv0vhej4xg5i9c6sdva99tag40syufjkjbnk6cd9i643iko0q8m98pb3j","148":"3eqw8mdsc57tf0r9jjhb9oec91ek2wc8shqe37650wksrtpcf0zpzpvsjrrus7zknnv1jdo1poj7vwuii6zv0t28qug693ls16ud","149":"xxgn57ac4bbpfzjzhie9ai5d5sxvel3f1k2b4b9p58h2xlxtxm7nuduutlhq2bjchb86wdjwjlkkrmwufi3spmiem6u8duhk06nj","150":"5s05qp7b8g3n32ngw4qje26zn5j0g46fgkthzhv6wbgv98lc79y072emympkrnr8960swcjmdo9htq7ocqq9gmbdci0ss7p0bfk2","151":"hh0stgfz8j7mhefdlksgcbxboy64tbhfn3aa3a9f42h93gynpaobe4zwsfzj77kkg7ipon7t41qrq65rg94mv6j2wtvywi37ts2a","152":"0o6q83bcvj96kzb9g6p52w14ecp36wv99slpwbvw9s0k5x1gi4o4ws17n7gjhcz1p5m1juxcrkvthmj2tnj240xsrxiwk86ir4m2","153":"wx8y710o74ei8q05i2xfx6sv6x1jq6svtife2hlu8atv6g039xv2szmlog6sew9o75pzh6mnxlibhkzwxbqlnvb3vzkikna3gtra","154":"rsu4o5fg5aqb4so3lxhqkz7g2z9jelcxa1v0wixg9afzczfdlsdr4cgeyws4n9jgcb63sw7ki4y0jmb1r199pbmyke0ie7f1mvg0","155":"jk1d0uvbwrxnq233omqsj3yegrpn5slrj0ayecjew23xl615yrghpnw8p1z19rh1cg4jskiohpt85civid9qwsofn1dvpt2z6yfw","156":"mmo7wp1sqxp2vg6qjqk8sump5ho1v8yi484r9jvhdp5qrhviiga99cedt6ghmoc43uot3f9lcudiou1nbk3p54rp9hv4693v64x0","157":"11p5n80zir9tveeqqoxpcs6lk97lwq8pu07tsuk6ztxh6dzbhbidwddlvm6ohik96m3hulmeny1oaz2xcoltysw4ruw7evc3jz0u","158":"norsygqksro5t9soo7o4r6z9aa5q473ohxx6t2twiok01s0ujigk6m0mmctuctp2kzm1il235nmuzyts39j8whqo1ejnknr8nklw","159":"ei4a1t02cxtiwp47372eukapcuaey8axg3fio41rd8dpv0xlpqlnk30ettrpnkbvwsx7q8oixp02lwuiv6kiqk0k31r8ztof7ql3","160":"x5um8vgsffdpv4sk7vinwiqrd8dau2wmo5ln3dfwioet3seq6bmkshwql44x7lz83ov6cjh9bp5e05utglxtg4ww7att12x4ipd0","161":"8iin5zqyxi8njhudrfo5np24jhuywla2ahozjtp4dfesbxjqlsm7689kicibhtq0re0m6w6u1ouetyiaylstnycw6ps1u04omg98","162":"mujm15w253de7yzbh6f1eejyz4w41vjqcu49zum5e9e0y772itvygc0ht3pgll75a0w5qqd2vicvl3uziouy3uh55h2jf6nlv0hh","163":"qq5l9zlz95jzghjp7a9qjf0yvj4e7z0xefl49bgqnlpol1i868t2s0zo6ww2ztptrn9g00puhc06n8pwk8e6k0s4m30bychnmaph","164":"s6xzfglc56drmw1yhcswffcy9ed3k8c20seuvhonjteha2uuw0x21fzxvnn6c2wbwp8hljle1exenj1f0x2h3xodpq90w0kuhsno","165":"c28rfhxgc5jogi72abnfe68jksoj3cfyrubx8y1geldeakv5vs4sgr1ajk1hprwqwxovkdfstxwymntg63uwmo6bbqfuxzwosr7u","166":"bbvr6ogo1n4qbov9c258giqkykki1cbvbgcadq4jbneqlklft39zi9pxo8pbtwpxpgso5vctmj7s5boh2gy5jh276fyi4cme4pyb","167":"tewrnprftjspuwiv0f04u48ltnvrhuwxd282lz4257dojeu9htgnbjq1v5yap9lvo5sno4gxxuehexvukpmks0rtt4hiu2vbbi0h","168":"xg3gdx1tyww9iki7l3i2pd83zk1y4kxxx4qpxr1yaw5c2uen3nep1fzjbgvp42n4d5gepvtdhtkasf2qu92lyidq759pjawzblpk","169":"yxi6s7ho8coddoh31dp364nnmw0oe0kvd4ggvi1jiseptpw0vyt27jrai4z0a26gb61qj69psy3g3290678uou8e6kdzdxcwh713","170":"u5qy4hia9ww2e02m6tpviw0y5ax31u1to42g5qeti5t7jaaopo9x3izvloqjyo2d87im7ios6p098uab6ok3j3eikmfs6lggfhp8","171":"n4dpqer9o7uoqggp2lf9w2ghgcs53s1p83glyn3pd1sc0lj02aj8ljisq6ax5es0pd61uasj0dltf6qgygf9338gp59ipi4i2kby","172":"l37u7vgl3z0im64afsyimnzxojir31wrxp8nxqwfywiw7m3ncikolekmvmly5h718ik89m4oon2b5h1xva8rl5j8hcff8lslvqs6","173":"7hyqq9kopaiy2dg0aozztfx8v0a93w66e95cnzo77tsb4acl8yz8p7dyxgwoy3ijdqp4xtjs34v50amid2tw7cfxbsog7z2r96w0","174":"9mbjoifdaws7z5le50b4j45ompjal8nb631pgykp6gyp1ilu6hk456laa1pe38x1f31hlpmc9nw21s65u9q00yqee74oi30l3qvr","175":"3tw7r5pblv4uzw8wlcxqn4jydyhyw7r5wiurtyjyx9qu7ssf8dozj20mntyhplak35fuzi5nn00le1fzi4mgjg797at2fzwsa7e7","176":"1bknz0fu1um6ef033sc8tvy11f6w7vj1915rk114uhrdslvkrohnxs0cpjzmc7s7cxmxplx9rjnnxkquv7l1p51jayoofn8p8qo5","177":"wblmdn0ifyz2ys7fc3k2cqmsk2exowqb0mthd815fniiztx3gt346hopvoxp3g6x32chv09z1mo9ltdim6v8zwh25fdwy1c2fom3","178":"p0t3b5tzyjz3cdcpv3z1n2r5g3isr3qhmqa6hvj0xc4z4r9aag1kp8908vn5cvmig8ywc3qrxxs69pmkym8xqbm48p3yu7dwzcz7","179":"a5qqa34wuhxc3ruu7n8b4l5qm2wngk3anwucs5gg4bqkv0p8d49up9f6tn0aesao8sbk8l3ga6na4b3vqgrgi1gtqiwz3es6mu69","180":"ogqlfc9tfmilnxlmw393ql8xosgogk8x1w21oooc8s2whfot0phrjcfjn144xflkxmg1rfu01ynbzfbqk3urmeqrlca3swvw2y8n","181":"p6ftcmd57t2oi7gtbibqz3n1kbjvgmu5ns0s4yjcye2kqvvzrwdrrpl7vruvz3rnjeelg3adqg6ug03ak63pq5akspkdo35yeo0t","182":"9y1hz17xd0keubs2k0f0wfoz4cp8kl41obfw3mpg9qq9lqg7dpps3lj9gntwxpf3m0nldfnpoiacsi2y6uptdjw5hxd7mpe5tp4d","183":"6hkatfwu1iwxp8uj1udxaf1nxq9jsqsa0ruoyepd2nvaty4ktu8f0di54idlbnc0zmt33c2gdcukli99ses1a4e8ulj260toyazk","184":"12ey5y7e0xa1sv6mi2oug64xnf6ahhh2y9vs0svb6wr0pzls6gf9z4i0c8lz1ba6lif895zqzt89tugrzxnibxtzvdqq30ny0evm","185":"oozfionyl4jwntkwvvcqqvhe0e8b0alk8c9dnzgemzyj306yjgrh35r0jp15su3lw08s49i2adr0qr1ci7kpokxn7t2xit23y9bj","186":"6b40y9eygp22qjxtpr42tqmdt0576jvuys72yndp34praz4avb9ahhkodgce6mffls9ixsawdx4u1375yorbtlgieia71ffggjeq","187":"73y08uczc9n1thgd7trfo322tvcbb6e4o6kqwib1d1d50xkcegxedsgtrzp22pjulb8144rrreie521fw7ngcn677lb10z4o90mp","188":"gzh0von2y1ktdw6g51rsky9k4rls6wckmnj1152bljzrxmqeqvxxj2gm1dep1zhmd2ca0vea12wyyv1p7005px5bvtwn5nx5thf4","189":"bm0ctfspl5yqm2g92iuwzdwpj0i49r8nlbjyzly59ni7ifg1xwzuwzbgadvhx1p4sue09hikdaggw0b2984kqely37q3tmgurv7m","190":"taox16teqf3fta3xl4s46wdjypd9g3l6yn3rm5ftxlr6tyuh7ncjyvkovxa8vwau5w1rd2z3d88kcst85q1m408xkkr9mw7jnp5x","191":"r4n48ua4bayusx9r17k9st1puvaxer4mcbnmlnw4gjutrh4tn7cgvy6vak6vwnqeo1sux9z3yma8lci5vnnfmjoyhdl9bipof63j","192":"p4p4nq4m9ustmfaf7x7j95uqdv0vgbxwvk0oclqa1p331envxhe3gr0habj3gd20f1ozbsnn4gakkjh8ovgqiq401w9xxtlv8qa7","193":"aszgfxbuk59w273zpe5kjgvc32yc416r1y89i4ak46j4mwnrnz2pyd7mbqcw2pb4h01gxdgpo30j9aykv8rxpneltkk7yjr2r9di","194":"o2pdhzcsv1o29ktad3mbfwna3dig5nk4j0r0vpvyh373875y7rchd4mjpk4mbj6rrraot7h1rsf02jp1qxbylc67cj93upa8dczi","195":"1l1qvchrse3nm8u5a0ds7pkht00y44gct3x1evjh3f1y55pxcs7dywb86iepi4yabhbkvfz5mahrgrg5idk1f3xbidcnc98khw6g","196":"copf2pxwpkv2voaiovk4b4p1ur5kt1pok6kl88declzvm1f7m50gyp560pg3wijp6ufwt5ywkwq4qlq6z09v0qbusdeqytyhivqz","197":"6j6kpqvccc1699j3pdc8empzdbkssfkcsusinpnb7gqg7mhgviul9plmufwi1rotwvezlbd0jgtl8pn1fd0xz7kb9hxiu0wwyx98","198":"vvcfq29cal0l0hvlvspm00cge4vyajaakq8iyt4xspdugh7whcv5tmx71hm1ixkz6isykvugkdb2491sz2j6eu2p646uvfjw4wbq","199":"0avefsocl4t22ufq59kvpj164drpeimmirqwx69hipwsigz0yz00mtjrl6rxsb40b842ewfcqofjmx7gwbond4tqr18yu6ouui90","200":"n0totrxt01rmwfux09juaeandw4awgjuyhzwmm32z31soae21ghngdq7wgas8k4ltm6nxykbm5s833ika73x9xs68n69olso8xeo","201":"zwahegre5ceimwekcetvlpuyy12vjtnvdj8grrm432jk5361gxp7cdiaspxmqo0kkpm74uj48m3crbrsfw3jqu3uhk6ahpzlcib9","202":"t1omoddaixp78pyk8rxdrvdl0umrc50y6nob06c5jyvtjv3zjd6qyuf522fqzfm0p7tygolkbn1zfcgnwjbi5lkp4xsmwl4ha768","203":"nr2watu2zm6snip8azqpqndlcr1985e2sun6rn5t8l4w6lkh2ojkguy3opuowv4sb4lx746jdogjxw1d8v4j5s82mc8n8ed50ymu","204":"1npkepf55p91uetzwmastwx4u1mxb6wilbjjgy7brjfuqb14xdypsit3kfpa9rqq5p4ydyp3osg3bzuegrr2caakww8lgls3s97s","205":"gdnwraa3cc4nu0mledhaa5tldua1fjyes7xb42u0uom7m11funo10pc54szeuy8i9a7rhi7ju5d8xx3yuzwvpq0uo603owmfb6iy","206":"iwcvx6ika5cemtmxuqsakgr0grq40bsr1lrp3b2hzgqlceemyv3zuwcgre5t5b7pis7wurgzxfbjpbcg1eoipf97se378jgm19cy","207":"8uaj61vb6yov0g0r9heq1r5db3wije2fhzqwtjwntw5lrdl7k8y0diwgixe4b01r1ximpjdkwagjzo9stq01x2twofo22qpdllv6","208":"ia27xs3vklx7mqzmc55pq5nake8qwztzfpnq6nqmb6lkbovmlu25oybz94fflpt5ggebsxfhu0qdr5pfpyl9dn4kp1598ymyvc4x","209":"3zstth9y18cfg1golz59q04mujpfbiwmkgqfgp2l9yzwgie3r6ctl2xupee55isl7w8hzhsb1op90ocu4gb73vuhniurczma57kh","210":"tnsk31k8wdtkktb70vt90sdsih8ma2yifta99zndc7ocbz4zk8u37braj17q2gec0kak590fowz2dhqctr374unvl6q81nwob818","211":"6jnlv2jwpr6ntsoy2f6xrslwf0bngcy9sjkfagkut66191y861vv2ixvl3mjx6ec9wcdz0hoqrg2dlizdu1yygjhhtwa8fpxxunb","212":"taresodaf6olvfcd50q6bychop2r98ts8nl8zygdxxa3okuyt0q1arzj66n59relkamnfet4oechuck6j8wj4y8ec50xv0i2ldei","213":"3n7knb90u1k97qa0jdj6mixxatdoqkpxakag4q5qv7zt6ympcorch1zpdatgqgjflsff0u3p805ikp5p3qvh9h9dokpq8wv9ekm0","214":"bkqr4cznpu6vfuleckdmud1bgveu938s89rrvuyvyrciz657lpbsiodeuxcevzhnyttxbqwnitzi0wlvb51dq2apqdoz4y1hm0vv","215":"mpo9lf5alb0mj5jq2tx14uh7kudqryl10deszheu40omerhhoauts0hf84ak2q90q2rxhnoi2o73ytojf8tbivpnv7akuqv4ci7n","216":"io6udvetempw6y0ufn6wbu62qf46wqmzdzeu7zgnfxtmw2eol0yn2jgi3a3i7bx4hqdymaw41l1w3oqd2jk3lyfr7twb6pi2uwah","217":"lmsd7f1zw1usugeo3cylv5yu4cnzm71cchrxnamnt9cnixw5x6l6cwiue6oqzgf0vuntcyaz4d6y14ivt0w96cish7dobgefk7i2","218":"09i4zvl56r63vuoswh5pwttziumkvggol5q3nd57g2zkuv7nfvxkgy0fgjoe0yjm95enw7vxd8z1h5mr7qwgw5lyittopdegdnwc","219":"22kapnn8cztge0mcwywgw918wuns53gwdznhs40cdseq46f8vj5pzbtnv8nzji6ng0x98yowwdzrrn0gxqi6h5pfdl9hwl3j4koq","220":"jg8rdmqbtzrtviejcu8xpfcfzp4ozxe468sw2o626x6vrbrxeawmcfouw5epu73uz94o7wh5q3h7marty8pis89t2qkpmab6fhp3","221":"n5fbgqd0qx7r2yrkzrm5liwf4v7zcksko2hawjfooqd8zh94x0xdgm3m76akqyc5yuahpkpp9gdcfg64dmkiw1sb0ryz1e2xdhbl","222":"kd5rnytjdtuwhrd0ykn89747cis850ko6gs0yuhrfvbms53zzmpvdjzwpi2cq485hzf8xwwktxj8llw1nf093eo62dibllvmpqc8","223":"81eb44k3s9ys1jsrf0g6cm5doptjk8wle7n0akdyqqvy0wlukjrcjb7hn53x660blvouyyhcq977vdjrumfltnf7pnsmlh1hf46a","224":"6fv21cijhs7nzbr1lqq5e7n4h27lmbm642lkkkegwuw68fkm2dhvv6i0oh3vysnie9f7yyqd4yu0pzw011mfqslxmcf2gw5f9qwl","225":"yiijqry8hwytdmus68ovqdqpae5s0xoti1flga81y5bh62togivwuls3u54dxyxh58o8hnjsdug016kvx2jrrzqirjz43mjryp31","226":"u23rt8wc8ngl4qizth3dutel0fok7yzwcxd8ilee6t63xi2wwuuoy1hykffajjsiziispv8lbdd68dfject0dq51junm57u6k0kc","227":"193ywst9pjjiq3hprgh1cu5kg7rrd27m4jrkck690x9q2al5v42qrispfkelpk4dqb06dkplmt2o5u2djlwjw4llme8mc8f7ncx2","228":"hiov41beoaxc173t70bunakplv03osl6p06fbdhju15howp9mce1vo7ormeqsga2ris2r7h8weqgvoywwu1pjo0sk3w2r0ztswzt","229":"6035xk6cebnjdzhai3wyprk7bm38sfnt8jjplzhd5gzt389mybeofq01cjrt15wrewo3gx0rnhm1jujc2xfb5r6r9p2iot4ywiu8","230":"gqc05ebl1devdzh8g3rkdpkta2ox1bkv1h52rco89zr63aajxle3nfrxjocdf5ht8zk7p1qdfv95svh6zof8yh68tbs2pe4why7a","231":"ujtk5d9zbimmde9njin5jenstdvaaxiuxa65if599vluf9q6h0qhig7pwc4pmawwyg2rr9ybnbp1l56nskk7wwvd1vq3j3bcp5uy","232":"4bgbf8h5b1mfqx6mz8vjs1ni8am2tzjibn0j6eaisdkmemy741s0gcmffi2qcydpmthgtwhf4tlvi24abg6rtxjn2i249in0slq4","233":"i4ur9rgl4b7kcfeny68dekqslavyj9lz0ij5ll653keel8a5u0g8hmbrdsob385oksquapt2b04ly2hem5sbzyrcfelhjujriqgu","234":"d8p494ktrrnpvbmnoef44qh31wihfr10s8ohg6ubueqftt82eq3s1591h76xd1odo0o2u1cky8c538fznw3u9dpfrdc0ae9om40m","235":"2gpyxo41echmtzl6h78smeqjgo60rholodqs6yrpaq5n30o1nfpmfasd7bffrbhqkg0jskajd6g8vou3nizy04lccme2d6kjslea","236":"wbqoimou2qw5h4kvrq3e4gdbpbwemtry1va3fcmnd6prem8bjh8ki7x2vllx2tja8ykx92k1ivdijsu1yk562eif9a0bnfhwkqvh","237":"mhed39q7p3tqhvqmvgd8q83x2rrfngg2yt2xhyrt0v1uwqjnm9b64zh622tm3vr06sgoiwwpks6bk9qot5lp4ywhpaxkop3xn57k","238":"w3rrm6gersrxmucc3irmkm0k7jh9oi57xdyyvz3qir5xwktq2grbuvyzgti06q4w9tolclan8wppxrjp7qc6hvzivjbymzv2o30g","239":"mrq0igvclfgn6qvkmlhmt27pj1z9whjq3kpcqoz3rtx64rf89hx5nvyq8kvg5fghzlvysvpjg9t2q8umpht54tbrdq717roayol3","240":"giyo3qlbprv4wawjcekwfqhg9nt797on2c7k8zr87rn58mmru4gweovkg9a97b1dn2muiqjysd8ixaovo5g1p04vqealv40we7ln","241":"zokxszxcq2ur0p1ahunfr4zrp6jazk2grtx6g8r3rhrkfrb9p14kq1ebftldyer0gfc4a5nngli98o68mvzvzmxxwsvcx11lt8qx","242":"6op97cd95c53x9fubn3si501cdg85u5v648p2szlnn741ztykgtbx19gp554k19z2s4pbl2w05a6izsfegtu9vkyi8lp8o05l4s2","243":"xc6ste9w0r2ruyyi6qffmaxk1rc5vx2ia4hqqh6m1yjc4psldtjenfi1g3cnq3xf899gqp6s7i77p6hpv9lvcut97ewl2437l04l","244":"dcsxx79kj5h6rndk5xoz4bh99xjua6sohhlu3v2dzht1qb4zbc5hcj13y0sy6qlpzax69u6ikigk8xur12stdturlniv1srqqrre","245":"n465xaamzkblvoslmnyeptie144s09lqtd92g6j5ze31ujigu9x67zbfhl98masihmezh2dw1oxs0udu4eqnjeuuchvv2u3jm5m6","246":"hscwsotlvb1tw0mcckee1srv0xjfhxmxt8fngsh5muv818bq7zmpy6v0n4wbyqjnj3hprrmxydnrezu5fyvmnc0rhvk2wkr82gss","247":"ddgkdwc5ctn4rge4dv8lwg0hf13o6zxuws4ket4kfq2c2bml8z038kg43kns25gznch9cs4w7m9wms3ge4cstlzu22qnywqi8vfr","248":"ddlqfrcq17z803smwed7seah64qgftvlamw0svpxa47le16xezcus1i2mdixywmsj00y6u8gvagp18pmu6yiv6526v36ea96e6ji","249":"07w0tr2387j6mwf6uzhd93k7gmb8at1eqpydgv6wqnyimlzmb4a7okh2kgyjsx1cv9632w26va8gmeyc6brctxbutvl7xjb8pxnu","250":"ot5nnmawmyv5hrz7nvcaue2ceigumgs3b16pzg5n3tjbgdcgpckqfaxmphg3jmuo6e9znvlg725h5wdkzrur1oipfv14nzlznjyn","251":"fugi6rsba8j43an1nqm5kpewa9yraxfvc7he8a8hv4f3c8ervlx0tyufvkj6upqrbndidmllzpyt8j7okpmpiblvyrve8zyh9jn7","252":"oro5cwpnl39jbbdi637vg82v1o3relixb1fb3riu9xp6ts9nlnq0wab7fee2gpy57po5iheu2z70muu7mhajrdqgt27fihc5eq2m","253":"nt915dq1mrghzcsfbxkdy8yf36grwioatp8ymdyzcql6d0ycaf470ljv6g8hs20sngjrnhbjwd58eye8bfsapo7xf61v5xt3ncjd","254":"x4zpunhsly1oxl7ukzfihopp4y52f8xi74g4lc9z6k038moz4bqdjptemtuso49nbmhiawr00gg945us4lcl0jo359xlqvsguz0i","255":"gqzbdy2v3v0ik8ox9xdzuizkp7c1h1mtww76jrg76pzz0soj5og8zp9dj8928lsca6h0lbgtgfqqvdt97v4w3hwdzzf1f0w0xgy6","256":"ad0b02xcu60txwnycdhaprv7urjpkugsj274vp7doxf8n1x5bplxeuq1deotrptl2tqcabvzi7ghphu0ok0jpg3coxchu4iofr6h","257":"onoqwscn0gp43q036bikgjppf0ze8ehk95p3fwllltwqcqp2shfvf4d12ksykxmsif2n6w3m8n34hlkpuzhw6eoq17zui4su9dj0","258":"sgyi79w1hsirfcir1480a0lkbek3akhx6tm2qbiweym7bmz7m47rzyymvl460nnaytmhjnj2cal0jz7y8bdkxmq2htob776crp20","259":"tu6wi2ad81jdjpjjwmvmw57imidpajnt9kayk0bh540og822mtucvq6t93ienc3kl6p5fu03iq1x6943yxgcehfxdbkymm6r0o04","260":"sl0jxxelpwib759254lsi79g7oae1s4nr0if7uu1yow98ukbw1feidcxfol5pomym1xitjlq6xonnsjbp4lw5m8e6oj6957brb1z","261":"w5s7x93m38pukdcctr00ghztim4341wc6pr5a5qqy675b2ft3qc7o6dlo4kfnu3iqji7tkzs2mtz0swb2fbi9wbmd1ytrkhyxt02","262":"hyved1bwr1iwmkavp3yfx8c1e5oq33j8glxawkzach9nvoz6gyht1g231kh5el8yj9q9ls58433c1v2cxl18gb9k35mri0ehjmc7","263":"74804nxt7ryr1zfnjahoid70knwbbrjtbd5yyiuigo6jhbsg324s1t3xj1vhw2hdi4775hbh43uvx6ld183dzm3dqi9ty8ixozvq","264":"v0ddn48n4algmzvw6rov3nijtnbnvhk7hvlndac838w9xh4mb3v7zavl34273dkksapm4c97vzj7r8rs1akywvgcn9skj4wd5kad","265":"ommt727rc86pniqjyvrcd5384ptimc7wpjw5b7beoxfs964hjtzi5sn0m3nclx9iwj5lamjf3wo3l44ka1vxp7tfo6dygzp06g81","266":"zt1b5vqubh87910bqln6r62mupvlqpvzhy0ox0wt44ru901zkdb0rk1comc2cicws7fc3ip5t4ayvx4xo8ao2a2350jjj4p296ac","267":"la0qxee68yjtlg2z2cq19rbo5s0ih5izknsv96yc6v93o6x0o0h1tvqmlwdkndvt8jct9rc4w078ca0cyncgdtmm85eghj8tp0a8","268":"olygxlhap90cjmq3v3xq2ztvo37z3omsqkdicqi0j9x1bfmob4vjpczh060dm6w21adrzrdu26ido49lpp3htdjpgneyx7mf5v1i","269":"lx9y6munabb3bxzubm0u4za9f5cdwwzmxatl0zdx2go13m91fmd2uwbqb79x8tarz9rez6a11q8f5m1d9irlbr9b5p86jl4gguf3","270":"4hgpu6ql3hnd56qrojuwz19apvo315w0g83d4x71v0bg1fmnga4mx67xhmm0adagdyb09gqgza16fpu97ihc580g52rlie98rt2b","271":"uxhtovmqb73il2xug10plxadcdboxx5iz261dkp2jz8mh0umbr5qj090ggnczoj6n364wg91pdr9wwf9uid1fohx40okmopm8h8v","272":"a9g64lifplg5ew424qgc237y39xumwrmnxg8fn3kip3ifgas1maicryywyqm1l9m1hfx5a8lqp1hug7h5a1u7ic0313zyzdb1m7s","273":"3al95nyd2wnntgwmnnq4507vhpsz5j4jelbqg8ycohx5knwg8mp0rnnvacqwr0tq6pa3t4onq4l8vb6puwcuymc7djhqf3ia69ps","274":"9ji7l0lryah9ej3q2qaodk9dyj5fqj5ojjd1clhcssxg3kr8t76ounwckhgv22em1l82m18p6dmxg4zwh4rjiaz4g9n09lzywsl3","275":"qn7o4athcmquuyxqys7g3g9fxkw9nrm0c0fsal5cd8vtcrv51nr27uiv8zya9dseub33yuyd5m7x2vljtgvb8nxjb008a0t0jxeq","276":"fajei46378uoxmonsusr1ku54makmftzvk7ay8dmwf954wvm36kqz6c4m76q1qs7ivyqkhig2i8lt8aqiuvsrcr884rsfoaumdsm","277":"agysqhzj55s3q2368hqbu1685oaws0d62xxz1aymq96sa1dqm1e84o16jd5nah1yaq8urb969n8ngc2hlt433ai4sgvl0ctylg85","278":"nc1nfr1ugld1hmej55lobh6fqj6oxkhvywgi795g3vj5yaqfk19khbg2b1sdt6dk51qfkc4phrffdsegleyr5k6b9ryk2kht10t6","279":"uf3sxah3fgjnlhyt6mt5zsmzqeplxrzf6fuvl5o52h6a0u8i33cxr8fnl8yfsrwmu9pvqugqddiy6l7yws95atluaxlvnhjxt65a","280":"yk4hhulczm9wnaforud1pck3xgwjfoz5za7ul165kavnxx11sz1bebjxi04h0uxtm1xz69xdssm8t5l7g6vf7s7gkqr7pwgapceo","281":"spnajuhu4kzzefdaql25hy8xuvi81jvl0fslmnqy73o6pbdvzxdzegfmdx4og5clnizw4wjcmim82dpuv9j6kbpyix8470y5cics","282":"9xalr9bbyx4l7dj25uns6ejmxwk4eekl5bbm2a1zr3uhl4bl1bdb3p5y84gng6pori7agtacrsksbydav9ettjvolvhxi2tj3fun","283":"pp098lligamo3l1o00xyrzsdb7f9aorw82ucoy4gmwfzbj2lf8hwqt48cppnidf90r9d44esyky4qxd4xzw52mqkuyk3ktn4hemk","284":"fy8z894cxdaaegsxqkrlwowsbecagv0dc9p5qc76lx793j8u7j1kq1mxosd13scn69j9jxbvb423tsmcznuv7x0gblh73jlt30kj","285":"fsggcf2z0a9g84ntgttte9nq4m1c6urfk1wv4hvh7tq84wthcylp0ooboovorl52rnru02chgrvzfgndn4cb4k1l8cvv9meqltlr","286":"266yxddkwyp4kbb1stqeauo1tot8u1tbxz1j8pe37aliw3895hwrwt2ij5ozq0t4r6mdahtbsa20vpo1kvggapk0303rzbvufmnj","287":"botzitjf3i4hv0x2vkgmo0ogeulrjswl8l6ytypcyotmjhq1pe2ct1ol993w4t3mv4khy6zu7h6y7xflgd3dwki49zuqi1wx0hbp","288":"wku24ugphzwye1czch3pjwtm1ilvuiqlpkrub9hxactxsyofd4me60pvro585hz41s17hp5jsn4muvlky5pp37tjmro10htsnlop","289":"fnpv4mz05a8ollbkx15orh68eeezxrtem2doio6gu61tk1knuzjf4yrc6s7qn49yzxwjttr2ldkwvqfysg19d4y7pp40u85xgu36","290":"douy6rp6ahad9tdukcb9nt1wqfdmy4nxbmo1h1erss3bypgew4fqo87oxoibjckld6ueg0t6ppvbjjgmdy10gicb9n5emlbgmrag","291":"el46w0yojmnjtnxbwkh5rorzo7mkbuw8lg6emzyuki5jtibbjw7ynq13lzjo3d6nz42ik82aiicu2zicef9es31zvd6fqo98vvtr","292":"ko8j1arfpenkhuji2khk0as1u61zyid6kgxx3hyy6irzgizocua81dsomwz5er838tpeamuhajgyrlz0elbz43cahmys071on4ak","293":"a6biniojx4iehr75ujz055z5wrz8wj71sto35r514xsvby6p7hp2xls4r1i4eiq9grei3qrzrix7hm8pxf4tn8zvkomjisihe52g","294":"ha3q43hodotlx24tg7qwk6zp2wef25qrpnl3qc0g6gbnfh537gdmlvt4z2pubbvsv66lycrg1b6vruhd3gu2tplgwfc2av5325lm","295":"y4if54atbult4op25a32dzjufb4gmxuv8yl8jl58iy76nc99o67lzg1i8qi5sburqyiw2qr3124hevf93f8mmrbcmkd0m4w19ku0","296":"awk0gw46j5b34vjuag1w6nrrxuw11el90jabn9hy0wmrhshhz6h52stsxzkxl2rux115ny6j7rrx3hilioq5v7w1d9inaep894js","297":"y3mfc92usxiww6dz94cyxwagtgapxgrvxgd8wgkeu8wql8aujc5q93yfv0wfm2s0zgll8ui1h989djy9ho33qbw3targrhciv25u","298":"ghk62vmgvuktv9izo5d0kd6uqnuqp62jslqdf6uu5wtg2ubd0p8goz40qnpyrxnp3y7mn5nw781797x247yizqnyoth441nm8kkh","299":"5qka0xnwhmneyeiexmy17upf2qtng3x826zggpyb8dszgizqm5jxdxuj9myfikjkrii3ate13f3l7i23ih5jhvhaneordytdih6t","300":"mmkbxr8a14dggftmck9acko7gwvztsircdghwx5fg3c4smpv61pqr1xy5i73jcj54veumjq0t9dwqot4i3kwbdvsr4ccutnus3i5","301":"nkephg57di0viv5jfqr46f3j8s1zhwvfw1j58r9sun8c07780xaq7o0d3narf5poyyu1f29rlahgxmmv3z1lvh9rfg9tsnuk1oc8","302":"8r7zlckmcsuj5ex4zeh5i9ngp4d26gk199niawg7iucvgv0w0wbtx4c5w719i39c6xqt1wx0qm9zhoifoog377416lh23f8bbjfl","303":"j9y39v9dkbxhyqztlxhb2wf91o699xnecnlgfbqt1r0pj4byzdjaa847938ufsciyop46usbxuj9p1n4eh2bvzojpsw2f9st5nhm","304":"nxvt1406oufyd9mkzdz3uurhrdnsqs2oca7s56y0y7gtjx28rhrgnhugm42ywygjo7llxyf7okcbaq2enudvbbajb8f1pplqup86","305":"jxu3xidnabkl9uwemu0ylgx9ofywf5nle09kwee351851uhdjwpyfj3je0h6y95ompxc2rqmkj31k6ijb6w7ovxo35p0s167ichn","306":"ggylyk9o65bag9di0ah9k7ojihhnnzciu9jiobkuq09psxdq5un14xxlqby3kz1o5xhvtkylun7tu64tj0hxxeuqzhzmheco5jim","307":"84cm15fwt79egxyz6xde26xkkoxq43l80gpe6ujii9wbk12wgis5t9lsromlc24mkl89xv4bmm4a760h6llu185g9gsfgrvizdq5","308":"5ow0sj4i42w9hd1mehl0sfcdlw4b5pauuf88tevnfn40ejr04yljmrba2l8ylcvvhdp3xvx9z024alwrtnjssl793eeiomydhly2","309":"a09yqsohounsgtgotguhmb7b4jot2k6y4x36u0fgviiwt5skn3jerywqckzmsnbsv7ppp0avj186sldple5o7cim6ipqyaffhnr4","310":"jd4h6k5skcow05dkwtu5c8cvb8tttkgocrjhntlnw0i36nah08u0lptfrycy45n03aolsk7juu27r5es6c6c8jaj29uqe0rbdh42","311":"7smqsgux61oklg46rc4tkxrfht7r5b4dha08bxs0y2yl08vj90y9nt7iruhdlnh910h9r8a9fsg7kpx917v0op8ofy50m63e9lq7","312":"ipdhzzz2ar4g5eqj1fpkrmdxcr4gh7hr07q8p7furq09jgqpp07odfarq53urjg72113wm6fm4u99g2nlmld6hu4himyb28ftoi2","313":"neqwuvevi30txnlvtga2in1dx0z47xkwabsc6mmp6cjk4iicy83o7648lj6qdjia9nau0ziz14hm1n4xarty7zw7u5mwfo0nezgk","314":"86fc49yx3206f79cfm3tcf6gwzj4qo3fd2offemkmoajxmzkfflli7b8n9hcaragv5p63vqx0fenhqhtlto1ja7xb090munhttn2","315":"w0jhwj11zo7bz8riuu8pldewgpuu572pirllykvoc7eb5171ivrnlk03h9h6a5qk6byy1t5zuth9q44yic0kjrsistvkx3emkfpj","316":"2ct2kf2xn1rgrbmj77erhqk1k4926etcjk2439bxr55bdlyj3r59g6jah4gnvk8e4cb7wwe1kefrscgocfyskyeuphwpw2eyiach","317":"5ko62r2megjctr65ekf9avenfnijl6d7rrb2ktqlw6tshye33u6m5ewuynjrnvkqwqr5iuufzhk9eaefawm9l08feasxxr6wgjgv","318":"j0lfkupk01zhnls0cj0au5zsnbvsx8cjpvh2qhoqytdkq9gc5bo5nt5ooe88n3ykj98rpl5g0si4t9jvujzmu3057ly3xsc9c72g","319":"gaseufrixyspsybukvtao1frsf7ykc5htlzcwp66vs0otsshsqv3c4awco92du6mpn0j7874ng8zxx5hr9ws9lsopszeqsd9z2g9","320":"lgubizy2kyf3g1x0e17u80ytwy3ipajfisd6b4wszu3vlzcsoqpq1ujwoiswmxbo5mnuozfeh5ov17m6hcuv44neodjkzfdwhbsk","321":"ewkowtovxd534to4firdhcq9f2jne5k3a5zft6rm40w77r45ylth1alooa625oags8ledoe01j26ukv5z4nx2xw4o3aotbrw03gn","322":"00q651pf87xfvtrwfiopx0nbj3z2ume22v6dj5xsysjyb9yi6zpkvgohsyn2q4uq4umj2kiu9csfbb0522n03py3gzsmf03o4g78","323":"htss69ku78ovuek1q2bmm6kgax5e3340q0cg0ixjmkru90wi5qqo2wfvd8obmeg0lzp8pvm3ej55bmznj2711jkvqyb3am1518qc","324":"6djvxvip06br4q2kfudl2fh4aq5wfgyi53p9zgz5v7ltkzgi94eytsv7ljxoe69cc5g3pfdt37dv00nwwgdo92wcfukngt8u1kte","325":"0udb9hobwjn0pks13gwz1xrc75nv2v429g8r710bc03ftwfo4nprb9bx0xrrrl4m6afppzj8wyjzfaf7353zh7i1r1y7fw1twaom","326":"x4fu6h4s67jkt0ut27wpl5j91h8v5adupzw5kyd0vnjkkkhzwz8u7udrtnscbhakc9tk7gvdlteq0tzn3o19gb23poagmkgl6kyb","327":"ckypkqgnc5rljrntoh7rlbr8tsoyl3klschd7wbrgvgwfvlhy3tpoefwmjrnkf8ca365kh5jubdsvesvs41pb9ko8sb97d432kxl","328":"bmgd3sudjg7nkvcqbwfp38n7koc90w8s8a16wsd8mq9bk1baig41fz322hwow7yj68usg5ab5d7ds8ghb1jwiogmi0bdtkuhu4iq","329":"4c0qrtl3ninj3wrea6truor16t7jrqzikiv67uza3t877ppkp7q5t4h9k040cv7x5mpr27tx51h2ib0oflgr3q43meepiklfgtzl","330":"1bgyqeghb9o3zgioyd5ypknz7tgk85dc5ir1fshcnyxlhpyd5lt7jomdn7v7s6xno0lj64zazhh51b7h5ubgzs0aj68nqs9jk6ib","331":"e0uokc0jfk77tzvp0cy549mw5f9sd5qwa43e7w042vz9cjn07lluo7awgvjnyi82z50yun5pv9ntv6o4whfwkuuc83zg4ox044ps","332":"5m8v9upuq34n58o73qgh8sfxt5sabptv1jxiowg0nvw8lrzx6kk6haggmyokodx0ivnthrn4mcq7dwkpgrgoxprgmu7se2hcbelc","333":"908resz5htm6m523dxgabprf7dcqwyrbu9t8cd6j4j4nh7cj7035sge3oiw6bjvo9onqgyq5m8ha15ltqr5wswwfmdr6kezlqt9s","334":"niho65zqro82aoszbf5yny6yhfyit4s54wd2pk3p1g4vc8x403za2x5kmqsj0i86gvqwqsnoh36iqoqxyd2hbly7fw4j6chvcj2j","335":"psuo9zsus44q98idts3djlfjfpbfjsnoxtfshq5ltubroyhrc9gvf06ij3jjmfu68ugr7o64smyhg4zvqjd5weaj8kuylyvzcvgp","336":"6yq4va0ywj6j51itaw6eviyd944ug8cmboui2gdzmyzyzgm6p7mtuqb4t74d3tc1r8slc4vtfu94zei9yjvdobv5rh7nbvfazpk9","337":"7biir77e7yjqj2wk9rq0nesew9gbymotgw6x43r2artyd00vonhg0ouqqjl70n1y4yc3sjsmaveruae8o553cb7wfjnmgj2p5nlx","338":"hb6engteflht93o1x6ld6minxwjw7al0l9kvfaz6oxh31vemd5p3hjag12s1imz8oz21zbhz052f7n51ntb4iz4ndlahjcnltjl6","339":"lumepbqo9xfyqerwyv47u754vsc4krvpyg7y5kiihi26wy9x3zlhvn9e4dqvg619oo4x4lyb84lxa71ydb8o12xbq9nglitod4zw","340":"vdzqdm4b74k9mnsi9s7rs2r9g19bi8syabrgxuskyev239882x70tmtz41v6ewf1bg8blui943kkmlc7mzroukt2i5llw5jbovt5","341":"0bjxxztrakvnh9n8z1c6ma7ghynhto4ojzrzpy4h3hn4asfbe7yrzq7svuzoghvw8z4f9kv4cy01epk0c8hltiwyqirelt8wxbz7","342":"w73vi9j5v7pep4cvecjpi3e41gqgzxrpt7l2h3q9qazliqra2cuwy7iejy97jei99bhthljyeklw5vtvx6ohj0f0z41n2q7eipq9","343":"vs4rmiqakaxdnkzq5ffny7kz2eypko7w2ybqj7sg731d3y7w40w3i7c9vv2nekdsnb9rfx84osvohai6jmgvze54qhbeidbze3ae","344":"gkxvudif8im7f18ajn6z6nru8ic3rv9d4wghue9rsfvsn2euirticlhfmp7z6gxrbqpqx9cbw1qsuympzvi4d8kukwmyao5rsll7","345":"90furtlq83xnr87fqc57853hum0n9jbicmsrtwtf6ro07grlgwz9inlnflh9tgep93divvmxiwcrcxthiybyry18qkb80qs08pma","346":"ggymo37wltus38pfovqtosv9uudz37of6keg6sosoh96ahndn122iffpax4beqiesh3qyzpuv7a05uay6udopttby8g4701mkxig","347":"24a8f83l8v3u8ua52lti9y7t3idz7o89t3c0sjjybogip99raq99dpn7uttuhq9drvvuc6le7ug3rer5yanvsdo1ppvvpk141ld2","348":"yg72a89cpnhu6zvz6krniuc00apkd3bile9w3znraop2fykoci8yrz6i2yz40wwwm52a4vuebf78f818ycw4z8w65dwwiu1wxuuw","349":"f0q6we7pkh52ouxhzl7ady6xua4wzsj2ouv3c9igwy97kunhwdn9bxbycpwaam3lfds30n5fwkkfiwojp1yko3q5lep47x7u7y33","350":"4j2i4g8tpwcxp5vwysjr568qmjxdf83zkdvtairfc9h82qh3k024shosckjycfj8gis12oiyeco5axhmwi9sns6q4i65arp08g7u","351":"tn5bsav8brp78rkfbuf40oel7gc4cbb26kg1y4a4suf0j3f0niqbt0wbmgmwsvrdugiaa148m780l859edwmlxh1pyju8715gcv0","352":"4650kxhmis3f7bxit0kdhcwmav50ppjqrw2vlok9i46j98do1460i5474z5jhsudxh3u0i79brs42kau1jpr8a62g15owv1cjazy","353":"mj16maj1z84uw9vak3vl8u6u47j9jwz6j1kqopm5pxoffrzh8fsuhrgqt8pguqsar49aua0kdf07yg1vidkjjrlgye7z2ul7mat7","354":"64h3e8824o3ajdgyo3nfa2i8yhbgdnpyaig64249w1aa6d65qn21ssww3c3pozs7lt7afm3bqm9gbod8frr17wxf0yy0153i2yrv","355":"fheo3nup91flrfo1wlt5f4qssabh40schhswik2lgl8jiprsluqsq4w5vqyh3mh59gzhsva350dg857foqc28h6ot54ufo418xux","356":"upublsqr58ph43ykpfogfalnvhssiib1a2g1v4uf5kmy1t24uok8uf2nwm4zh4gn9kmtk0o6f1fmme7u025pzjr0ytsj1wu0x8vx","357":"udonfzun3extjswqqnmk15ygv4h1sdk9rgyo73aq2bp0bf53u01ah85eztqtx9o9qbekxx7sqz0v049dya6bo8nipu9z5huuy239","358":"6c3aqje1ungh1cn1u943ezcnxcs7absl4imj3czgxo2urbiuhrbnlccskabqcw5t0ssw6mar38obs7abpcxkeflk4z6s9o5zyfwa","359":"41cvdg3blkcoihuo9xim85mkihwbr8lmasssnauq1j6bhel84mnilpobrmj2nfmzffxugry7xk718j3v0xwnc1uwwdlt029be2g9","360":"3orgk0wpkukkgiabte3nhraceszpewklqkbwk4gxvqvz2iicqsherae7aspa0u6jztot0jioc3vhe9e8640kildm0anq8hk6g4tm","361":"rcs47al62pxn09u8k4nyjcijppzbkxswdx3exxegykvwyzm3z58fwqppxnh8z1welwb8m63m1kxq9tnd7blv0j9q3qiyq1et7hxx","362":"574btostqyc93pe3spalqr8i2fmwigf89rk73euwdmtqmdaap987c1df6geuolh2bnwf6ascdb4yjux9v8y7hainuv7c849mub1m","363":"7z5y2hc5olj2wcyhjsbzfuvj9ws0jegf4i7xp63i0nbjamphpyctlsi3x49e7j7g2qbnv45urq4y6sby19e0jevuh2vxd1l5bapz","364":"grw1dc9exe3xjh0bzqtazlkijo7t2q9nbg025u0p9spwnvwf6qbngmjnqioakw7lc1ksoiapciekpviwvgj3815t8egamnvf1htb","365":"qa2jakmsangnl5buglagkx8hdl4vi4n5m6qn9w1gyw4vt3ishdov8wlzcu2gbzsleozqa959o9du54kh6wv95pusqe8tggxa5yxw","366":"vexwze5vxbz5ule4nsyqbbrxdgcye7fv2s0w8ib0bg0wqq2r2r099al3otyeyzzk4u9ynmuszgithbje8je780qctdbzvgeeg5q9","367":"mvu0pwvn8mz8qp3valdmuqh3q7lx6vcv7fwlhicrdamt5c76xjc1xvw9qbff8sypg44aicore7gxuv4q0f6f3tih14e9wfgrujzc","368":"7wbkgw13ikdxrac5gg95rrfiybbk9jyikqh5uam0dpfirr2l5lo0xpd2pipyovj21h11xb7fldik1z1hvpubesa79ho7kz7vd2mr","369":"xz514qvhxqpisouq0of0z6cc37tvib4lhq6xfqfiul7xkfgk8c6yfyne1a6u9pxyr1mmnp42jpr90nsy0fv7htd2l6xyhb4prk8j","370":"928hdgvmpf7f3n964l4ex44vn5a5wlyrhuus6mb03biazftl9fslktd3mughfubbt8ak2zlexiktpqmzs6gszp1jn8al7ns4sq2b","371":"1nai1mmqbp6rmwa0dztin200k4ag601fxpofdpm5kelcwnkfc2qliuxx9fxrixepwk84nagepc9u7o7oor5huyvcn5quor9yh8cb","372":"u71dzywnlfhuvllz9xbshfwl925nk5nok9bhkeyfcladsll92slhu1te67ewrsdagfen6nagk74a503bmkpwwoq6mh0e3poqy6jy","373":"uk7wpfrj92q54q6kzro6ony4fog1j5fn50zbv0brr2vly0pinbbv2bshu5ekpgqks9xe6w5rupzjvfl3yrobo2evcduuehhcs6jw","374":"uweei4v3e7xpmdobxn5a6put2ih6zm6xlyxerupyktwl8rltoyasscpp48p4dteaz67hdpxp40k1u8vxqty6im55o4kamjyyju84","375":"5tiybs5izql4p9n57waj94y57ha1rs39f5koa3ompthxg9eluq2dgjie86xkevyqvxb25ooyhz44muabnqj9mdz9naumxgyn8qiw","376":"hpkxklug93n2afa4wa63d2a6gdtma9c67tdlvrtysqjp69ys5y6rl0wjmxl7p8s67y93pknijaaqx3y7fmookfsu32nmg8xw0xp5","377":"g8mvnk8h607l9jmw2ycme53ryso83ifgxmuoub0ojnb9a1p294cecrjfmd5jtz615p7d6vw27aot7xkohipp9a5vpykchw54j25c","378":"aiit1xn71lyvh578a6gda10uc0slk0p9v5bwqq0olbnnhxbqpiilwafq25srwv98sh9h3pzv9qiqgfvfg1rvesx5rrbt05bx8faz","379":"t0nw3ez8x42tf3jj2fdvywemj8lmhy1ifd5jj6fn9zpots0wioq8xvxae58y9xk4yf6ryiencls839wn1zlhbu029ungyzzrsr8o","380":"hh2d02oeojdiworiizi37m0fqli1p6xquzkss8jzmh0t8zkyf09tzm3uo4jxkjv790ybcwtne8ichjxvx9viwt6s1scdjg24qo23","381":"knerewlittk0je5mms30h22gcgoh2sy40rkqdhlekmh4urs4rm2z99jm9hocvsvclstkmjqs40cxj5dyc0jz8pgdyz4y1g0wjng4","382":"75e113l0j2l0x5rmm8u4u1ruuk83c1h3l4hh645snnp742ub665xy1grbm5m24qt7l743i5mjrbqnzv48wc2obl662vtvtl5c6h8","383":"i98m7q9v08tgi9606459sibb7qscp2ezxvxyg1p8vflopxerjrgqc7d085mnwi6pgxz89xv1wnbkk623kwz3z5iipoee1mwi2x03","384":"mvhs2arp6oznij66n0q5mu5675bolyzmfkzxybjusa9dbgqplrzpv2y3jdd9hzgol2xmfi9h4v6ra62c5cejbpcc7pa17kw07f8o","385":"mevxw2bghrlsnepi2fiqdk8xqi67hj52xvqmq0vce09e23mz13u4gsnabll71gqut4l8a05r9j03augxfz89a6rag6rgnb5d5ebb","386":"ak1rz2jyt03ds0i4a5xl447isccm97sq97ou2b42i8sw48kis6mfbi0qnxz39685vpum6d87ilsfyyjjm7gz1mgcgescwsc78707","387":"pxv6oslksusezhr0yy93jdlgnoe1v9azfa8k6wpi21gatcwo35nuw5m1dwggyuik1srvwj5jt0m2671y5ktbum13apg46okrncem","388":"93kcuwzfvlcefbrrmkxitva4oqx70bcsk1kamthyaxi4ix8crn99gv8hzzb9mzko3l9x8kgivb7syfhdk2p6kk9dah25vx2c4ft2","389":"b723kcyls7kcnqt8zykbofgchsjupvkgng7urh1dsa5csgwrnzm6pxdsvujemf7xugk9g0td4p2y1nz3sxmfi3jcufa57n2ivjel","390":"rv5uyrdv271ne2a6m1efytfpbc17b516wgujtbwvexpt2tcj25tkhmmtco2moz63wk4649ytuf4wmpnyrrl50riq8kqe328jofix","391":"t3ik1zh2sh4v3968wpy8w19o6bcoo6ldfvxsfg8iexicuiaoesbrqgvnsylz083kt4si5k2mu6dv23knpmy7axib9jh8wbaccshx","392":"e92dcc0irnmgnj0ioemsl9yxbp1rzo1jex9clyoys5f6xfh39kvljjqp0ofw4xplx1ske5kftsqve7cziodcsrfvyb4e99i21tkr","393":"fpox5lpifj361nh7goht9895ylqu6fy9l1tlsafjanttewgdx76sm9ul7nsli7cd9s2vtmf4k47mrqk4x1j4ebwfuy2mn6o3cyp0","394":"yuwqna2p58jx8s55ejlv90p4mjw22dhw2waw53z3q3v502j7rf63begcicm1x0od9rzvrwm7rrw7c682dmqutn2evru39u1c1z24","395":"mn2xlqohma3ui82gffymqbewptpw2onwy4h3ylr2tizyh813zq3lxv29fvd3bm2v1z57xkxiknh0sfkdbdw9qyp413jmgzmpbu7j","396":"qcx44m5r7ld7bx1v0rafw9s25ccvn0z92mt67zxh3wa5sar97do7wuhjt156f4evqy4d7pyw9qsrun9ocpjj6cz3nyqpa9vbmbgb","397":"2y19u5o6yhmy29qa2tb00q9yhzef1vc5nk64t3j7zszg2ajjtjqh6ikb7cupd7l2ys740by8wzt5g9x9u98hkmco0kk3x9rwbs9u","398":"ai2a3pcl6iwjyh2560zvyl5jntx5l4ynq6r2f733ory7ag2ov4cxdl6xn2jon6ibdiyhagagi7j8heqjunus6dvvowwc4jl08ch6","399":"idl6ibqfoydgbobeb4g6fa1dzr7j59yaatytd0auneej1xuss5s3n8ytj4rdtj5uknbfffs6ek12n8qr81rc60pd6swnezhiy1l0","400":"6fe08pe0qrvu149taltnzrgc3hubo8ewhqtn2mioib1rv8475ha7kdn7ew4zjshnqwxmichguqzd3xgawlt53mnyxt5sp6s3diod","401":"womx7pdqod21p1z0m00zg1mn0mlsk7toc0poask6cugh53mvmg6hifreyq1g9zuj5jp2pq8iqf1md6iumlk8p30z0rhro6oze109","402":"97dswr6wcpywdxb6t3st72efi1fev0rntjse7wefu3wtttgh75ljh96wonjw3jpeo1wsmtc5g23apa1db7e3kvgauzpgyhjyohyy","403":"s8p0kgm17d3n5vidbqar6cfzhp3r5omsyoev4o4c095xcq38rbrsi99j4bxzj3ha2vrt5ppvj84haaoui963g0lh4069oqr7bpjj","404":"av7zxfko8b6sgod8l20f4mlbbel0fon8it3kcb29yhoovt7bhdiwo6ocv2gyz85wnb09dr1d19vtfufqb5n664xsjigi204td8la","405":"s6mc55lx3bt77s5qd9yfp4cqt7zqvwk371d6rhomsbqijsh8wi4m7c78v93m3m3od6l1ms3u0102wu32w1i38xhwvdrrwt69l8fq","406":"6degdj1hl6ihhdmjuvkvrdnakv3qfnexm2dog32jyy2x7zim3wm4yowk4ahn2g8izly0l9p3c26hpvtitpre1qz0nzma8165ruv1","407":"bqrua5fwc5izezwtjog7da8ol9w667zukxoy1b964h7dfseayjgnefwktikx0ax9q4q0h3qqzxri5kus2y0p2x6w1s4vekbiqfcg","408":"nldfhruadwxz4d6yvwzq58g82c19pq4beb8gsox407rx4jehmw5evbocg0apox1qrzsijm56zwqhwxrigexryz2tm872ssdecede","409":"s31tivw3h43bi75ba0oiy7whhhagzkdfvdzvtiooyrbo5ii6mc9e5oz95oz6igmu57pj2mityjj349x0uu8pb7oi2ajt5ckpcxbl","410":"17wc0pwd2j9ca6oqpc8dsw5mega4sibkcotv6bn8o4ockob9q1vyfirx3iisfhvrou35yidr9l2wsyia3o27n6utfg79eel2dpq4","411":"yafxdx08077vtemttwuskzwrdhvxpfcpxvsv8q4110zl1srzmc0kqb7bvdoplpsuno4sihs707z23pwss45onqd22nrury1l5rev","412":"wutclbo8hphq1al36tf35bdrxs6awaru4colkbd64s4nxyj2ownjary44a9z6hlf8u88h704iiqsk92oxe7yyfvpyawf98phafdo","413":"7jtb3lituqgyyqsyo9tmxfwd8viqj87yqmvpqm9pikbg8r99a75xvwox26zfd0m2kx8g4kybxjq14ycp1nzxy17m6brnf79ms2lg","414":"cl4dssn6dzx2ykjjlm0ybv42gv3s3v6f09rpvt7l557l7dtd64um4xmuzm11qcmobbp7v6n62ehrvslsngzsv6vlwvmiheozglue","415":"89v8s0bi0pds992kn8anezr24scskvhqbtr3ru0mdlujzklflpss7mk1dwtqrwomivxnjyr8602jzdhyzs8bcbffjzo29vxpap18","416":"i3bpc878piadmpk3859bhci0c6q1cuta931zxq3fcxs5jwok6cthokjkqhoq2l8gcs0ix60ix6qmmecgk8m54y1b2smvi2f5h4qc","417":"bhaadegfnhfvt0cvkxy570dpyp9uehxpc1idartnfzab16ctrlap8vceodvifueyrj0inaaji45wsh3cp0k57vrdfbwckveupzz6","418":"5hyk2xdr4jwrq1m8gc7u7qeh7novty16wenv36rwtkpg2jidcz2dnmiro0jc1zbkie3p27u9zo4lc4fentnadjh42il3r47sserc","419":"d09ds0bbonp9e614mxz14wmqvunh5wbg935yvow4ku2xqw02jnabkny09fn1j9bi5i8ozj7kykx0z5c9r180rctu7cf74gh3fnkm","420":"uyjezu2rwo1239e1yddlb1lvac8cn378pfg6t2di7dpeabs91m21742gnkbu0pfqv1jtbjar94m71r3s2zzele7i1v4wfuwabdhv","421":"wj5s32xrqhoq0sng5abhjcmgp1ymmgpgwejg88jl2w2p1wng1ser1lyyxt5torf9g0nmgy8755bltkunjyzin6x1zg6w1thyn733","422":"es9vqlb2z8d6c8c29s0bxvfdmiya14a6p15ar0qpecmn11di1jbr7oqse1ag5saxkoqxbdbofwzx9zkken5kyk8ivetcn22sxwpt","423":"gzgfavessknzcpxr9eieer0ndaahdaqmi900s005dymqcfryf5ih4nkr8sk3hrmvdtytk3kpxmvmj9h9xr0vyr6btmq9s0ojgi9a","424":"5lz9l0sn2t2uep30hjk2utxf9xycyzjpo0xi9kfz2k1efswr1ahh883dumv8q6k09eeagslyq1gfkxwtw1ubybuc76o1wddz2cim","425":"mws2hut7tibotflvwiz68qa3fk7aeyaloyoy6srg67ntd1feuehr5wiu7swsxegssgp7vanzyjqajjvnxl0ugqp4wug9ktuj5bsw","426":"vsmtgf5fip4bsvmkoe66u1qeydnvdfqw8vjviixc9cg8cbl0d522o86s0cp1hdomubj3bxy7ntwvcou56f990gr91t282avuifve","427":"niorezcolbxa8drw3y6p64j9iuzg8e8kzkk6t7ywfditqamoailgshh6bbor39r1vy2oiyw594q04od4jsxe802kh7tyqwathnml","428":"80p3h96htd9tuanfrpmca0m40aa0d61s8ubx9c32khnf8qlk8imhtlslqhbm9xctq3iyihm749zcxfvhxcbuchtsgfb8fsjlyule","429":"bplqzpn3rvd3cob652mw7yp0rpbeb74dx5461ap9elht4ruwlzbbbsk9ll7vu6hd7l3x296gjxhr6g4dxa5nzffhf45idzbgil4o","430":"trgclw6vb9xnea1owh72k9xfz2vdb48qzjyi5ryd663uth7qa2u0ajyub4m9gh5i0mvj4qzv3gq7itqx6ohw4fupnv2dqqefekha","431":"aitz66h50dj52bumpqiy0u0w0x28vyc22ck8ae2gawe1hha0u7hknv1htn5cpnu2y5ylrfxnmgjjue05gki1uwjgdxl8hd9fl9g3","432":"p6kvb1pddhxd3h8b02fws3s1sej19vowj06key26om95jloui5yt38njzghdhnn81ykcym0xawozcr0vx4s6rtx00uxp2p7e1337","433":"2bl4x9p6u740q5aph8u5nv4qiz3ipwfnau6e1lwrr57ndq6b0twccxw9nj8eqr4f5bbkv3kfncv9n3accchqe5vdco2ezb0qxco0","434":"y6ruq7wgyly8tvo3rzub37ywsl3ob2f9wjgmlpdqk9p7s1ypojr8jaox4xjyw1ooqkygkcfwd0lf12mgwzlqcd7lp3ieths59xnc","435":"sfxzveu7tjxuioe2jg15p5l129z22itqbdustlbkw10se807urxjup9ywkvp2y12hpepoz5v55yw95ns97x9thqad61ktlfw0oh4","436":"6olg39m448a6n5k649hw4ebchh6i5dqgqtsff7a9aedt2iqnhon2e0w2rle97frnab0rffijoh97uq57faygy5i8fvjbuq2edjog","437":"wugdz52uqk3opki764jannermfy6q6ouza4evvhkw4e7lvsfus8nnkg4lism26rwy5o0f8m8491sfkyo83szw7zbdsweg0f5fmwa","438":"bx9h3s2uwdsljvzpvxptqbp71ob38oyjhtjnlc1vocf8tthtb13v2xcksfldzsgcddog34qxxmu91wfkwjk0q54sord8hxw1sfxg","439":"ihch5v5q2d12e0t33b8t9u5q6tqbpyq77zxyfz406bgklkcqq9b28w83p3loq62nt8ieesdghaghjfozxuve530jbrn2l24foitk","440":"62tky9o09dvhhw1jfrpdyburspevrj0q227qh3yrk9rlrsutp35qdkza891mk21c358bg7ycon2lewjsgy276nkqj20n8imcab3x","441":"mai54klkjzewlxp4w0yoo8zpebrkwcy3rakpzvtn6lits1zf3mvnvyu7tmgqloztn8oypo0qpaighuyz7q0bfni4ofhj1xrm1nyu","442":"c4lnzl41n2ctoegft4yl3bpbxtm7nk8vwrxj4fi3tcn77uhqre1r3qlze6ygx3m994x2z2qwez6jreqrh6b1li4jfba235lp5dvr","443":"aenhju3o8oc8fcry0qun7iokth1uko3fekz8v7h8gh6t64lpz8gyzchcvvlmqtrjfpmnmzpetny15vr9d8nsunc0qypuea6wr4dp","444":"3oplrkm2wmqzchcl66gp5rnmjsiqgjzh62i044syw9j05jjrzw9zzp00k73ggzw781btsc1jvgct2jcy8eb4l0ivhmoe2jzepx33","445":"hqtr4gmi6j00gb2luu7kq9x2j9ck73iw39w0sqfzmvalupc4mhiz8om1a57rojn12sz2niuoqbacbvxzqufxobrki8f4y6u3opc2","446":"qd0xvcjvxxvr60mu0ezsbrf9u9998ich242zlwt2621qanqarll6cn3v8i9hh8uxdcw9g4e0kgpdzx3ul058uas75j7ho45w3he2","447":"c6cgf79x0qss5hegby8io1jvipobga6rmp2bt731qnkbpd41jduhfzik181txf49n8r57tws9r25zvr64nps7l3i8nimdcu4qauc","448":"rn93xyppbrkb1lcp7ztk97imstt00qluapk9jn2pjbrrwvrfadna6hl6klpodmgggdj1iq7eq06lt155oksbsr2ytfoefg7zehq7","449":"hir70v1xmq5thhg2sapog9a3tvgn1orc1n618zb4eatqkl8xum0t4jmlhtnejlgi8dtcw7kfwpm14oe9cbsmaia0cljxsltzmm28","450":"54h0wd2grbbmi4433yz524ft6c8qo2yu6w2trwlej80omimk38l6yofwbmlkin6cmz7pdzrzrbca92m3qcvlzimftqqe7hj1bgwz","451":"akixve5q5rjjar7mcpem8ng8oshgkzaeaxsg3vgwdugpdh3b3g82vaf2y1t0gg8oy126mfp6pho8k7ic67e55rieshfl57i3qhvs","452":"aibjtqxl7m2baib3wp5bmclfmaoydvbndnqyon48e79ppm39tdcbgtwrga5m7kg34kqwe5h3v6vkhdj3um8jpkw0vbll1i2vq19l","453":"sb9k3qp989ej1q0pc306pwpy6vz7yypkv5b5uj4e9nomdojw0e1tgtb3dpvzywr1m0ej8x5fmegjlqqlcuynq0650fjtlsbfkpoc","454":"trf4hzqsykypes7ipxzl6xrlwre9xn2xesb0juzbu9pnh34264vayonbuql21ortyqeomqjh5z6hzze2eu831asbuut7op3p29k5","455":"973epdsmbtbk3ebdd6vse4znvnsd4u3jp3ti96umnhp5y52e5tt533kvypt4zbii6fqay54kmuhn5q0n66yf6o7zwybs94i2350f","456":"tkm0wc4l73e7v15i84bkxg2t1tjzlahspf91dq0dz42tzb4ymfjozefzb6tz19tash8hog49s6l8rk1vsn2wjjt37rd4kp67dkkg","457":"xibmh6bgqlryyorzoto1ta1v1vtmoog968ykbheqj2h9orgng8ngs9uydi5ms9n7ejzzvzobpei8ek05s2vvzhjxc0vbya3yko5g","458":"6genn2eo919lgv69cm2vcfey11yudsxp57abdycl29o2xb0crfuoz3q49r6zodyn958cjkcghhuyx2qt82q2pjiyygt0ocqy1ob3","459":"2h4ddskqpv4gb96oi0l92y3kkeg2mtgjxw6uoqixd1p4xx3cmo56hph80c0pt4fsqtysntmb1trbu30gz1yq84e1ekresmfnan4k","460":"8tc8wjy68u0pzpth57n69hfbso3bj7u2z1k51cx04kupthuqgdlccu0ppq9rgndlwxyb356ocb3liap3ng4hbz738dsozx6vbtqv","461":"gryhntub2e5o8mt8os6u35y46phkke2up53x9cqug1qzqwco04a1y1v7mjdu1imo8gaebp9f9yul6kv56h4n18gwbz16yss16pfw","462":"q9umil81mkp9xkdhxnulwvalt28b460hbe1rbzimmylq491kskgc02o9k1knn404alnod4hpjzagbrvvsz1ix2kq7ch1ar9n9de9","463":"yye7egsfnczqbb79r223ug8gh40cwwrv1oopeqglp042be261yrc2rgkud4erhtarz2usxzjb0ha15u0h52j00aygf7x5mrjpvvf","464":"86tsvwlujqwz44d351t6qxbv0uxd3x7r3jvaqr67qk6rym5r1zq4lr7jtdz34ehe3khrzszzm20cdfpnm2vyo48miq5951z3kw5r","465":"bfk9xz6ici053ckvlt1w03srkm8gi8k9i0p0jhqz2g280xa4ltlb2tnbfgngsaf0u9jm92wqwyduxbmdrg0vphlbmtxdiw57sdll","466":"m2pjllsibwwsc94rjp73fwrywoq0r338nyowztc4943nakarohrwrx47epzsxwsmrnunsr171xtsss5yx0gp98aurf2p4t9op1gc","467":"1k2zs51jb23o9i9crg0wm46y8603vsyrp1vmyah6dpgvuwk2rtnewmv1teq473mm2nggag65bb3i86sno7ep3uzf3d925ymirq6j","468":"0sahijqerkbqszjkkt394kyewltma8sf3y0rd46mct9h99zzh8tgiyxc98cg1y6n637qrhinaul0ncl6epg1x8ruf45xwwdmmtq4","469":"utwixxmp049np2twi77srciscmd0ir5yrlm4n6v9lnbvq0ei1cdixpide31gidaidglx5gvd50gwnyypt3tfneb7wenqurxmiamr","470":"ylatmsnb6zskkemsqwupm9hhmpxwqtxjn89mobjimjs5z97okm0tvm9j220q390jb2o9o8oo2w3ay1qbvu6qplue0tayokdmcsqf","471":"uexfrh9yu2m6umfhx17n21q2ns15fphlmxjn2z8tg4xul5t6pskao2kwezmhpbkayvtky31hll4ya8ckakkyh0q7j9c5od8ha0kc","472":"koye9ucw4rade2mvoxzt6u6h7kc26y2dzbfotd9u8f263n7mpfafm9h5qsnosgj5z9rodrdy9ha23d1vx7p03vpdp35q6kxg9gf1","473":"gkv1qcvo81skxjwz8t6wpn4un7ebjcaq9zof3b17iimtrp2dtv7ujxvxu65stvis5x1cvioc39y958y42tbp95lvjiu3sl516i4g","474":"9a7g78t79us4matvg93drvqw82mx11dp6tzitlgx5nxac8kvvmv1ekdrfbkrkew5r7kddelmb95zl55o6s2pwgbcsm0hgb73kx0s","475":"mghwmiymsi2r90cjb2nneq4tmxrnamv1w30ccl1s3bt1hs6uxolgvgfe6p1zmuvlnjtwhbi9hx2mrg3joxz32qww86c488d7ybap","476":"fx4g6eze4at1l492o3e2qn6c88bynd4g2lnpeigrehapufsrilso8tx5wvb0fg2d2sp1g4hksj59tv5u6fanmk6ys5u559z8pb3l","477":"rt6o4ns2cu29vvx4yci635xzht4pashu3pjhw081nkvlu40kb13aextm31t8zhg270loopdmtpcls2eu3qksilyo34prx9fweyhz","478":"61b7pqgyv93g977l6icver0ffy9y126gwqpc7rfbm9z7zbdku2ppp1jlt2sylmc3p45b3wlosgmq4mxm5ugg4xr6e8zl1q93qxwj","479":"64yybusbxxba58ewlr69g4vn4opmxplw3d15eawj4bdr5z7k31vtmny3lbaohwunpom6nivpdn9nnuooxvvhgb4vmk0pot81j44u","480":"600qxie3w145qt4pve93irbnpz8zgdlqovhqhwa06uby2yh1pcic4bx0zcctzhnl8tl87l76f95ni08lcq8pg2h9cdmuw7r0yvaw","481":"fej8lfarvbkg4u6xyxyzjkmuvpg0ae4o6md5jpkkmkfp2vpvvoyi6p5svkjata8g1hz3vgc3v2x1uw61oqquubxv7mzst8u6iw9n","482":"vlik553sae88ag75xk62nv9zbf9hmru7jigopin0y21ovoe7t6irckh06uzhb0fbphikiou9pcs935uosttxnjviag9orq7pqij3","483":"uzaztoovytaakc500769jedzh1hf9ron1iqqxmq73jll86mumnwa8c9nb13eumalf4n37k6ya9jftlmekx15fzoec8sn2l92yf8r","484":"s1cq2zg0pk55kq6ls8wfjq7c5o6u87gj60a3fbgtobog6o2c8ne97z3tkkpznsu2kcg2hb8ko77fszgw5notk5j3ymbnse8o62us","485":"np4v4u5i6ghgl83vvh7rnq2ud3azvgtjjqgw5w4a85t6kuzm1x1p4modg4xucixa31cikwtvd0uhkkor2j9cleuqtc3ggyt4lpcx","486":"oc20v7fow0i7t8d9iohxbk6pc9fyf4nin9q1l44xzepf6thqn5p8n8qy5o26zbzqdmdk9k7je871hf06gcqwu7hs6k3ivxai08d2","487":"tqwxacukfxjw9vea89m5tmf47tfy0cpiwz2jjch8ajt5kq50gyyk9udwblpjgnjnyy1vv5rlczc7ysw4lln1l95m05c6cu5mktkm","488":"813js17sms7mlrcxfb9mltwxa9h59uhsgdczb5ds4amdx5gyjlwxhamdxdmkjhwdnvalhjvts5bolgi4k6u0r25ydprtl94bpg9f","489":"dtevbmvx2f75tl1yge10ugnkbwkxgostrme5o9856uwgqo5nxhkujr7jjdhdw76040i5lrbpfqvk54y16semxoufee3djp7jbiwh","490":"9h9en0dni2vxg65s4n1wfuhytbxulnj3zjhc2566cft5td0gykkncr4popci1t8s8swu08msx6xzyhyin06mnsqkqwbz99rm1sju","491":"4i5sbnfkm0b0j4i1pamlndg4dowlmvfeunfddpeurpzinx4mweiiklxhyng803cb151ru9piq0shwrqidcimdt4rnbp6oo0oy7kh","492":"e2l3ws3qvag8skkw9hh1cr1idg5wkffffmfcrk78ce3yv0xlfabuxlszwfviuvyyj7o22t933vxvt07ddnmf5366rr4zmnmwlcoi","493":"w65leyn32wyxjck76ewui1dzku5xovec3r306sgzzazwluxx2346rou4832ja2k6z50zid76yl2haq135q442bewt93hwrdreeht","494":"xl3mfoq7rb4wc4xa8jt01u1atl7nt6d26fjkjop3pb0g1fc9bv28l3idj77315hbx5fb20645aeeo70rnsyti9p4fey3oascpkzg","495":"awfp9om6qdzf2g07k433eb9jvswo1weq56nc0lmzea3es46bccqmab7h3lcvo3aad183s8g7ggcyenpcsfqp7t7l469kq0rvon7t","496":"qf7yonw7boeeqci12cm8j8oh7nx0nz36178plw0n33xeogezc8f0qmrcpkalcto0y2ik620o14bep4twygbcfpgsmc1r0eu6eja6","497":"43zt0dkrpiakrs3gufel8ga22jaucff7h37yrwhtz2z03widgr24oesdhb1ljgr7vy79tfhi0hch88nmjs7t4csc00y0ibe289q2","498":"e0b9n8yxe7tnb9jhhvbfi1znbrb10xp349oj8p40580buut13fe4xsyxrqxu9phpajglvjcn3w1z9435swdf0gwl3gn9aqg2oovr","499":"ufymtqpdfwemfupskqufgclvimibq0hjqgf1uiv528xldskr83ncnpza0zfxzj3pcq7yg1qyqoxst1n4rztf4jo0s06l6j591twb","500":"j9ks7rrhmu288fvsgd5gkddhdch02yrd2o6mxmt9ydhce1ek92ym1njtwb3z5b0sybwob3nucueshoo7di615zxiyv2m5fdmkbos","501":"fe5c41z8t89ozvl2tjk2i8whcfkvls7ncu8msi4ccsevtkdebp2un9xncl3l1zjfv78gwgysgegf6oy3tuv56ilztkg93igndq2a","502":"8mdipv0sd055najbc8o7hippk8lspsey7y935419jyoy9zqxyz8iuxbxeqt1qv0pnfkj4f3zplaefl3svw9yay0kofldmgr7d6y0","503":"tgmr1o2gkeq19l22i1dg8gvn8w6em6stvlghz2rfh49tylq20ikftbqo1so7hug83ppapf2qbfl63m1hk8x9bx7pywwka9t2d4qc","504":"dk6argmtkr5ye2c0qtq3y938ptzv09vur5i3zenlio44txxvlporo141zzc2w4p79ri17gb0da707pc5lbkgw4psdayklaqv602k","505":"8zb2ev4tyo876uljjyafa6sy0i92geiydhsbk1xuzuz6yqvwuw6m2ie04mvofyqj7m8ydnvw5s38tndrd8fo3nyp449ob84u1mos","506":"jppoeghrzgck0qf12k36vbpg9di1fbzri87j9hezt9w1827lruecee5rillthxat62p26zhqz3dzcg0kr8w1uceuh7bl7kb4xo3i","507":"14w2n2jo8r9w84zxopmadzqx3x77jf8a1crdllvn1iyiyce131m2pqopv9ueymrekz7qc7dsp3rf22nuijaphlesu50x0ly4kjiy","508":"bndtxnload9a8a2l6krv8g8dv5zou88arsi3a3pd2ujhvjbujzcifp55i0cr8i61zzdxanbx7p8nzay94lofatvnm1csku2511xm","509":"q7qhm1qcemomt00kfrgy104r9exp8xphdp9g2jdk0t5t20606ovlcxfrgioslbpgnrku7colv9eir1cxvb7kdkd5pqy72ejvx4xx","510":"5rhhrvac3dp1hk4xnqz7yn7y6uk6lp5q2fttswp66izquqmqu9cou1egcttp0piczeel43uu5sak6lf7v87ju89uzijbrv88ko2z","511":"golevrg7x618lnrd95eh7b104j7b07dzpjxr7ce0m6twprdcelhn5zehe0atvi1k0vwfus67ifrboairhjf74il2kbj3fmvsmp5n","512":"muufm657k90zo60sxpdnsos7bvxdn0jqguuq414jv3ohdng7ztn53fd010uqxeiwraercpy5ed4s746m9pp2m5x7l926ugxvqg2o","513":"dohhv0xi3pp4v6zcm6z1ik13bxjosd6tvpqo53mszz4ylzvce26umn4p4de85gr13ry1b49nof51v0qy847f628zclrvamzfzwx6","514":"zon72ug0k30iq317vn0lo2wqeslv367pq3rrmav682gzv7u3734aiq3oq8r8vlnf1mf84tk5322hfnpmsg3pq387blc20r8x2xd4","515":"r3x8kibf7zhi3o5edo0ck664ddzl8xjmthoqbqcd4rhpzgp58vdjk7gg3l0hw31rc3psdm2sqjthuf19gg4e8xgfjbg3h566wlnb","516":"gjhn5lloej02p0jf0a18t505j7xxr0s08tyyke5pr75yyxcldwacdvumczzvqw2m7q620pgxjtwicuo43c4rzprklq4gziqvvrpp","517":"anjwzd51pjg9iwkjs5zrnrpr8msfnn75y4c1b4n6bq52qj58sgjq71i24jnhwaxvvnbd9icvw70832sjra4sd3fs6is9isokqhc5","518":"h9ncqmk3pnaelifyjqlzq3m6g7cvpq0dydhpowj5i3vkoktu6kl0cj1w6wq0nqttaqv6yqysb43ffyfmjkub7fmjd59h3izb2kbn","519":"6ec656xgdsrn8e4eaxlpc6fjx2eebb6st47c7mvqq49w3sukjsn6dynw7cvwi3v44u3099zjdf6yswwt1iaowyymawzywe81ou01","520":"0ydbii1gt52iv7txy667s48c3jh3sqni8bax8g2aux4dl2nefogjms73v9p0v71lgl17p4k9lv523oci67pdr17km3accodrwgr9","521":"rtz9dclarg4x1r6evb1kha3c0klujekv71sjibblj1dzko73e3eue5zexbnhozyuw9k4r6s3fk9ojpdrli5gkszcqu59qkv53xx3","522":"2zyvdbycp53ha5dqq648zcznhornutfa0vkq110mw06bm3p9fwbdiiqtudxqr5cij17meoxjnj9a91wf4j1coo61dmet3bfjhgev","523":"zbwoy607nxpr4da64f3v41u4jcvhnpfhd0u19h82ml3l5fk35f4srxwdp4we34bui1m0ejgxjqc6ib0nuu4g87qpha3gjx3goqh3","524":"0hja42ox9idw2brll15w7324jxyegu9mkndsa7ix874ob2nthg6nksrfrchnval7twpv39tmmurzstva3lscdo66d3c4qayots8i","525":"tultofg7zc5xdj19yc1o5bdnyckkqj2pnxpx8g7qrx9bbcj7cy6x191yu43c3jc28nx6bl4twin5n0at05nvismsw6ifjoom1aar","526":"m8q6o1rilae4xashshef6w6y0r1fj8ctn9rto63cuu8dfezmy9b0m9y6b6azxywgt24y3jnpskdu04cnanx5jcig7x3g31435dr9","527":"cko8zmpgi5ac5pqy6yvb4k34xsnkg55l4r6l1y0cnk8n01mzd1qz3tmeuc6zfc55vqsu6g00ian0mzvfr4psmb4qu61tbacnlu05","528":"x9c1068f62r4uj7fyjiq42e8bc52wc4mixj3bjhja2v078f1mzqvihjtpip3fco1ovmyfe654vqffd6k3aui2iqsnx146zmz7fkk","529":"2v8al31dgezvhvk4so1ws9wjrykljw41imasy3g30jeof6cxw4nuhuly8v0v3qkdzjqirhum4d78wqpijl9xo1wxy6v5kgux4od7","530":"1xx03mbz9rv6tp7uschw9oa5b2vdnrzzz9utf29jsvk5awkd9mi498pqur4k91l6ewhky0r2at4iw80l2v28do6171tlk8oq50ed","531":"5vl1kydxvcl48wmdn8v790kxtt4gwuxs6emuvhojfodvfnodgb3tvqjj32prz4fo6804umzbx3u3okrt9lqctj3lbmuo9ggc6ag1","532":"stii8pmklxzec7vrnx4wmzdyvmwst58z2ft9vdnshx7hd1v0275hr6eermtvusicav662chlgtvxdc8w83uyzu0je555r5jzuip9","533":"h06r7jdvo6fwnlh9sj0m8r8sp19usttfu2z5albizrh9v96fnv2mnh1ishq9x1frtth41nmlt3vgs96vsvhu5p9bdatz0gmasilk","534":"ttmvht215eqfn84ye2jz849rub0zclfgqdxdlioarriddpboiechng2ltln0nsjwdpvuy3550guhaldbwdagqc0k2j89r9ms5nz3","535":"jomunzugrs3yo0182ajvukmhq20g1tjilcsmlao91o3ryrs1ilfmgi927ayedicpodmwn0m36hsjlczjeg45qizrq0gi7azznmds","536":"bmopm5purphawkwzbfhgzob82mg8j48lnt863vq3s4wcdl0ep7v5zrma7oi7dxvveqy4jucc9rs3ba6j1swf706khnp53na8ye2g","537":"919ck84tjbq2na4hzuqva7ko2zybh3j7sz4s0btipoqnt8ovu6cn9ksd78b72bfla7ai86ifonm322nduujzyh5qk54la8qgdtr6","538":"twdhor5s84p7s4pmfx9i9kieaumk7fixmvhfb7l9qy11bf9bo9dwqzb5wxdsjm5kurxl411m5qxovt83lg8jqlngjb7orvdo2zv9","539":"73mol29jn7vvx2059rnel458qo4beyfj2onwl3w20zi1q874gj15lka5a3wqn0ldk6gr3iuj7w9bs3e45uyysplmq6yjdtzuumu2","540":"vrfa4sgx47mblogidi7ai8wwjwrct6w5bfaogdy05rvhk3x11dla4sxjei80uytvqn1c8lrea1mcwx32r38o150arn8anhqh2urn","541":"3arlzgyrlj2rx1a8uzhgr1jqfr45upqo2wwgpt9an2yltsu65efyqpjhryya9n7h1tlcugppja25uhw0ymf75yronlf3nowybr3a","542":"k86g9g4fsjotesaynhexny4r8kfmjw21q13ac4phg942bxs0tjhlcynp5hmmkq0es36o5fg4crrvof0pth8b9ioyiqxbsdx5388d","543":"r96sokibjy8v7ucs0zq0rjayqo0hxt2bs44jpztbqz69ys99agj3qesnypncmzae70c6w7mz3nv9hts1nabxrl7hndpqcv5qjlsh","544":"rb94hi4fw7vvfoufa8yniomtli878vs56muluzzpuqrk2xtt1iqjx5a1me5b6sk2onx25ttt611sr1q65izzuuvbqaaok1ag4j7n","545":"0cqkfiq3t342h8x6mmc0e64qxvdxz92iod0n05omjtekgm6fhk4ldg2wwh1bjyuglsci29bmuizyemuz8df2vdqpn2uljm7gmo28","546":"3v99vb69shqdffe4u2fjqny1lna6yphijlx2et37bmpw3zz25z327rj45mvl5aelbe2f42w5izbs2o1tadw6usqs96jeydve8vdm","547":"7fq6j6ay2kideoz7vqrp6x1hss5prb84txxxeubf0wswlmzkk1iac3maun2n8shfqz8vdzi3u3zbw58w88q2rk0awwhkz4hd6h9o","548":"o1yghmo44orog7yq75oe6om3ebqpw88p2t8amm5mcgtjs9eqj2a2mkd0puasoql2j1uwlqu6klh9v7tsg96r0h0mukb3gqdqur15","549":"329ru3bk83s3hlzgdinup6trqqbbvjsout7v5p2t4ziyhcob4d9fs0x11o09ox040xcbchj4egn3fsuk46tt75atd8nes5p9kika","550":"eo7dri6scqdo0z2agr19m8b358pss71nhoavt57a1o27oko4eqlcc80zqiy7o2q3mt049l0olmex4dto6lxis17khgs8npi8mvuu","551":"arm486cp0s286ekwc0kbpgcug7mfx36sr9f7twq8fy290qpwc3vfaq94x9f2na95pki5zzc907dk7c6skwrlhlc9wp8hjchkvpxo","552":"0z34hfgfjmaf655ptal5cdwmo76ybjrpg90zcvo4pe3qt6vpcegvredqulebxha08z70qec0q2gl6kpquaezj9b06in07iwh8zkc","553":"udll8yf85k9qgakhddvi8u5mjfdj8jo00d4nyxcrgiej9fjyhnhid5fjy0ny7q0t6jrsta92z3maxg2hk8l74jueujobexinpu5o","554":"qac2iqc8vkhthpt1q1r6anst42vtvrtrwmx5lhr7god2jqv8qyz5xcj8qy5dlq4ctr40rrk54hrzmhj4qoei28d2nuteu924jzem","555":"dlke37py65740x2iv774zvxwev4w4ynnz3ycf3ljgpzuvvbzl3zjfbd2j1th6sy5axq792vwqm37o268bd2qij90tzf1p5l2gze3","556":"u2it1zm5al63kq53td9jm5m8pk572ycmi6gv5uj2g1tt0ynla6au4omescro4zfa8ex5buwz8uw9dhofhp09xjo7c4snxay3x128","557":"4rtaaxk9cefpg5jqdahmc2ki9sbkfvnx0qv6yldjg7dwb70ar8u0xeohv5whe25tdnc68odzkqbfi43e93ulett72clmqnbdy4n5","558":"0hicz2mn0qo9may5qjfxc7xi96fbyk38glxb6c0b6ryll9uw62h5edr274o4rq2a8sg6wnk2ssd4oh6xu5429flphjhis3fxi8py","559":"307ucsn6tdy5jg783pedaf6e7j51vrixan824kpx0pgcozuq4hi5k69y466rrb75h1wcl54m8u5cdqxgicbe201i7cy59hpfp1ha","560":"z0s2w84pmr8g51t92h8dcl2vubmd4a0ntt9cwq8jqgz651fd15tne1hf84su3jyvsaxccj95b132v1nqiq6lfu93pgh3qq01288z","561":"8z1gq8wc9oxf4eokursuvttm8cowzvzpwo5xfxol7pguhj8qnhnjzw3z4cukhpo0rhjwldj783dbbo0jbl4c7vbcecwhpnwizx1a","562":"nsawtoguaadabxksyoqry2ux9nywtbahagvqr5uxum1l12ph0fb1dcu5mqq5ukmalmclsgfdsxlkjztnk6z6knxq5gf6scyxjm7i","563":"tfq9b65abptub6gg5jyu43b8wkjp7ca5c580802wuw6x348wvlvp73m5yd8uyqhun0unbhlr332a4bx0qnye5163dq86sirbokfh","564":"ryembkx50dwpzotuxwxexct0rgzhe28s87r2c6ujotdjs3s7119pd6y10n1vfdi631gs74zkt7xw7hksfy396xs0ot2tdrgfqauv","565":"1r2esqcz3gjmonky5nfe6owbilyn3zgbnaiu9q2q89mbr3t0rtc45gv9jtaph4t8dzqu8g1xegz4i8h0u59w5bekbt9qa6bm7mys","566":"3q430c99syavh9gm4lk7qr9j81p5vyfljy75sthkf4dv2g2suxo6ix4g9as6tebclvq5nsv94sf48jryd04owm8t965laafzh4ik","567":"30lrvku7w4c50mvv93ij5s1sp0fckcjfbg1782v3yvjp3fdpym16q005buoh4zqzh70gtif71v5od6q3oo9vibbmy7m6ak6d98lh","568":"es8okisw5v1xukmxdmj7g3pd5j6ozdk8cgsd4wqkawiymc9v6gogmiviaspugbg6z4fu82thsud7tvron0b26lna5y2dbh9sjrfi","569":"cbyhhacg6i70zkjbvg1xl2imp7jtz58d8ogm7k7k4qgqecjguvz259wsgq87dw6aa02d1mi7g5qspanewymuwzk5yh8y7luye23g","570":"bljbw99mzq7nvt8cj84sb1m89rlelstz8h2m0dbtu6iq3yckbkslysewpr4wgpxa8ybzo272loz028mnyazilzd1hjdt2kz9cg3k","571":"i5i6u18f1rm0ht7nzo522d2nbvpnhb9qxb1nvng9og0b01ddiu9hnpb1gfm63tfuvxcp8a05h1m3k3cp4uk2os8vmreowwffjjir","572":"5ik8152m4htuqfjkhxyksmhrtllyz721k39cwnc0uwa7acuu6vwat2wfv39l4smuo3o3plsqcyyhd8hyvszzfro3a6wtauk1debv","573":"ad5wfsx3z3t9yfwzidekb52icrfju5eizkn5moxnroma47zhat9c3b2t9ze9qu5nkoaymxbv9xsgmdwak13k1c4slfk3upl9y0f3","574":"7m80y8rkmc0ezizdyqdh83kixriuk5duawkuifjvarm9xmdmxucl3tu91quaf5ascnj5zt5sgajfwoj5j7bhtxnqsuyrdnyg42yb","575":"g457cbb5nvgclogzvb7zewov5w0o9sx3ceojw7arsj2hcspt1j4kulfp6q46o0tnb9f97yuerik428ndgm5b08dmac10qty2p8qm","576":"vxq9e7uj5g1lqazi0e2wqnmidv031cduz5thacgutvb63ylatz92xktkn6v7t3041cqvi80wpr90dtli3jb3bwzhj3148w0mizki","577":"wdk8axflbh9q6y7hvbw5k2iau4yplz9yqlnofv4p613ah577wa83sccb87kjyydwihvawkcb4t20n2cvhmd11uu5kun0fqa1hzri","578":"nhhgxx10mhl7zbonsshj0gnvcka8ameotlpk97wpjtkehoajookg12c6lov8h838k0ethenn9fgxigy9u3nm5a0vzgl54cnp8ft7","579":"pb8ziyl3s3n2s4160tem2yw7vziwi8u1dt9u9nph5u4aihg9bkge48lh79es4taq9sdzymzq8zphso8u149rpmmdqjleykqtk311","580":"s67h6b2226kffticji2a42wbj0zd57c8gromyg8rid0kc2jqdxlq3ri4uscxsrbi5xmr26h4h8q6xurfqckwd7a2seb42tcj81x2","581":"vuyo5088uh4czhi2y4hnztcspu7rmf4vqpqjrpadoi7pfcguhf32afmmv0pfdo2orgkkkqmiahvip321p1xzxedcq031k2fijngw","582":"9tbm3wi1pb1pbjr9ze31f9l076u7ktrhkofvnt2zfvz4k2sov2ocwzgqrwcd0964fjp3t7rpc29jhez1h8jab9e1l45neol7unzt","583":"kfaxi60xf8v8tjxvjqbm0wvxquub7hppq9z0dwmofed7al1vhckum53aa44hfvj1lz9ufqupcjk6a05tntfnmx5ni8bvid4a7ek2","584":"3j99cevtenbtrrjwhp9jcwn7hky1nfkufoa846dkvyr19r7rgxlo82jp3f32xzmza62egirnvnxv65vtb1nw3k1fivhmmuo683zn","585":"lqkdt05ni89exc9kpvlfsyh556cqlc3om0znx30yyt4b7lr4pyua0couw3uyd2mriwy9esombjhx0p7tplguv56n2k00d2usx9uy","586":"t5ojv54lol8evfz8bmzdunfjiz136hdqj2mg1x6oofu6i2zfm6aneijknd6cixhlmrt3hmasw96s8fa5lupky8lp0ckoc5mecb2d","587":"1ucva79elx45xmskwwfe3gg9bycmdtrb6mvuujzgu6ay6rten5ydvrrohuc8lj35o7tqysrsli7olmi7yg1comd3zidurs7r4qhp","588":"jmvu46j18k1vkx5kak90mhyk7pc2yhtw4vi61vcbz6vhaj053tj5ytu4jg3ltobh515ws6pjgglcshxcyz1ht6apynwo3sehs2tm","589":"df3r1j2e4bru8erj5pijyk9kx7kzvphw8mbkajbuejao8otpc52p5h139rje67wo6qlbyxw8mfd7sua645g1f3c2n6jfyejxgrp4","590":"t7yyzvs0byrtocvj4j9v19ughdnbtm2mh1frsaucwelxld1lexh50l2nrqnayaru4wcppg7g0sn186fwxz9ja40ftijwjct57psr","591":"cjqxdkn0vys64gbp8w0ru2j88m3mr05cxneru8fiqd2cukfusmpft498jn53f9xw6jvcwzjbn9mryvvvlw208vc80048pxj02uz4","592":"d15xzrnedsztz7bdocxsbuc80ffz2qxuoiaol59s4crruis9gvzdffgrldrerqkhppoadb7123kles099ccrdulhokwfvyvdysfz","593":"24fkkx16grl8y71uof0m09064hb577uto6ai3ubnxh4aq6ah8a1rr5s3v9w3fpnfu16aifsi3y7n3re7hrta9m5pb98xvbnobmv3","594":"81g2m74f2wkvmi55wvkfnzjj69c3kukqu6nhmvtigkhtzvumwz6jurdcogdzplizx4ym1isidoykrc6mlj9d2xc4i6gyocajtdow","595":"zkc14oey5zks7xjm16ha2pvvf0ufodgh3ar6n15a3brwz8g61nhivzis1e6kgh9jkjykyjm08ob1reurn4xfmy8ftq7wwlnr8wcr","596":"25uxl49ikla5ne1k1g1ge7zvr4xrev8s716702f72s9vid3m6uyhejsoch8cjm5fg6lom32u8yuq3iyu6u6oq8yngz0xeyj4x242","597":"orcpctxa7vep9dib6chsih4cj45tdt1u0u24mn8kwnd72zxno5c8auttvkpdahewm8c744h3cszd5mc8mljpuxo8rkr1l7wht9f2","598":"3dypzbkaw8g6wczjdd2cqi2tp22kd05etoa7wqpnk6nhj66h9wdp368mi51kyqk9af7xmr0whmaasailajuz5kdjmgaut0cic9dx","599":"yn77zo7fbys1h7ubyyyut5fmtucoiramymp9dwhd29xnilq2azk1c8rfalehsedinaydl9mc1r711pzu8bq4nlds7xsk7ogl0of7","600":"b2lr8j5jwu4z6xex7pgu89vrkrn0b3gn2nvp5t334d4ywte5cxztxrloxdz2kbmyl6mjyjn6jx5wjd9dfbsmi258gklv2if00rjj","601":"640c9b026sksik8sqq56teo5u5b01pv2vt1e9ovg610milc9yya7wo7vi5siqzhyarwf6h4cstev417nf7b4ybb6izo0dserp0k0","602":"9a33guewt05rsq0y2jbcm045f7hfjsd8clykf4wgsf3gamsw22v8dhieg4397ezo3efqziqxvckvi4ejsueao3m4ppc3tlhbwvn6","603":"9se6ym6l57l8x3scl6za8frz3bkwdcp4be4llb5t7u1ab3pnqoggiqor3uwg9g66zo5el1stwhp0f1cd9455zsrcugczzni46jcm","604":"obtwm5kgicg1snerg4ltgqsazv1w147bqey6b08v6b7bc7e0o58tioiaculhoje7efz5qsg7d78y42c30t07rbvo6qan2jfqii2g","605":"hru7be7oc9txcctlpn6h8as315othdv45yviegxpjctfv2c3j91n74bkkfdhab3af1pyf47o9l1hj9rmm4pf19gydv5731ru0ryf","606":"cijw3grcb45k2s2lr4e5a6kslqtb3ayjx3x71p5kfgfmmy4jxny0gvioa8eabsq8e9kk99zqpmeep0svsaxfc317as5nd5lxx0fx","607":"5bme54hwonmslq7nmw3uyj4mvvcx11o486jf3bu6s10i0h3iw7fj4yec0kvv7ndebwqo23d9kz95l5p2gv63z7n1j9vhpk1on0tv","608":"nq4477hrlriuuk82d9r6csb75y91h4yjrfcs99mrkpge43az9f6fzuc2j9shm152fwoo5tn38unu8kmmlox2kav9f38jtfvdunrf","609":"11lfjf85mfu324mzi37kk7fj4obhecoa34fdiki0ygrldni2soojg33rqdwfg7ob0iar6d98llm0tkii9ze18mg7bsgle9ewtqlr","610":"jknv7sxlug03sxfzsx94d55xdvu21kax9vgouyq7jcm7sxmtcdcxskyrm3qryztwc85vdsxjukdfx4ow6plt6v9t2zswgvg0a6nq","611":"uzwcpav2830vejqr3p7pg6w918oxeqo9db7utfkh3daq9xs4x9e0trwiy6m3oucsevtnxgbwu4uwg6kdi2hon57weixmnd50yzhw","612":"n1obb1aqus0gnc9dorkezpuhots0m6tjo8jlr1dxbpnjhqhfk0n2u5ym9akvopm9z9cpxe66icpuhyqp3hwtvtr8i8bitlv6pz6e","613":"n4jz8lleek623cy42aapej5v1vxgxt2t7pdqkmeoiqb8gpuuib53sf9mpfwhjvbzu06kb4z49ioprijka4k86okxpbvpu1mlkfzp","614":"y72drws0dgvefbvxn3lvlmpgdsg0qew08hyracbhiy7edbphf730h633rp9l61tcxowz0r8w1nd9yrpnfoappm7e6sgl4zdcj6ho","615":"0moz3ma80euu79idw0h7si22quu4mulo2snb0wokkphxnbjpmhp4ce77cjxbpo4j0jo5pcou6j8l76qol3gy1i7bz314b1ly5rh7","616":"y9az99n9wdyvyqs15yjl88x0d7m76oqjngbvgzt9h8lau0yk4tbd0ivh1zppy9knxcf8ngnojdgscxvizduuzr1jkl4jtbdjbi38","617":"gjzm1kc55e2scbw3oq73j8a5gts2lhutyzeuty7905lume42xs874ubx0pbabsq6yels1dn6xr0pyn8cisfloxcfeyx6z1pfhzy9","618":"f49dpkx5pux12s82zp2xmxv2jbykt3mx1nc63r98mvxn6pg6pvzvvylim0wpartj76qlj1o1dy9di2zue01sw55i5ru5nop6lj1c","619":"rg81dbxaa6gk38qd52bjcflfkkl54x9q73ghi9v9jtk9siir3oep6jcbthkvhzm61p0ixvnlgegpo6c3oe25ei0p0qrizqix947k","620":"tf6a6swl04aqmz5tfqv39hkw6kji0qn438hfxlyefwufu13gw1a5za3daiuzw1772t0sw6nb3f6qoffyk6u9oozo3ds9393cvno7","621":"2hqr1914t51hzd87ehd93v476lyf7cl45h7x3rs1izrtyj93ipchy7o8uwwcsswet64rsldqyvlczr0ntmvcnht7u9dscya0dnrh","622":"qd03bl06dkloyrixl8deu2fk2d45w0l8d8z66o7fei83n43r3md50obpekhq994zzrd43hq1tnnyqm5ylmep4eexwnsjuhowvzou","623":"vc3fg7a1188z6v3ww5x8dyx21kcfvk9373bs5mtfkmiputxhc7lrn00j3ovf09mzgwpyrhfzh8667328efw8t0d5e82scag1yvtb","624":"2omdag7g3oz2b7lzul10ia74shxmhgoxej21dmb4tttj48cew816ps0djmeffw0fm25b6x2rhghculj45648ikv983dbgiqxm0j3","625":"qob3e11qgd6d4gtvs6fma3oahlcnf3hsdoezdzw9k81ownc7xfq7495ejshblcuostdlkp84xwcorh8y5kayacraypyxnb6kmv5k","626":"wpt95dkbn3ihli6w9bwki3nb6z9jmu5pvao4tdbqzrnt1jmau3y8nxgcae91k2k8h9hi4mfgh6qq7lmebyqx3gv23of0plazai5r","627":"y40ykeb3a2qqkuy419xzpdkseun1mcv9sbj8mkj0aje3xthuhk0rnsa6vbv0wf3nss5zwnv5q66leu7he13vjsf0nz8sk3zwwbky","628":"74akmwshsnhk5qy6h7hzxo4dhksaq6cgsdr9xgsw878g1qrfuftitt46ez7c0260y1v538v0ti1227lqapq2fhzs2bgkwk3hahd5","629":"mmlvr992tngfmyz55nxllr2dz2kjvcae45qd4pbu76hbw64yv6buxphll839u9zzgkbz2i3arn19wjjdvznwobqjz02j0m0calii","630":"e6gv2839oq6xav0msw4uqo31hx2y38ktcnvi9y65jp84mryzua9bmyf0r6p1m3g78wto2h577ckiondzmzog4sk1bzk2jhl4tmec","631":"yikes3lg3skninontl1asq7qgzq7q4x517nkbk8yz3ivwe91naom0s2r2cnu339yjl388l8p12kuexwo123g20lowqv3uvpbtad5","632":"q64iua3ni2xh24kix7qt1c9x755duvaqdzgulxfraxt7dr2a3fyaggr36wj2rygk7j1iv42f7xxw885cx9sgm6fb3lcef0y5blyx","633":"65a05vz2tdczaxm1mxmyodfgvozlso4ds6nel8ix2465k9bindmsck66l8xwnxwwcf9kf8k7oja13rsp02wl1xv9y503cqgmyzqh","634":"7w2by8wnbdv7e4oeru03apyw9xgi0qhfvh8m415e2gpwu5t8ysw46mc8yts8vkde891pmd4wkibb2zzr1d6ghafy9yid6aq0t17a","635":"8juoz63kpyep5dy9krqp3bce9eyclym3of38bjcu7p8yyzgrdlehh2bkau9ccwuyha1frxk6dbihyrza54p426psvz9c9prnmh71","636":"hom0k597kdb599u4hvpbpjguisbkgvtd5poephl3dxjrp4pdk68pegtuk2bpnncapojok4m0r9indnlvh0jn1kde8ngm5l8xyzbe","637":"6u3pu369d82lm5vm2m0788ebsieqtxrji1upm2x2i2rrsq0n8p2wut7ua9uativxo8om8qwkd61xp05fv7tpuilyu7jirgj23mjh","638":"ghpfhklvj2venexyzzfehfl7uv1boafhgtfu9qjhe72xrnwlmc7cucj5nzhqez32buibtriolg0n7ig26veg49lzvwa2i6436see","639":"b5hacyw6yyqq9kci64qhr9j6kh1jpk7gonesrfi1nraibmno58atn2ouy54rpeeevsomxe3vnh5ilqjvtad3pjneyqmfgr3smicp","640":"7uwku55qp2ch4adijaeda4a4t1zvvyx4vmn0x8fet6k5ah54fs1cdxwdwuvwwvfhd342osydstmpuf3lnul1mwqeqbrjsevhpxa5","641":"6xvuqyvbqxhm33vbdp5a2ck4a5z7ta8ge0x1f6k0uvovcj1r27z4fp7v1qy4aypet6uao9d4ahufwst0retuzkoehjzfm9igw114","642":"evvd4s5sk8ercz2bzudfh4lcnbtrc2cx0the6bush35gvool5hii5rlyaxxr3mdg3nq34th7hec6xajwst3m1aquouqcyk306z6y","643":"7plg5jlapotfjbfzrep2oj8w3yu0c6q65lf3up2y2j04lrsquhv9fts35cf6pjc6ulqf6ctfiyokyzwm16tzvj6jlxkqoths4k45","644":"j88rounkmji19e8hq4q6frtajld2j2yjsajdqso9uc8o7r9upztls134fwttki6zy1we9lkdkp6c8nw5p91ic4k743n2flj82yrp","645":"sdlr0p0bhkw8dripr3w0a9u1ir6s9ls2vczvwqdlmxacejvwaf1ii0p9v0o1l6zyt5o5i9wqvy801uqdcrup21wvbkaf9ce8en37","646":"6tw5q9teo4lf8b0y2uxejg3lbcjrhg4xmsqwqx4muv6m7p85z9k6qgj9kdcj1903gh8czf4josaqh5uajfb0f5lil7hcj53v8fzg","647":"xs7c4nnq4ki6u49vs6exsw32467fo72dpkj4cailfp2ij1rkxx14jxjhpxloyh7blszfr0qarlfynbgvxse2pvpv486x1ewl172y","648":"u5h3bobfj5yul6p3nyahufrabsipnzgi6sposv74j6czdo6qogypa792099zq0xeiylpv4zj9evhlu05xvp2r7k8rp037pwfksuq","649":"zifnrv81pvk7axpskzvm25yllxi4f00r4931cog489eodozy9fn4vqo272d552cdrkikq8qw69x5ffc9u8bs7546k4a3w1ad1sa3","650":"vv35ojlm795ur3x9xibcvd46v8elrmk86lcre63nz4w5kgt6wr58bxfcg0vqbxiansb2fgpuetdzekeffp93m27rpbqnkfzuwdky","651":"20ksmslgcfimd36c2rmgodax3xvltbqwf2l4284ph5tlpp1hztp6wkr38202cjwgoqwwpdv0lyxfaztxyp8p2gc4wkdbels19rpy","652":"xj7vv4mctslmtjg9369kjagqxjbezqzhr0uy85h64vil2r5m6n3ymicxicpig2qv414nwnmrz59j5pod98900ltsqhihmeznbpe3","653":"63jw4nb319f0zwlr7jpgf32pyr8qt73ce69xixeo2xs5dercsiywsz2xjwhlqqz93hw8styh6tmoz3gb8lp773xz2zpggrv21xw2","654":"cli61pnr4b3czjz3l8r2xgz7rtwqs1ksu2ypnp8q5198mtl7tye85l6sogutopikha00rdkukf6c7v11v82d6j2exqmt6leudqgc","655":"21xivq5zhzru2cah5trxljy23belj9vs76kqfht2l0025x9tsp7pjb29re09m20x9e2zh8o0gaw4iri0qpv52d0g94yc5a66rgca","656":"07zk08rryj6hwpqutzc4ot7rw3ic3zf46fs4f34lftoufle4tnhga3wkidlibf32yfuoads24o1fxagvf6f5jn9abiztyrf4lnyn","657":"itulckexla6ywtj4gg38yt2gvmiqexsvbvv7cj9l3dyt9fer8pd80kd1kqib1cq5x3t3za4p1syn8mc9hd4j1id0e6ou68hs8p1v","658":"cz6ancnclhgwfn4n02vyqgcb7h93dwvgwooiutl0lw7lg2bb8qzefbylx58wtv34whiiw70hbepxsuv2f808aei84xw2cb8ogekp","659":"lx3woxtpd1bftxcawhwt0f4nndtahpuyjxnlsq5oe9m8lb8c3c8i0g2l0grp3k9i93596px6gzhcyis3pw696w4udu7tsdn7swg7","660":"cn8656350lg8p2xeu7v74rttbqqbd5myb0w9btncb4l2z3hgc3javb7kki03ew06k3vi3dmgaa2pedvx6vg06x5km3z7bjyacefk","661":"nl9uxgp4l5gtw17r2bnyvgl30log0qkn0p73553cpezk86ypw5ke5s3mflxe59d4j8huume32svp3gvd899brc1o2hxz5u2hjir3","662":"q2wtwlcamak3xq5xctql2p54mz6msoaq47bctywvpxlylplw20rv2x551vi2o1or32y8tuf0cj4utu8729asy6lowdu4nm7lera0","663":"kxcnenwdhbhi4jiazki1e539r1d74s5kbd902djd70byofbpmwkufhusg5s0u7kfgd20ifrr8c1yl01f40980gbmysj1v1klwcwl","664":"7v5fpif9affhqrtw5ezx2cxifi01tq11k0at57x3cl7stwwjgepwfnutugevghkhx5j03gb1aa5r0ogbi60a5ynbxhgpdxcfrjmn","665":"fqphuoucyz5ggt5v4fzyd554bbau3qhmm143gdn5zh2c6vncfp0x8id966bqrkbnfz3c2elyysh9wiw078oidvtd5cmjhj7n8wu5","666":"klo5e1qznb14db23nz83f556g5xgohvmudtpm7v8vgoshlkzd7th8vv00yegg922wcgm5y7kqwtjz3jpfp7qv4w3wdqx45ceum7h","667":"h6maqxvw2w6srpf2y6u27555dux3una43c110dxj3nvy91z8k9gz8ks7tvscbep5wo05trsft9gmr5stiyqp4p3t81yohjrrfw82","668":"7tdppws7p835g7y3abh5qfwnbuoctptv0zvx1ym5o1q5mg0b223hn1jiffr10lo9f5z39m1o5sby03ha48uprk0uatp6bi79oagt","669":"hk0lvq7nfdrhqaogmt3vvtvof8vdey0hi0lg6weratm1gdri6ezfsbcdzzj3wppd95fzhrd1pwprwrkew5i6jx4mhzdiqh5oq8yf","670":"x9pyguqp2kse7c6o4oc8kmxd26c3ygz9it13ysg4r1tt5b0kua070j8ctkjtoq70dgcfde8f9hxlgubdtg1zrdc1x5axzlsmnjfb","671":"ik4q3ws2qb6yyqjgkbw62scd0js4ubkbmmtd0z21pe0mz6p9o8bp4ulj7sb08i21hw8l2j374e0dhy4soifayxa8o46p3cwyp450","672":"vhf5nee9lxynz433mhtuegtfalgixz5qa4s8q2n79c0psuelvdhky2f2vghaiyj5amvfy94rhm369n0y6ptomsewz0ay30lgazju","673":"k8963gtpph5gl1cvgxqh0m5hix60x7rsd53ltiomk7mzegqejnyzc79a7tdyrp8exxly5aormccuihws5qbla28m5qiqgxo47eky","674":"40nwz3xaisoomo2n0snmg5i4mu98ci1xbi5rg839f84n3ukxooxnxal17814wxgjy7xaj0xfp8m3rb067hbpuyx9ho8cuu5qyirb","675":"818v1ev6b7veke3ixdp0zxfpl9kvimxxstzrewrbq35601350djois7x1tonus3vu6469u05ml6pmf70g45giuu3g234fl0q2o39","676":"7bszkivkyjga6r7qjcz7tbaxhjnh7fxo2yoqml806dgucm91ff9l0s6i0k7vddv6ht6vvppbmlqum2fzguewt2vvxqazqpafpiis","677":"6jfhllspd3a3zri7t1zm2r8iyrjoz8ju8hv3g12laxh9g3pzih2n86rc4ih6qz0mpda8lvxxal9erzzofkt0szdrz4fq34fdme0d","678":"nq9gqve316v0ezpyxuktyqk6sl2zpa2b37up17x4eox6amko7hhkqb1zj5bey30jj1vs9hw0yvnubit8n7ojr08y8u4ljv5uu2yi","679":"tet10cj353rrtdk7zy1qqad53tx71lzbhzrpuda4zcxihfob6j73z98ymqlrk8u3tum42wucy1n5m0axdznjqa8qryyfvejqmnks","680":"pgu56dupxx6tmc5gwlnyhgonsf1nehsc9crsl9ixbr5s8h3dr0odg0kt5f3aowwgctbjwb2j3fqp34afbp0qlqexgdwwwnkurof6","681":"tpc5q1lxxuejd13ev95atb7iiboyit4cu4sbu6yl4noeesir5vesxmjyahqm4hqrycjx3yqsu0tixtvu8ydpkkkqbqh9eao5lp5a","682":"6al56yyqb3gr7iccd1y7xjuhx3muamp4lp22gw2i7huqevv9c8u3bvaccjtfu2spfva6d6c6lm3e8bcjfi823bsrkyug68u4ce0k","683":"m9fdev4r8r4may8rnwbjl9ucrn1qgecwab14vftegp10o1d3el2p00rxpbem5khjd3lw8r5xkubo2wuyiaksny7qcw2ue8yr1253","684":"2ijtcl21r8e3zhh5gcbee2mc24pid6xi7w2lo4909yk9q26cyoyqn2rn97wi6b4juxd4x3j75ciop5dhoe3x9txp8bx92nbv8iur","685":"orr1kr274gmwuec1p3okjajj2fudvb88xf1ysmfzlgxjzys97v3847i0up4xas9g37y6j4c89wqik1h0ihsr7cf9zdgwr8xamtj0","686":"lftqz0lsyukuayu5fzxhnzgmeqlq6a94zirox4edg3omihbfdpaia2tdud5qiziil9j5t9gpak7phbrindwdpiyt0clxle6y808a","687":"i6xjgr4lr7ijdcdl37li9nseeqxy46x9hmridlnqw3dkaeclvv1daio3qscoj3jci457z3oqd0v3zqes4aydk1bggehzr3u5rsts","688":"wkyfi3vn3aoiuvp5pk1t2xin9ut30bsslg0zw4mk0wih7f7ive2wol4fcud6wm0pv1u7xzbe04elqiw578chzvy3jlsohqoiquho","689":"8o3lomk82f1qjwx106qlfp8maybwb4php9jhp5mnoh6vg8psleznzx82qzxj19lgt967pjvnv6rw96uywwizaw5f82z5dw9d4h84","690":"zf3838f0gnld4uy2h51w4cp0nertf3nb7xc3d2n866i1u2l2h6sm0ew0vy8rjnu6e8jfeyvms6hq2ru8fuvurplt1db4oy6fz02v","691":"fljg19bdlazcookvfky5vptg7m31m1mksr05ugofs6zj9sly6khjs795a656dtc3d121hqoe2r19o2bvpjx983y1qrwwqnssxgkx","692":"b9k493mjc33goq04a134ows84kg82b04ktr65j9rtm9cy7f0m60db9zw1to2v7gie8cvjx8wg0ya7pewv97zhvtrsa06yqm581aj","693":"ml2xwkua0ggftwp9shimv83qetqf5ai2osw78939vri8pxl8in322gjv5xh7v5i0kf6mtipwejk0bhdfdytib9fw6e65fqnq9vbg","694":"qlsk6z32ivocfjsl6zjr83zj3ubsmerywke8bo8s07s821dp26oxh6q0emuupqcmncixe9z4d7va7hpcfeel88bx65451n5b80qw","695":"8rt6hwlvaetuqb1lnm4ddzabs12kmb4j183drm5uijk9upwermuq1o0c3qm5xir99ffml04gmf0dlu034yr0g7hxuko3w110pc70","696":"qr39k52ai3hcyaqe2ydr7ve6hftyawbmzb0b51ettzm7bav6ltjfose73hy24fckuzk32w8ca97maxnf9v6ssc7bdlrfuvw2hxt2","697":"ajcml4mu4r02iohpkjwjegolnxd65m2u48bfprdnsaxq19wtihdb5g97bmxsy6pzl00zirsmo8t8vx2714z1o05j21qmiwa3u5vn","698":"einwoqnzmq8voly97fv27b86dghel57cthejezfiprkt0ygjaxvpw0nbnym59uev8f2bm45pcr7mwc21w3xbd1jfqiconybjk39a","699":"iix7okk51bi74ecp98t0ta5fel0eb9fs4029khyrmwn3ewjncp9u66hcz2dwli22k9aypy5ny96vuys8gf0fli7pz6cul6wew69h","700":"2p3qhjgm05i0tsv5kpqaqmonbp7ab2wu4qidufq7ki6ecz3601sm54thrfoeprcosxa13ui9zd1n2guzdyirxystcg1a91t2gxvo","701":"td4w5ho3gj0uouqsh1v7ezb47eruowr6mhlgce0xakd3tltaj1fq7tkj8b1nvd4d78l2dnkrtmyxchneum4upze0kru37zsgvmth","702":"572fer667u0tkpk1ns7imiiowlugcwrfigjagfyzxvi0v4jqzwslva0qrsx2f2xzl5839s3mq9dmwpt1jht22bij6z3nzwj62p2d","703":"o071og6om1jkgrekzaby07hf0blw9qbp1685ehotp3y1kbw6lnyo6ilzxd01qntys77uuswvzdv3uf79w1p8w0bsqjv7lh07vuw7","704":"9bc96oqo2sxlbyngrj6ve47innipgh9ztjyhccno4iw2utujb3nxrjgte1ik4s43v8316v0jzq8iixd2wgb6dqvg9x04xwxlbm9l","705":"lqkn0w7sx5ku8zyxhyc0naozd7a3dkv1pac4cgeymqtelkh8yunof9pz5qwrsj7e9ttgg2misyns3n5mmx5xadrytrbose590d53","706":"82n2romn7925dx3d7dxjr1vvr544h40x1ve2qoz2wckc2u8fffsqhkwd21fof5y5awbqm5vva2stki8d61d5sjbve6ukqrdyon1h","707":"9jfr8ohhjmw2rdvpeozluoijobwr43sldxgntky0iikeya7uxndnrvqzedzqxbi8ut3ixh6gdq77r0sx61rhdd8r5bga5lo18tyz","708":"fvqnct1c5vfavj3f5orz81hs3e87ets1wucb1zf0gj27oogfw4p296xyjiigeo6v8mntjs7gsomkyf4y5adwb6p4s1top4cbz7u3","709":"9id7jlupso3ideg138vptrlyvszsnw6nrft7ow3mg97ijlg92c5u5bngw5pjy0veuece42nwry2rfz2zjwb0r7rdjoze0dk6egi5","710":"sl9tdzhll9unib9e5bweausz4owtv7heh12n07ni4y4wr5cjgvdybiwe5k2ikciefyzx8iip7icfbg3loxorgxpaga0ztkhbgulp","711":"5qahh9z8pp9n4s76h5x01zx9nul4isvn1sk2r52sjbjrvpawuy8fozqilee8xwxzmcptf6877yh0cjqsd9eh9znthxo59eg1f0pq","712":"8fhnw28uvhmtectlln81kt43w2j8b5lrql8i2koue6ix5drb7w03w94tuc9xkyy1jr6ktmnq0y3p2rzrvtjblsgb6b6ewk33k815","713":"fo0gn72wj7vmib3g2g8cmf28ou9r536k9cw5f04ikappxzt04fge6o5rf6foqw14d2x5vk830m3bofnx653lrc2xcbqit6479mz5","714":"wktdgtye3bx9lj8rfgpqraifjcl4xnlsgqa65p40k3pvwew678sg2rwoxjgdis6eobo3xou4oy63pnzznpcsjfdwmyozj79o0qa9","715":"2zprst4iyv7t417rfjmy2zhwbx4mh3zcn6ynjsxt0wnd6tmmo1rw56zyljx3zqh56sfao7dqs2knndf84jwi4orb8y56p95ccaau","716":"wwjvgvx4ry0f8ohvmuev27joh3xtq3ryvzjsaixrl014v7hj2b8jasp6dqwmlrq25o8rcan94bqm8t4av3yo9famz1j645g2zk71","717":"1n1199dfhbqw9ofqpsatrk5uerq4jao6r6hy9x3i83qi7wujrnycwqv56azr7y6xkziurtr72agc7k1m48ob9204xt26g9cww2i2","718":"6ak9d7yvw41rwryu623y8t5b80l31fvrtzco6oxzo408r5ol94i9zjregnljnqxcly0wer1vbr5fv0jdpbehtmxrqk0vyqxiyvbx","719":"e8z27sqmi66ezeod64qeqv69si233t7z2vu28dv9j2ux8xycegk00qqfmcmxciwhjl2dlovcxb47fv233vhqzoilg931nc6yzljb","720":"4c9xgtdpdmo0mkvfmaqabyobfhz9ie6pduqqifqojvl3f8uzoz8fs0to5i118wrv8zq0nab5w7dkicatn448oqwz8hu3mdagu4yj","721":"2fu84tyue0ekcdv1az2meh0fky4ibgfte9rii44ev22h4g2c6jjp6mekd7gylaqfrm44qh2jgfphwdhebw1bzwuuv7wzl1apbyrd","722":"xz4ojxvsw5s1zccrn4p5zt7phdhc4gvuq30a8gd78ngzkvdrj3ojwe3e7uhw9tkr32r0r7yz2p406phi75j615012oc6h2g7v07u","723":"y2y452898jdrgjv3s9z6a996qmnlkmj24r16uztbcb734h2us9hmlbploeiffxk3xcw79glhwfzrz5963kz091ipfe198d3sq7zp","724":"eldwjpbiikifq5zdjhg3d301fjow1mxhouyr8qg47iq60kwr7j26bycde33d1z5jdu0gf7oyk9ts8gdos9ef3lcin1puiewmfrg0","725":"3fr20gvd1swspit0hx2jx2xbjchoc6xe81ef5x23f7htcu1y4f0get9zh6lgfwotk4m6vov71wb3xdvo0fetslrq40ikqprdtz7l","726":"jshdc40sg1x7bkpho4myaff9sazynfxwqzvtrmkrogenarwtqo8w0goanienqfar6f9nmp8bq3pcxozzv450tqnjmlu2v2lr5324","727":"en4q96v5wckt0refklt8bcinwe9ht5z5c27ltz6hgt9mre5izqhynvtemtp9y3cfq9zkm4m2wx47uwhaixn3v1949xpq1i2clvg0","728":"nq0ztc2nwmj755gricyhl7eyrl3onxne1592awbxqgdhtd0a3v8oirbu9bz4q4zqeuso2adpgji0l2ty9mahngygr5gn2zv9qc14","729":"jc2l4r8kzhqwq0dy8qjkz7yyat9ij3k8cz8y8azacqx9ww9aybb4ykik1o5744qg855zkgih9beairhmtik9dz2x8m9wntzdzinj","730":"suduprkk0bnlzvrzxydhkg51u9ljhynasg1xth0xuhzjty5z42r905lrre4ly3ao6ax9jm1c1txsyjlnmi4arq3h1z1rr3o3wu12","731":"02pr75ptcc2lgdkdrs8hfpsswoqgkrxnhnpobhkm9y6008vabos06xj8meuzqcqyixt43c5oixbe248whs1qxzlcrl4w78xemzs7","732":"p0ixqsz1inoiv4og6nducv38fuya8paasfl15fxcejrpfo1py5sh3n8xearjoqbyo1j9ex28da9s43byd0kw75mnuu1yylr66zfe","733":"y7yo2hi6j19b4bsbgf9340bpu1l442dmuhcgtzhplvh35gwmi5jaihudt0ml1ms5ezympyofe01zqcn0eyppd8hb335iidu1m6r8","734":"q1brgdqikxhe1z0bu389v0qn931mvfs6r0i1s7yh47pyd7vhou2wqyt9r0n0hpaqhsxt1sdvxztrq1ota2ofhlc4g6ymd6q9njmn","735":"bl9u9qjfqepqweum5hwljs1284midemm2h3rvmu40ezi4n1fhw599imc0flvaq8jwc0y5ju33fcuda6dmu0x1kwuo18cm85fhisb","736":"5zzpy6kc0k1u4uwv6mtb363k0m12ylegyabfw7jhzwadjazew1wjkay24mz1p77x41eazizb2y0xafmd0opb3402mplem22ej6zy","737":"ew079okp1lmhkaxo8mytv46xetf56kw8rdyd7023pzls8cv9nilbl0d70e07jm2pxqfha5balgdccsgtj9d0gei7zc30e4oah1lt","738":"3gw1eb90h29j163y940m0aiz0dax0xwksq92h4fyyk5svb08lk4qiqask2wl5qrllldb80ilx9ta4lcg1tib8rowl8xpc0ajtg89","739":"wrdup3e7f1vz07pjt8thq4mmnr6iaeakjy1xwiz6m3lzjy3zq1pc776507pdjk0ddwzfg748am673bwumjsttahkyzvdbt4abfr9","740":"av4a447vht66s7ahti2kgvhqx77csvoqs349kkxoezrc0ejme1jiv2vpwmkl1yxhusvjtl63ya1iz5xk739fp97kid7rztmy6ddp","741":"4qs5zh5pjbjeuykqs3nosvuavofvu8j1y93m0g934k12q7kx10rodhko6lt0y9busxvkvgkpyntea1cc4xk6yugp2v2u3l09uptf","742":"tpoglelmdkvx9o3jel9jd2o64ony4s99t88r97ye1zmheeum9myya7emcwd4p61p5m6c9ravc3cs78ix8wfoz7jyexukj8qzjjrz","743":"h4qxrma6rpqyudg2ei2dqcu8pq10mx2kch37p15dnysyyc78yakapoyal8rp9ud3yn5jn73t5e10yfyuviz0p7l3yy0d6srum87c","744":"6cfg7g1vrhjqdysvizdkm75bb8afczmvthzv2ktkwqjcyyarnegbbm3h98adfefr3yas43387hfsulp8uf8jai0kx4hdiyvao22r","745":"wbj3w5qxo42au5o9egptmdemdgj5m49sd68teqqqyid6zm56sw5m9vwmw8dg3hmyskkiuo0nqozvh330mr43hfi9vyucm6bzykr1","746":"zglr0ekjdbryhnqe4ivprl78lurpbpzoa4ae7k5qgjcyn6d7lfag6o40xpezm71n25fx64bhyr09k62a81ifnwmd02k3x35epwcb","747":"nup0sp9xu411xl5ry5mevjudmwhuj4uua16gpgtekme9ap0u6dqcgn7lffgk9yy97lw02b9nruwaomcd93nwfvb9ws6isbnnywm2","748":"9mbetr8kjstsi6m15bhomkto7ha8f32bg2mc4ijv0a6cp3cls1b393hq8mnfxagqz2rytwax0mgz94dv7k86s3x1o13i2efm0n5e","749":"94801zyzm2tax4g6d3qj7pdta5etz7dikwoka2sapc9jph1ga5i8fsn2m2kkd0n7b86mzqgn7a4vmryppnv08j0i58gw6dp28q29","750":"45a0sx0bi7y1l806bhd3hpkuzoyfzo4vgv3dro1wu450gg8mdjhzdbbuzm58064mska5t3t332eum69behgqqzc6afirf45eebos","751":"8xevp087zxh6zwl46dps3ejv2r881w94hfrib7b1z6camn69hmced5r4sb7dk048g4r25934s9e6mofybq0qtzhsvzhf8d04z6vb","752":"dg1tbh9ltyzaew3yl10s0l0z2sh7nmhiq4z7scsdtsx8221ni3x9bczjseci6rn1jyf2itmoljev4gcbfzt51q0exvzxw6o0fuvq","753":"w9ya3a8lsu3xw69f1ewqqm0nffaag82wzqylaeqgtqp0ft8qjk5hv83ebytasfm7pd1wj42xf2eoh826led3ldjoh1281c7ypslp","754":"euypzvwppcfzokxfw7exi795gb4gqxmir6ev5xzt12icogf0f54p68afto9blgzs6lsjt85sd1c6tkhravy8b9p2pcu23mmilf7g","755":"39pwecodjxkjji2lj9ihq7hh2vgkwbesafgl7nb0nx1ha4ijvvtvsu5mf7wd98mlcziod11kx6qmfxpbhtja94z81wj6orcabz3y","756":"i1l0zx3qi4d9qbibfy9rskd5njuimy2zt0m4mleqbgsa0pjq1ev7qvfti435flao0kbqo9ssn72dhh7iou97xbx161jbmsm2u4r6","757":"7m9w0c4kseqibij1dkiq4jursw1p76ytizl4ywysv8t2a4o1aoc0ezmjbhx8uqn3yu7bwa7ocdu7bvc53xy2m7nh6qy345rv4pdt","758":"incxzifqswgq6x6408e4n32wh6mzpkuqhvrcvc977z23ia6ivj7yb369q8djhi7z5ow7hrvsw5xi48n8uhsue9e76ql168ywzdh3","759":"jt2x9905hkwi0enx8s91yxndec64kiqbelnrcdt1bapdfrfa0no9vzos8d2flv3y5xkz94qr9xv9tsg8vphm9ppg93yvny1rtp91","760":"we1zx8sgaiu8jq50mqgzc00dybamgva798z0bddzhprs2qv7skfz3vkrkf90avzd6vyyff4pm5rgpryaqrhkhu6s1mrpkmslmtx4","761":"nh3p7a9lybgbky7ds9v4xyo4xt1wijyt1sypce877u0fi0wgsezmc31wyzazs5pwruezip3dycoshxo5aiisuc18eoofw4zp6v4p","762":"2376lsobhftzlfob5cdc39f5udq9j1wkf2q5prohacd8648z5rujp3yr2k4xwuc4rpnt1ax1jy96kg3lz78fvdgtm4qjt3g20q7g","763":"jfctaw3vj9g7ej367c4dcfqp5ktm96x9ig2pbedidlkb3o1g4b80x85thqg5kb551j5jywd4c6rsqsktpkhmox6ud1e39b9hefnx","764":"gkc9bokz7w1hzxhkwi4wnyo3owj16odgohhs8eplyefs5ncin47b26y3z7davtosb3mw31a7fk93mrk0b30o9z437ljnmmhbgvn8","765":"xko6ga4sonz5sbd3sf5m2i61jbzauug1fl17ecouudhlf9rr27usty3urqr05500fzad06r1p3iypflemuu0qw6ax4lalcu5ei8z","766":"mg5oas0p9vg2dahdsp10l4oqbs42fhfl6ndhpizyqzttj700cgcjki9ete1ve7rt2nqk4lyrjoaq85uh8bzohbhax0mbdwh0uhm4","767":"hqfi4niis4qutppqxy320qo9037p1uolb4elbeqaqbcsvwbn18np3twnoz4flqeksxz0r673ysim0eit4qneq6vfqsw5ljd4h7xg","768":"7fa75s3vwe7rxr13jf6g4y1eyv27nakwyj9y6xg1joy0342deyoxpp3zl4cg53t6cjpbrxvy5tiey7xrguw1kr4hpxc2gdyf1x37","769":"zk5gl8gqmhos8iiswebvhsrfv9g2vt4omzi1dz92ub72pa3ojg1qa1rvsrzjncq5mmbncwpoas9stvoyjbclgryaqzgsehm0b49e","770":"lml4ugfslh5s3bl3ahsosepz6bwrs7ycxwk5di7ypfbbpy0zfe7q0ppky5luk41y1c1jxs1vwd95e1enjxu5iblyjslbkwy181go","771":"poevtlm4z6y2aavfqvmwdd3sxbhta7ip0qpfpvsra7yx5mtvyqosa348mj6fio8j5d6lo7tea2eokr4e3wfsn3ck8kyb0s0737x1","772":"uyho3h73nubsbaiu031vjuea3xv7yknovd1tbowu8pryruobhm5iylmlf4oyflgx1lvpukvhtcetjfmd7177abycj2x6swkq1ujx","773":"7zdojcimgnisxoshzrvebmqmihgip3vpswnwsey8aux8ujw271k2kcrxqbp0tsifknlr8klh60ok8h4u9efaebwgu5c3ceqbwce9","774":"hbcwjf9rdhwdmtqiainnlb4op8iu77peibhpi793kes3s6pxni9dfso3cb6e3pvm5pflmowi7eqlvqx7dm8yjtsz8uao6f76uo3p","775":"abhlgqz5770kx1yd8p1dk2ger2gfy3u24dute4q9ry65ppf38mkj5n8zevzjnx7bjmnu55p7n2vxtt0ywwkeatepnfkc6fxujvp2","776":"1ng3waserhmss5v826svy8awakvykihyxgspo8d3zhfeyyxymnhf53meme60qf0cnr5xibzs0hkn3armb0aqx3b8opvntff3n9h3","777":"ey9w9r8ycmp1jhd0l8b1dfelufqgtc325cokgmfoedo2z3cmf9odoemgf368uf0na2v1oawo0x0egyst6nnxvzaslkybriwko90d","778":"ghjgtmz1ypiwme8w301qeq1kodp2icbc90cu374p13s0sartndzyadnckud2k3x337kjyqri2lk49n1rvobc65914pfs0xt1ntw4","779":"ler97kh5q3u9hrrbb743xvv8pci76ttk2yfbg7noy6ldwplts6dgkghnirkzj2o0cc1lsftxr9a376f5ydlpsf74aijekihibnvh","780":"y8l7a3n3mn1kwgc4nlp84o2o626wnhpydjjnxxyf2eoumkiou69m2k66hxyen345togb9nrh59osjb8hvrv4zcdsab1dpsjf0n4m","781":"dlaevh8dpgaefh7wbu2l7xj1fpbq6jxg3d3tihlnaxc5jr5p8emvob3ngzac8zt3cq4wp9s5nrni1y2fjyab59zkwbpsruvqaoaa","782":"tgso0ui2pgyvkhvtzkghop3sjrt3bnstpi178h4iuoouirpbmg9fxrx1r6u2ni29z3eu77okurqjxdduotq95r6etjhto5tmnvtl","783":"ccgtcq2fy76v41gzj1fftcp60o8fgcc1wudjbyifu213ksytowc64tclq480mswc93q58zd3extgvl8qru9jbyo8jr4aji8ktdx6","784":"lbdqjwfao49mrbw59vbgv10tjzru5m743qun57wi1dnlft7bj2xkdhbm43wggl1d3dw25ek8iozkpotmbmykto84ihn1vev90ahz","785":"rccn64260sye80kjk9dxdbll6prcps8o7n0ejmyphcqkmd0jb0x4ua238izltsmfn73omjvkujo6f95pl53qo79loka2f2uaxbci","786":"nmm795kc7fj2kmbcgqs7hxdl6g6949d62wb6jbp3jed7dn4auaptqzaymdkcd7jbd6eg9mscwy045ckvg0qww6uogjpvqert51g1","787":"03lt3d61x9aaxvq6l8n3wecj713zr7ezm30eo57qjan72xwpcxu476if17ihxrmlp2xv6irrmxs69thlicpbq0vgvy4px9227tg5","788":"x509dfujlibj4xdrtwwo4hdekj2ddyg4dp3wyrztrpr13omns4fn0fhu2in0fkohpv70b4vmybb51kjn2ybx0dn1j3o3e7w62a0a","789":"n3pm74ncsxket2keqnoeu4vz5zssq9ej8beb3tzhopaaf7ihgnfj5imjtj2bha02dlnlg3i6l7y74o9udjq7p6cadp63sky13gna","790":"a2p4czm7qfgwl2zcv8i4szn2rux136810oruqafkhhr0p32hqbaijewonob16fq097xv7kfky4x2cc13x11mficug42pgaoadvi2","791":"lnm0wovofik3hs9mg2gxkyrqnomy657jw0pb7l7wsx6yl0pjaalylxxktv6d18poo7dmzpvj1zknra7ju2a5iqp4dyk4v0xj8pqc","792":"9qf3wt4tiauxdueepuoc68z3x0lgpr25mve2aj4358uma05khyv6s3w0peflueuybbkabyx9hl6qnox1tvjjbfe6s852805bkvyd","793":"atoy7iz2szkphvoolbwgxytb0eu18e0pd0w46cm04sxtah5ja4dqpbuf28z8wrjhui8gurf18rn0dh5omhezw053q5s429nzpuha","794":"pqw70og1l1oljrm4cdmivvlatkqh2c8bwzt3fx0u5lwn6iyaz46mmwywpdno0fzqeyn7yfwa6dfqtgs8mm0idax87skhi9984b4d","795":"9gy97xr2aj9qwdvfb62ewb7jgfav399k5guy49j6sl1a5mpvri6bre9bhjq4k51hul855v3lonm37bgcsancvju0xg81di8waki5","796":"riarkcly63kafa2bu5zabjxu6n7fwkk0o39rcgqjedmmqlsplsqn80ca508sabpiud5fpstmqpcigouglh3v756ovty31kr2o7rr","797":"n7rplshqivkpo5oyjjvva5kkqxk61dvw0ncj069s6q5utdxv7lry18oks4y4kze7hjxjlmll4dzei47wrl5f8bst96herbal7dt4","798":"i34ep2hghdmizd79uxkt4qkqbnkrb2vgcqggt3b0rv0vwr3trosgv84aeo8dv377zauib86yo2to80ebwo6lcxfoc7asy3qpv64d","799":"gt7hr648jkpcs4j5a9fl9lefh8m6xmuau0xfurg82xilk737eagcky8h30bq311vlctngoybdx1xynqxjbukobmhrtdwmskaina7","800":"r9t4y5f5k8lshp4qimrvl97yvlyfokr6b97mu6ydt4bnpqy31ypefsnoa7zzrh23wybxf99l4dszdbb63svgzctg5a3uu152z1io","801":"pzd8bi78huo5ur0qkomufrxjfch7ojp084lh46uou5z4u9jxa7qvcj8foogf4qsws5xomfx8aq1tggclb8gigoe54bd2huz3osxc","802":"n6d8krmmjwbmn9mtk1lzo8rj6ai76osar3azqgqftvgc87bapgrrom2smfy35o0z85d4bibgiie7awk1wg0u9aovr7pceif3n5mi","803":"esay0d6lx0wocdbrnyjduc4mmkd10n8yyouxpdoq017yw4zc5gvs682bu7mjxz5pug31ulre033843bxnjucrgl2kw9tbzv2kr6i","804":"b62hfsecwvwusec7j3jbgutfg76wfk93fy7fuli992c8d51qqfjpl6me9tvcx6vvisja2434fatjjar7g4aukdjwkls5p4e6m38j","805":"qhlds0yu9ahqypys0exg4nrrm9mu249b3y42rzfly180vazqurubn609ddfnskstuntyaho6yqgmf0bg6z5yntzidrljiwastdso","806":"9442ctiof0v5mfrcw7vz34vhfggguzr2q9xbsg33iwvkp00y9n6zm5nce5n076a3w1t5zssqtcj2eiif13j43x1y1lkzvr5dkcv1","807":"rnpg23r7givgbh84h3b79v9hby8vu77i1atdse7naes5wwbxshvnl0clqygyweqiz9ttapwfv8i7822t3d78mpgdqyx4w92k7hq8","808":"xw1wcit5vkiss4rfl4l9srhc7nfueargrkkq4feabnr6u0qfaei01xpzxbpjvl509mojqwrdlz5i756tja4zdhe1y1e1fp6dz8ev","809":"p6agafvtivwpokwmdxthtqjpuyxnuxn3qbisui6rjttvygzvs4adjhve90ldmrpuuz26l9hzoylv8szu3lvuxt5o0k1uzvki7lha","810":"nxkaxyppo2dwtbxk3ffpx64kh24dmykypssvamn3bm2rujieoiccmpkbljr6omagfkft7s58r6wvg4wcywsdt1s6yksk2egdb6dt","811":"oomuiwy53twkytk4k4pgnyfn705d4yw3oil2d3dwdupo5hosjk99wt06imme258vu3488en755cahdhoz1s40dk11jriv6cdukd5","812":"lnj1sp3b92yb1tkwjh02so2au1xh9s9dnt8wlvx90joec5pd05rkp8drizaf4q4o0ttvrmsdvo7a4k6pyxgkqkliu6fuakpwh9qd","813":"13piog5a9ccdwys95hai0i45zpi4hba4jayluq3nberlgx4efhcup6t5xuam4y4dum4968t06djbq7ac790wr1ru4u6u1yoq9wn8","814":"o929wyn62ph7kuy31swozryb8aquvhihjh87mxqr9gpwnfjrc5ctkddm4vj2lh4ilik4sot5xaz9byzgyfbynlek3pqasgci7gma","815":"46787p8gh0j6rx0j8giaorybayi9ch1mocgsfrssi5hf6dhh7e7ibfsdc5h9ljuaugklnh6okz363rcvl8v7n8sz5etlzfklzz19","816":"2iuahayddlwvytjglu7gvgeclck09hozfigxrtgd7nt9gkpfjrtnj3d3e82a69yn7ccucgkg4xva8s5rvt855pfqcbl653yw0zze","817":"58us0l1y0ffklubr3lfg1yl2v9hfri4pvbqw0d3f7n163duuxljf3ys0rwmvvzbrzfmal3gmc8u5trfob7sy5jqhjlcqxaywq276","818":"17b0o7gx0c62paf33bkcyv23rtwfnbvhr4oghc0mx5s477wb21nhjuxelgfp5ocj7q9t5dsrpjxb2y7a117q3yqan6jzwmlg6fel","819":"uvv1mg1giuuybg6erauat1pp06mw9p4aodf4oyl27lbefgh8qow63e8k49bipod38c3zpb1v895oz333nike3arc22xsxpgo8zmp","820":"zwfj5c71gyxjp7heomjpmbjyqev1ykhfo1x5apdl2s6l835fjc93g507siu4movjdnuwy72rptzuawvjui800ea0drhnssb0428n","821":"svultd74e9mmrxkmd6kgapgo1ffpfc667lrmv3lwyhxy1y1pnlfkvqwoxe1gtqjmoz0s9eqc6j3gbx6729ygiawlyd5s7ix8j6r4","822":"akof52i2c3mg6eafhgmpp4sl3ty43gvkyilxkyynui8wwe074fvckbnpku2z7y2w2jq2h4bw5g7sm7n4ydy8eqxk7x3s6ti3aw1h","823":"eqou1m7nnx48er20xgtnvi7icz58f7i795gc2ia58ejx06sl6gwci2h38gtxx1z4srm4myiklakeg903921hag8n2qij3cjblx7n","824":"wh0gur5jlaoa615e6uz1mxoqjmhlymdhr56w6c5iaoiq985wv4k76kmuoq5pr44bg50affkepd9fuzx37atgj4tqrsfv6mx6ujc6","825":"coeniqnia55guj815rvw700cymx5wws3hsc5rvsg1giq5yydaf3yfkynemlycz7u8dnjhugjrgpc3ws2jg5zy87srre6xju1mer3","826":"vkskl21xwyhw0ssnpknclyn50gcwusl06049quikckn0c9001knol2d5vylrzgxc4iuy9tdj3rq7vez8v0o2x78rhtn1q4ylmo5s","827":"0dzvd005xdqumgbezs9x4as2acqjlt3pvro6sx922iwj25ti7bt6e2q87a5840t2kt31mrz6go341aym9x3y4uqv1rxqat6c1l3z","828":"ob84s8u2u1irfg529caqsi3bqd43nbbwpmae4ha2776telgd3nq4r1qkvvot63sxm4147ym7fu4q6hsuj59aror7t3k6ck72285f","829":"6znboph7a6xfbwdugjygsun679grjt8qs3zn8xrkgosp5g3e96fi47mx9fkftkgx7nc2pvlkjiyu6zp7e1psf9xz5ko59uwxovmm","830":"d114pwxo3zezexm39kdamriiqni42y8zi24uyrgedh6a3nhsva9zbfrb4bzi9vynzm5o22szb3ewgyxfrw0f61apcvlmmx3131di","831":"kiajxo9xsw5hjun3gn4s9ct3e57hvyjrnydlg2x685h0w2wdjp10e92b4xdj355kug6ezjhtxk606dx78k8rf8vpurp9x0cmtxkc","832":"88vvvksw4vl382dwj79zo8dmaotbcdd5i9ygb3e311hmhhkn33om6jnlvmk0q3bfuui1sjk1rf1mqzlklg3od0gn3rlwco8w9ez3","833":"m07y7ij6vvveq5f2ucvlq17990musn8uql07pnkvnkg5v6cpag5dcr95s7l1h1qp91rwjk22m9iatt1lrnsv919i779kbvg03odp","834":"qq4louzyny16ii2vblctw9rkny8mwufwzfg64eb7t0h7a89gfnwxryumq26f9t13xnmpjbkryoba7nhz1fkv2wsk9yj5rxx53n6p","835":"himlzxzr8g6r36c622qdl0x4i54y78sue216vchcw59i270zuaxqfgu1s88izbue3scrrku0kso7kplxf94llgpcgoxcr9m9csn9","836":"n4dsxsunnoevgzevtohu9uf1duabp1bnjwcsmb74e3c08o6lx7jsjg4eibh0mg89zc926uhif1h1znghh0tx5midilgrufvbyo1f","837":"lwplxk6ywe0gip9dy8mdqe8e3gxy8o1uwdzd4ts0k42kwuukle4q9wt1464u80fuaumjmbp06v7usomk16btuzszjzk98iwmlwn5","838":"g1ir5vappwoaczutzzm99t8uc6xzvsd5bjlnc7hnp4xknie1iro1ogtp0n2fhcsq64yr0bd1vdh2alkua0bhb91m3qxi0mdoentw","839":"kunx9jraioxuplpixcwtbuu1kw82kqt68w0s6ic41ujm9p22fxulqx4uguzo7ewecl6wk15ct57nc3wxy2hhfx7oihs383qm39d5","840":"snbbd2kugpjz5bl8r22769k45fqc9yizh2tjbdp3ra0oknn3x2kn8yc9a9b1qxp6t5tylc2msvbhtpgb4glg7pphfqxfwpimnhdv","841":"4ytirj87se6igc9zyn7klfl6k9gsiscke5c2pthus2dilaw6um20qfmbqi7oygydvr4oxrv65goe8gvcv2f04aotx20axmd4aapj","842":"6v5f0lwu247lhfybq6ms12clb98ti1zs4n8f0mz30hes7tubnna14ei9v7ri5bxb06k2u2bgzchwvl0w8ndshp883xfxwans38uy","843":"m9nijsaak8t8m96hnrfxau2851e0l6iqwiavydu6c1uylalqvuz09c315kn3gxde3y2rfiuunrmx0tv11amt5afsqg847hjmv0yw","844":"kwcogkstmcplgwblacm7j7thldipxq89dyis1r81lgxtuipgwpyy6ts85n6x5x8oc1kyxopjmq43u9ekxzhrw4vianbabfd4j14e","845":"ry6j9fwgq63ldpcmj1akbvq55bx9jxjjz3x4weggctrzksn1icc9klo10lzoqmgnzinkrep30ncm32fcf6pr2agf79wmav8j4hqw","846":"pthsvp36sklij1pmifyhetwwu6by6xykrg8dptznmesywy75b11cm52tbzf1c1cikks3m5d1kodzyrn6y5e0pi5q5au47pde02dp","847":"06ahzgfkdv5fz82l9spy87lkwicuc4h7c0m8yi5vqt001ur16v79w2b54u2cxu8dpro1uvk1xq8xzhh82eiwcq7me4sildsks3jn","848":"m4cibzqdvauyrblmb9mkg9pfto4n0s7iobh2vfw6l9anvvtath6gg3sqbw8wbkjfolqrr689ls2voisnpiqstspg7d7rr7znnb3m","849":"gjwr5si0tsyk49wp6km5y725h439tu11c2p7xffk70mutdbv8q45c2v9eok7zs4tvzfhflco84rv9fdykgq8txm1pmb442thtanl","850":"cvx36okhq6xiydnuvgh95rfxa6l4j83lpczbpnefipn7b4vi43y9jlw722gnsgt0unan7a07szylcgvlabvf39rtocgm9w2teesu","851":"me3p7rdd70kgus5r9alkkhgp5dlv3g222yyrt93uyiucgo0j214z8th7wl6qksxr1v2arx5ikpvosi3cn349mknkfr8i7gnglvui","852":"6xnzz7ztk3sgh27r6emq0l5a7gdftzjveez3feu4e566i3alqfl2bdc8c0385k6rms9r7e0tnn1h43drsekh0ttig7syws4xoqcy","853":"3ejael1wi815e31egsmwhgyxg3r2wgjk5j3oadd5s226oby6txctp5tlzpe9ke38t7qyodkpu0ml9fet323cbd4a92zpt6bdzkuz","854":"hvq4svzlqd1fi4fpy4pfbsrhrte9couwt7v37l83np2ppz414zs21401zondm6kqkkvvdsadb1ns1dkvtil7rspxc1jej2jmcq9f","855":"khxxpcrkul9dnxl4xxqq20oii2nq6pmo9tqv3i7eq9ystfgx5962gw010ml36kl7x1w7am4cup2nq79ahqbizhwo1pnbwut8retb","856":"i8vgb3yqieortrf29q9r17rngerjjlyi118x7bilghh728qtmq6cxlobh06j930a6pgjog51kq6we64b56enojj9djhef9fm0flo","857":"ynhxebwj7kcckdj9k6st9obcldn20wtdyx7pvjphjvtqj38w8q7j3b6gd4brzy6tjlr6v98npyzmbamxrguh3mpm4n3r0f0rocmy","858":"6y4iycjj0bo26hesytgd1b1fv91n2y1jeabyurp82fszqpfot564e9wdsg00q9wesk90131fueyre3lxdgf9i2xn5h0ba98t4d1z","859":"dxk17hqwd9ji9owy8whpepyw0sn3f37h1p76g6vk2xzsu2kzj2ndwv5lqr7xb3epbph0jf40tu2jnxe2rzriebzh608dcoueqcd5","860":"sm5579b4y4ud04rmfkanux9sye42pzlk2xq5jfsc9rqjhh8kqgahmaf618trnoqt5dgb8hqxju9d5ttp05xjfy2i4s08uvhpx0gc","861":"v36lao80kluattrof08zsyr65clz6t0dzusxrqvk4oyjj8wmm7ftv432ekjk49itaaoqh0m9plb7isf5rkcrpdebprua774aqvno","862":"uugd25s4aklui7bfg3o49groa0makc25kvahy84h3jfocsw95wnpmjlsoh57r9rudeyilrz8bwv3jsir910xzg6wbk45bv65nnrk","863":"hgn06oi1clcqyps7bnqmr7jrchohzifbro6lazm0loh1qje56po2vt0ub5nmqvoewtx7bu2xs2nyyaatn67tegyq0hb9739njxs1","864":"2jqxcfxw346gyu0uihs799vl4v45joimyvnl1z9gzmslxtz0luhy6hppiydshme837f4kr2sp3yreco3xb987grskvj36z0efnky","865":"whi07uwtr34z934n5r6xnkl6bln4rrdfnv3xoabu8g9x66u81noydr4r0r1rakuwpu59rn6axkro1umb7rs4c1n14w2di7umqhuu","866":"imy4w7pvtkrqe5r3kjggk0t6faunbk4k440nef70ihz8qb8qm28c4banqztnk1n9ce4xvhrnhevix8ekqhyat8b4ryazk7f18ztv","867":"ons4nonjdptwy1deuaj2na5yte9o4zjrh8f0k06uhb5gj0da1i10rm2kdzc0sw42x8itol82vo96smx9sbwx8br07pm8ot5zvtz1","868":"c393ku3gjyo77z0vmuqstftybu3dtrb0fhf1mo7h34dmmpe1n0gjll8czqwpo9o41j4dtgjfnd6bmai5jrgakupwwiqa6b3zw66q","869":"n568l81mzziz7szzznq0xwrpkizwlmuvh7upkr0kb9u7q4b2q1b3ha39mbxkpn47igo8idp0ij0iml1n5bxy0cpmkm8eaut30udl","870":"ogtpoatowrhmy0123j4f5ks9c3f9ofwcl9slxg1uvdwrq216xwmwhlwzmuxjsu3ygs0yer5d0tbw0nkat9047xa0ziqvddxhebhw","871":"ko76qgjb4dvpa12zmaku8859zmm4fdgq8owtmtuc73v2qmqm2u5nd6purwzki9atvd0t1xr6my67h8pnjlczeur99rzwc8rw7jzy","872":"7coyxgd7ip4mk0qt5vfrkpx3lzt9l7gsf00p10zufe4pflgp5qyzruiyek1955j0tiuwstqyb35m44rraqu2wbfmvfzkf7wrma2y","873":"4u45z67gv5g1aypwn3rkrrftx8soq651o4wo7751jcycp65nfc8136ycp0nm44w3lc2rn8ek4u1so7wtthwq21agxbitv238je4m","874":"cpxli7myffi6fnhelh01ie7mxvbe0an3coxtgne40p7tb8jjfpxx1g0s1fwi66yjfx13buhi1pgupu93hn15njmtllxia7ovqwej","875":"09crkdydost7feju6p9spdbaa7viri4xc3aqqu2ewmdhv6o9nhu0m2ubhrissdmnh8rcvick9w6hsj90fg2bdi6qdm1rkxq88yor","876":"kiuqrufiftkoxjn7vm67c6hsrp5rblv58cfhvc6554a4vgped5kvz4r7hh717oy6r7o2d3lnmc0yxq451qsot8b25pukqn4yspb0","877":"vswxed30ybe7zdqs902zd2t1dzzscgbdtzjp40hmdhjav1zrkm12h3n25tfr2yok3mpscc3czur83plwmvevrbygaj1ymypv2hln","878":"xk81x5393cpshr4ecwcqycm8hkox76u8q8hs30l6wc40rc3i4bw6pgvfota77ab9c96ol3pab21v8khgnv1myzhj6tmu3l4871yx","879":"y15236vavpu8hz5wd13yfb503h3rqi2p7ycbcvlo4y3obrfx1iwl3lgpk0ihag4n0murx6nsvqzpbuhfhqxcouu686crhles6tdy","880":"rqcy5yuwnch7tcjgxchefx77rsbuelq48jlrvwenmp5xxgx1cb749757bwj7y0cfhpq6hw87b6eytonysujwc6xda6xsa6hxevd2","881":"j4t9pjpps4w16cr49xla9asklu2hb7l749xiwqkrkfyv8ri8r7msz7o5r1ikgrj0ss2k4i96s876cwjq5xnld2mgwbi6cuctlxd6","882":"19goj0bo8nhnj96k3k3tuo3h7f2z9528u1ksub7g1v2tw1rgwvaly8ks1eoqoc297s6gezsxsdl7fgpmppojz7elpf9s9d5eor2t","883":"rq5o6kk7skv4f7kpdqy9uzet7deep3ww1v13tzz2f4i67bq2ox90rwevmyw3wo9gfwypd6lrz27vs5is9mu851zr3lre789xpqgq","884":"ejqudxfcg0ajg6ww1bi88ggrod3iaf0iz9mxfgny73wltv00t5vvea75r2oj2ngbkablj3tcu89iunxrt2pix8buu21pdo755rxi","885":"o2o3d89whis4vchfa8hy3s8iaoefqxu5hm6lw8i2dl8o6m2dk8elr0rgtaa7m4lwb70fza4fe5k13k5qzvtbzd5s2f0zefqca6p8","886":"1u5twzo6innl8xa0j9ydphy722lll4r3fo22cewmlr40t18c7hvsfgweqa3a31gx5gk6g22ohh1l5ly5nzq1bg3t965luvutiyby","887":"4jw7cjn8yzrjs6iha96m0grk9g62pprjsxn8bcccl6cw5njh8p3tn91iuu8smp99qpccv052wao6wpgd94ezf56uki37bx5y95r9","888":"hyxoclybv9ku4lr54xlmblccsow98428hdxduzbwkm5zxd6231aaltupuzt86g6aokehc5ehdk2imqroa3164otcbs39bmvm4mzw","889":"iggd0tc62pamn9zda4nvo91z3y53mfdr1j2i8wa70gzo6i86a4747lwb69q9oxsp3jphkbyf0mo0txcj9lwi6ekio9709s7kxhjq","890":"ond4cocj1fogu9vggviqs4b1dq6i6fm0qkwjknej5z1vobkck6woufgxei9ikgu3wjgpvw32zm1izkgju5aiguof0jr67l6w2bk0","891":"zbzhjykvjapn0ffs5eipunwljwpkgaqqgq1g01hiaokq0d7i4jv4b6heo0l5v3fynddlkcbdb0kpb95rkti3dr5rd0sk56wuejrb","892":"dg1lfws13pbh9pr2dxk7y1e19twtk2wapohvf1ng2htykouvi1xs4nud2l19nf75d8i79sar0v53ir4kma6diwtnulf43z1q7z4r","893":"78frf5felszq1psvzztahg1cu1nq47uqhdgq6azoh4r89m97ylsh7hzxbrngf0yq4s3ozylioopdtt69oqwgsq1os4vxo75a27vo","894":"yyq9gjcfo6hdaxfhz875flrbqag43yetrntta35c76mw9wf4zn99wnit47qcbvu39h6ofqwf8q98ooyrqa3a0ehgw728ux68950r","895":"b760qrebtyhbnlib77xw8b0ic8q5j4r78t4fytlhoycm4j64becaxvvrx93ofp9bik9vl369sd3j2sv9t2w3aawcoa9rcyhetcei","896":"q00r0ttfhpn229b3k1zwdhefpgwghkoftjlt895z7vfwegdme8ja63bfqcyyxiuioxvi2d38k7duob3thk3guo6v97c0j1i03od7","897":"70g9jpcg5mtu3hsyp5gcxv9ilurdsduwcxmnlv1ro371uujzprk9104qwryuocceuwn1lig90e0azxfdldzi95wi2ks3llp78syj","898":"mu2rfxw20ucev4o4vb0vw3txrabzkr6d9768pz418aggb8gv6j87ztcpncr9rkn5qz1ib81mjkfxac6y3nj9eyap2x0b8gqhazdj","899":"7fkr5rdktjbi2n6xzlg9pplz3gt3lhjw9ivu3txz7p5fm9p7vwkatjbt912kag9zyiab8lng825k8gntvqkh2dsy7vjhssq4fqhg","900":"5zxju06dovr2qijtosykn73h3kk74191hojx71lj2cyfy812pnwjsgoy74148i8n7gp9oyhiezismb2k8rs8qs840maoke4yf8yf","901":"n33p5l101r9e67pzbo06t76c34s53nma1qezbujbq5moum0ek1tarrm9zr6ydbbpkyen4w4ypjciurbic6h82qxedt6nklz4t70v","902":"dlngm279ftcrbrppzjekyiioqhn7qlwhtf28ot766h43jp89o5538r8exaacnxd10xvh8v9vpg93bbl6gnw8ac5nm4aueokeiccg","903":"7nzghcsqra0b9peetec1ruta0bmopm3157bniydmayrs1e2ejp2ks1svdnxza9tqujr4wz85683q9iq1i6vmqftje43tk036v42s","904":"l177h7hv19i2rsemlur2ld2t14er4zx6qbcwjbv0io8q7qnbk37cosw1044h7jz54g23bbx8h8a6ecglcxlfn0rmddzdyw7smrx6","905":"ckjfhe16jpg5tw6ty13f1qso74vresx9yvqilhj5o6r0sb689h2a4qa72b7vqepycbkql3bs23lbugmkr5zstxikln7y6xingj9i","906":"su580jawbbtdova0qeis3k00r427yuuwuso9k35yhxyhhcsrmqxdhoeu86ic6jwfx1d7e5ln0l6194edg0vze8pxegw9ylsngcpx","907":"x54i314lpyn9ysssttivm5lzriodluke5bjflvcrb9ec8q8ikfd5xlmoae2bfnzme1l1vitwhbwjmzv8p9j0iwrb337fwxqbkez3","908":"ql21hpxjdkchh17c0tls3kfn9d4mq3f4oajtb5u7c0uf2qu4y6o2clvni4zf6l1ctgwynhwceldoxoouxd78940sd3dsn6gx82a8","909":"ijz14ql7jh4exjffl4eeel4ujrbecdhi8ainh110v2fn7uhtdzcz5q9azx2w8iein4chrx6c46g2bi5hp476cyx5k32v2yiq4vk6","910":"o6rnx8nkd5bwpntpcprfry26jtxjtuvry41xnyn38nc7wvgfsxwf4sbbqfaobubv3wedz8252kpnn0aq3t21ljn6z6tv3c7rl15z","911":"x8bn2mfu876wssjwigm5ff39fo1cah2uw0rpafj2qh44jc2t1ru79fmteevayaefobnrni1vk0jpit61ooq2c1lknsjf21yhlxmr","912":"oxbqfo1qk0bcye3wls83ghcjwyswiridfnqzipvyl1h9pxutu9i70m4pduvpne61f96uak69ynymb8o92u39efs5978upl406jjp","913":"oqu2nd56p8o0iqnzqlq8u0ejom9esymdpknhbb2r34hi6g1y2686km782n1qabqhkl36p6f4xabtyhts22b4ql1i9vq2hi0foge8","914":"inhjqcuoewbmo580rrr9oyyj8wgh4f8kwelszrv48jyjelzzxz5iqfcw63j5mbr11edr5jb7wtdvtch0t55cmxq1gmr8syypmj45","915":"5ogeqdj02r32tjjiko8e6933lvu4cp01vafghdz3sjo94pa5i7glmroo2my02v49exy069hkkt7pp7e1xp3gzw83fulx6mi09je6","916":"udk662j3l8nvr0k7mzn9cgbrtais8s1ak4xpfu9i4lx88cqxus5b4z5pwsztk4is061cqu8k94p5xv9ux2w98db37zrlsfjvz4lc","917":"qmlovpejm21blsyse54nthlsaif5e4s7lsysydy3ag30i1wtikeat0yjrvdr29am3c6in6uueahd905x3cl72fkddhhh8yfweivb","918":"fl7h0pgq6sox17asr9kn4b5fgx3vlko6pmbd3xiaut69j5tcbuizmbjqe8pl1aza3u3wtnvq2rnrksnzh2wh26q8dzu3hx8q7uow","919":"hwblf64j4id3rxpvv6guzupv8zaz9s8twn7vs6g9rghyn4cilrwk10mvglkt3d7fiau22oc9sj3tmr7nk85vwsd27dgrnx3jh23d","920":"2z2pvqpyo7lkb3xe3p7o0jp6co77h3647iw2yxtrizx5uug3b0ma1nf792pk634sjvb2tgkrp4ks6jyyzgglv5rlyt1i1zfu5088","921":"41j552zlvc27eeogp5yxv5lhbsq34lmx18zsejrz66qn6u5b3ihucj6mhh3jqgctt5z8xcal6yjpgutv3c0l96l1d81lwm6r1mle","922":"z6ijtmfa82gihziwo4eolzdqv7bfgabnujctmzk50a2bnbbvqte5vmjv9m16343lu6289oq27wl530740x30yzaxismdlvf22gxj","923":"jj4e6cda2c1y0z7mg4369sgv1lzhsb028ac38kj8cuekd4frng7kx34paqx8djnl37a2nnf0hny5n6g6lx4wy2sbvcs2q7i1oii7","924":"vzxf4wyhdhh6r72mwupknbc2p62yjor2iumwk6th2hzfn4avyw4w0xjh5pkgltbfvz7ayxi4ukstn7emgqhylhydz0ej6yco9qls","925":"v6li3eb1ys216a9rywkipbzt0u4c07ifsnz94jyzixkthvz24ih4c6tdsgp07epnft8kspz8v4bwlz4pb57rmturbkt4967dd5md","926":"gsvkrsv5rxem2611z7zfxvxo0bqojjcgk133vrv9m2knat67kzs5g6guo0jepxmtatdjazdu6kb954wu3filgxoxa9f3mzz1e8fc","927":"csauue74tlsmv80uu2wcbl1adsmqlde7mguktj9jyqv1s9v70ddmlf8p2vsme2okf6sjach2tb1s8hxzzweq36e2gm7wq8e6s8tb","928":"p4v1pelvojb7z1jfbsbur4x3z8fpsn5mli6m0ozmtkazt7b7s306z0uehu67fr3fugkggiydgsm86s7lkvm9dkdpgvyswo9u24u7","929":"07xoj0u96ba2jz15q2qvkuap7ksxjp2u51fwwsygvrnhiuegq73s7akbv0ziha35qjkrk268kn1okfwqqj6hwlbkc6qyf0s9y140","930":"7v0wi48w6m23qdrpu7jp9a2um4i0pcb6yn96m319iyx4bbhs6as11cvdrezy5mm8o3nndzkjffpg2gtc34uk416s6okuuj5493zd","931":"r7lpzg5lxo5jmk2fm7qgth1q11qg2wh5m4ao5c0ckv76h9vkdfevnlfqvdvksbhof71fj65vp0pmjuti68ppmu7bdqkcj44dx6gh","932":"rp8y89heyuebn40g7avh2q7rzdm6y55vl33z8c4sl4x8sm2h4qfo7kkhdyo19ddoahb32r7v4focibj3qynys98ukahj7p67lfks","933":"zvcicttq9yn2n8n4ht7gpug4mvwgicpg3w1q02hluzmwjc2zflsj3170hroaveg4dcidevsi80jc7cgwt1wz9ojsmw7j2v3zhn39","934":"yh4q8o5dn0gtj2jtqihzj6pfngrv5chth1zulxwqzg7bgz9y6jhlviw936nicegjiugea0op7at3mqlevizs5xzrqc2peysk2tet","935":"aprfep21knj1g8sr39eo5g17gw9skitze16gomoz5z7ioqu0912lhkoll5w6g4o7ilxmsaawduq2vrp8tzadf6i8jdxg7xms0nf7","936":"3wpuwet9f2toouy3d5hyqiaggj4yrje7qw2bbhfwsmc7zqu2u0kpl3m0zdh3u7rfwdmxp4poer1mp42rqjpnbddhlqyr3l1g1gp0","937":"275jycdzfpwi8lwonfgkvt2tujcp7zy6nb49vh7tpriw1hy6v36uuwp1fyxj3fwqlblmt9yzq0gtjojrrht1o6n0izkz9itpbz31","938":"b002li8hz6vcvxhkcpztfvzx8jcech060rv12248zcelijw85rzg1l0osesptwa7xiud2dn26w9wkafxep68mwn8hvfuna5ltivl","939":"1lzdqlfu9vi4ljzsiub0d25jib0la7l4odx905btb1aiool7qn7763ehpzjp9knvdsqzh9ui7n5i0hgotarhb4x2q2ciy6aqmutj","940":"vm2kiq0t2v30mvxznicbmm4soa15j3n2dc9bdcha8nsxf102w460jlprjto4k1e8cmiofmfvoym2h8xfoqf61c4vnfoz9wnxrza7","941":"nwpb1rzf8n4hg78yj5xaz1ahvl7fggs2iq9jh8iz2akkuhawbopybbit3o7gzqau72ulffm9y8dlcdods8fr8ja1v8vf3mzpmowt","942":"j5hwv7oprjqy8hp71l0f7y7eci8jdlitnozh0ihpz8emogilltd8h7qq5g4va2rpd65w4xm9vwcrpafw8erk5w8j46ibwle0umw3","943":"p7c4q2gmpz5qqs095oojezv0gxa0ge5yq70vmrjlzcrtme8e0jasjcdrkl9y1ffhlsqqvmllarw0yq01016b4qgeirls8iplhldl","944":"8vsi8lyjfaylzntpb68sez1403myrb6gvhswxb81i8pyjwx4988pra0gamv8q7133ba0anp9amtw8x76ulfsn8glsv3drzo8w7b2","945":"2mfzcvszl4fokd3q1dm7029kispx41bwaba88xdbdvrzm14qgj3zkpz8vrsl1u5lcjnc416c0rl5mhjsav1infxpu2y30xsfm9td","946":"q72icomj7mavtb32sko7f9651ounxo3dfbu5wl84p0lyeimct2pnf411uiv71n8py2mdjttoj70f3qkxzlaf4ksy4m2a6u1rrz9v","947":"upiknl09vauzl2sy9ppsichcewv4phgd5hn28lcuxsjgv1bt6w9huqu6r5hyn0hnfu8xrgycxy3f8597jjqugusbd7ht1jo3i6ej","948":"wrfvqx7vw4chewy0g7j9bs9lmkq5f1xlpfk2ve4z4onu52lgd32ymlo4bkt8t1pued2zpweuo0maoxplj73m7ezhk82shzg3cvbv","949":"sl4fcuuzu224axaqdyb3ld57f6ljjtpgrhxz0cy49tdf2lxnnfn5jsaj9rlt0vx011rvfarmi6ak00353i5c6am3v8w1d94vojtu","950":"d82ag8xaxriuu5sdgn3o7od1e4qabpssai6yu2u8ucfu3sao3owkm05j0lfwo0fcz2peidt53n949glwe83l8y470oufhyc4hv9n","951":"2k4gnpb1fr7dimp8i502mb9lkwwacq2e6mzdlj6k7wudjyr37sktfiegos5461wu64h7pl1ofr0qcp3himmph6m8qq4wb8k6anp2","952":"2g3jmo4tx6scr0h3vr2kgch1ehwfk9ptz0lvt1fkgl61dsobkajv8qhmtuhuoe2kheg4n6ldqafgphvoq2ss11et5iu8rczu9kqz","953":"1vncmql1q5lx0yo87wda1sgz0c7h2032uuy62ohmb92waqrx2ntc7x0ymkv6y6iws0bsrvq8vuzb35k4xbsp95w4l9ucnlrr7fpp","954":"azbotrm015ip8jsuffcwr2avrije935d3u2qex6dpdqy096w0p3k0e7x4t1oqjexbhac35rgnp79p7ctkcgvuavhjh8ck060i01p","955":"e2jkrd4phyc3tpnkwokjk61xf8gm4u5zhzqecwj6oci7c7dfbhbovr4ssiw2uunjx97mpzx1va67dcss6du7142oibdjb83ct15a","956":"mqev7ssxivca34cmt9tqhai0o614b6v5cnxsbe04gttl6vijxykd0pb2sde2b0y3m4jciery0fkljvtiwgx2vituxcm8pd024zq4","957":"xttkp1m9k1l64prygl22fk1ox16io69xu2fdpm8kzk1t1gecf17pis0t5ahpwfs1r7d49b4o4fsvej2eoz9ncvej8db5zsg2jx0i","958":"3ech5lhgns301tjp9aas35zmn7a45q0ly8hk3rqgcqaq983zc6dhrlc5zuer6auqfkp6krj94t9ois8b3xqop2sm8dz5cu9vgf89","959":"uv2o6pqtivwzy3y0dd5z8kap61b34xkbihsnjwmsku9zfo0ro2gbivh2mc0l38tr0iyjucn5xtzte1lhr7rxva94jexbux7l0k4h","960":"y3n4b5b9twfg8uqzo4l6fa1hmg40w92yedqke6ujl91gwu0xuq447od6c5j060mzqs17s1d2cz9rergdrp6xppgmiu8aj0kvsj8x","961":"uf8z8onb6y1yzz0s3u9db29v712zo4ui01hywiy84518b51gezzzvnjmexau0r82oul85fnujocq412rs6h7fuockah5w9crr6sv","962":"ie5qfm7u7kxmvkueza3utv7djb60pud4slxoa8kpgmnsufdahjx0wnz69byzf6kn17tilhclyt1bfa0lkqn1tor8dvgywcl6b0di","963":"olo3l5fi31xlnbkinewwgajfs4mjxhj34mxqg2t6js6hjlkqkc01riuu1hxid4d3z678fk9liudrh766h4q2ymep84w234mgxl5e","964":"8ikxr2zl1zc0ywnn3zvpe332hxryitdio0m723l2m6tozrbe8lfz5e3pn7lxztudfwtwmjb29dyg8vl1jfzfykpdcn05f45t878n","965":"97laj3w2kakt9fwdvz1vtsqixm4pflyugdhf52v0x3ovwoytyj9buzauaq368413ohegyfhnqii4ydx9yjljrqjex2xpfbb9g9be","966":"z2irnvi0vhb0gz7yz57lmjs3jun4an9laiuj7ybynefwgm81zmq3ao0qzqm7498scz0o7fkj0q8770a3x5gae8i8rqcy07j39ezw","967":"z90t6i9ontiudkjvoj7yap5ur8c3ub01lowgajdm6swdw4dun2vz6z8g6113ieqv6m4z2n51mpdd9qzcgmfccar5nxe74dvylnkv","968":"271r7419vorchfbbik0fnc2xbs0ttx73178hgbg72sc6yqtgln6dpi24qc9tync768780pr2uoedtek0fdxwki82ravh8mppli9k","969":"5vkjq6bfu76i8i9z2yh9007v0eotjqk8h3p07dc19rnkn5qrg6g9ki3ocq9n5bgt91415a44ha23y5qflxn0t86qor1bpubrkovu","970":"wq3hvlwovjulnzmk9ai4gb0qvxq7u54c401fiiacjtj0znpbywxoeejrdch4jo71s9qetbjxtec8abkfqfdm2c4t3nac5tou4uto","971":"5afwhw35f8hkw2xx61op8g0ol2v5jgrn4uwputrgi1y7j4rkuqbppo3hlxgjco8hk1sllrmth54p2nnvfr9o1btj7vxbf5s5z1uz","972":"1psfzeka0945i080m2ribwzu31a4867wu8cgeilabzx8bi0isodjj5i4syi6hdik808u7rvrt3izk2oynhx8s6416ajdtzptzbau","973":"1n7zm8qcb1fstphyc6c9zzok3ic9t8batcp3z3k2qfw5wj2xjmwsj0ihxg1z0y4fk5mewhk09z9x0olcnrv9cmqi813addm6j4l4","974":"5x97linfzhjci7hiu3jzbounj63um2v2rnbc3exbdock3qktfku2a2lu6gutdyz14k26xjmzuoh7b2bu3cg8d52eixus0c5gcmml","975":"hxv7l2b5fezs22lh73glnwo3s319mcyizc0kpipt50l7yu3kmgubr60mrd7zrszbuu875ugm1nv1ia5gs8f1fb7hislv6enu0mms","976":"x4szkh45mwdx0gawtx5jahsyt3punczvedx9tepasygbj66rvqwievl3l7xzbofhyb11t36e15ylny92algz6gd9g6fmrff2b4xv","977":"r6nbo26ftk1devjjat82zcurmyw1cyezut4x5hs68rmcb2atcje3xtmo50d9eki1qy2ckujkmmr02fsi36wxgufmnlquy1nks75u","978":"36zp0t8lxr8tqc482z49zffx7cyvzhja1jcjcuwytf5kulvgr1b0rwjnop412udorwwkqcqhsq9y6j7lrg2jis3x6ndszsvaeye8","979":"nce0461i8vfchan6vmbardg6lcau456hg158adph1194dahnglhmo05zg1ue58ih349qxn3himpkhdsqgo4sxspz2y1gbaiy9sgb","980":"elseuq2fnqvd788wurf08y7sec5meq3bbrjbc7584f2xaqyb1fcar5xwm546x5xejs3pge809zmsnmu3xin3os5dnbrgrna6wb8h","981":"1qk5cbfs4qpzfy6kdj9ss4bq4mq2446a8aav5eybwm9ejew9ibo5ne37bgf8o3tqlv18owad13zkpcv5p88614z83vzyc73xsa1m","982":"4oxuzrr3aftaf0xhhux8gkkr7awhoycb5m837grrb8wszjv11961vbpunwf20b9y97w9up8ifsuk0yxxa9ksgw479m15oda2w2kh","983":"3p342xl8ivnuq62k9rkvpcwom6wcokj8qmtnncrflqfig5x14m7exnixxf5ryv6dv927vsolqleq3blp95azgj656qizrvlpxjgb","984":"5dvr5j9z1o1cvct49uspjzwdlnet2ydp7gauj0ejp3du6z6dq7ljx4rrkvoaeq8810y8g0npur2uakr39tu0o8kvf36xlwzix224","985":"72jxwbmjtp1vfpnok82ayra3ccb6lgqqqvwujn1t2bxhp3i1gzgz8cn1j9n1bp19khyqquj5kl9vdbsranlhiyxgio2a766akxcl","986":"eroaw4hee0jilvb1r1syu4sfodddubmvhdopqf7zn13u9g4v4tles0imrkt7fhcvrfzlo3whulubcvzccm1z4h2h7w8mh7s58n9i","987":"2i78q6hbwtq5agbyl4pxvcr7vgx56yeqerugpltylr9j90uyfy42pufacn4dd85ki3i8sgd6s8t6uff3ptzs61o1kcnlghqcb9x8","988":"gpurhv5mrw0b7we9f2gan12e3evlybka3b2tg02euts86ywq3m883dal84dnggyuqcn9pidfj81aett2g18io8wlh7uxpydsbcyl","989":"o7t43sjw81jid1we8hfjf9v9pc755777wygsh0fy2aacnkxuguipr8sgpt1y67z284wm6m3oej3gc6t53xqwrn36c3i7yipc3tv7","990":"xuob6idhp7mmpkmc382bzn97nd4xu7fz5x82cp1ua267c3xylhbm5pk4wrof0wvv65lfphqmmvkvjvcywwrvarar56lm8inemsrq","991":"2yzr8wfqcnodh6awm8zfqmk4hj8l4v880amt3xuvoep2uf8bdyv9ns59b6k6zrxykv8mse4akodyy5ujnlhqiemgzmppege6rtpr","992":"8bd3u9clr4jr6se539r7ryv8quiejigat9qq0nsa96ejg616zbj25t8fixmv4kv3oowrfw4sgggqroddp5rmlw6s1akzwkyub6xk","993":"pea3ivw39ouvguusnpwcm65b68b92wwvu63w915ojfivgmbtnel49itcgo0dizg9xujvebod8llehkztowi2uu6ntpsvfe0qzh8h","994":"19wk0fkmtneg0cjojqduu8uk3l4e55y4hmpnd4r29o4urhe09rl60v90gcb25h2wrbhihtdvl7vle12igl7i4kpgh64gwczpgnji","995":"s9xttp24pkv7kg0e8tu9kbzwt2k5dylc2or9d3bwnh1nivftaudkpjp28aw46bjy31g7hnv0fblyiqmcos20aoa398s5fbk12t8j","996":"vtzqwkl7ehvn7x8k2wmjgfahjjc2yfiq52wo47ysp802fa8vmr75x7cfnbok0ev8ir0qxttwl4oi7tir49d659o4dvly1wlqvddg","997":"y55bwzpod8snrp683az73wwwt6ecvnra3uqugca4hml7ojdaf32rl7si8b04evxfurerlrnkvheuxuc5d5lhzs7484vwiof8qu8u","998":"90nzvj7u7yvn9xqeiz894mdc1lica5umaf833ia00t74scb6vo3wo1hxz6g6we7hprrix6m9oxmqm2sniyjy58zqykw2zq5u3aqi","999":"vckchrmp6rpyv6gxb4uh6gupkowxutwheafxyadnm6a8takmy6aiswcvzpnwqn1w3pwqhpj4u5ru1cwdgmgs48vmmvy9ry82fvu3","1000":"316byj1gz6u0jxl44sa6w4lxbd7sxps0fl6tf3ndtymv3ukejwejjc205xhjs7wi0kydf33rmfa27vem8s8wjxeeowh82pffqelo","1001":"1vwdfr8bvyd04hgcrn9l85a9s9affknnteb9cdgassxqu2pafxwc6ey9nlgnr4p29z6xoyxob34uzd2umgsbz8ushoverk74tdbj","1002":"2qqoluknn1sekkehllo548e193c7sm67tedpmuj9ckn8t133ysml70kv2f00io61o8wl03bcgy13ohmag1oxw3wzz8oil8irpssd","1003":"2wh70jarqyittq4vdibof4io8uq34sx4nqmfda7mpqvhgdyy2zr9u5r4wtvyi9xwv6uzqqdi6vu1gcn91q8xdu40gmnbik25t618","1004":"af6qnao0exkkg2i8uvq25z25d2wi40apz6yrrganzwvwf27lc5i0dmgabpch2vcx7jq85ol9gto11rdpow7n8auwr7neadnm91y8","1005":"mb9c1vql8ix3fbzd5iztkwta3r43q9j8g9l3n6ldbyartll1107c6yy54p30u4l4rfvdrzc6e7ank7m56vla16d02is5il80pelt","1006":"1vp2vms4h8g52ofwkdrpsrfoikikt2tz3gqk5rg54qswoi8o6its3x448sb7dop3pz4fxdpey4gfxwv0x8bk1fxuyz82glp679di","1007":"kax8ccci6m4318tqd4ail568zuesfejyz5riat3i4wazuyk86utebp2oeif4is2wcx7k7u8nfr9los82f34fp89ob02fgsae67fx","1008":"30rtqi91s7c2cfcywdgcwms73r6ujp1hp84nxoghscniye411e6ctmjw7bku9tssgyvsanuore1kr557jt4q648jr9z39wmy2oj4","1009":"d8nypgeq1i699vb3e9clt1elyuols3cbnbqiy4uc88tdvdyre424qhphysxn8sroy08utdkaoo3tto0rbv00wiphb7lav9lytxa9","1010":"q21maxhgzkoju0ltmlvngci5myt758ayyk0n6hpflkd3p6cbfhvc1gwaypbiby29l6emxwzhl3fextqfpkaygi4ke5xy06nsgprn","1011":"57n4v8bx4xyxb7tulsq5p5cw7j90vu4856fs3cbp640czpx6t1whb9mlj72do85zl7hk1a46ymcwa1wsu90vb8fxnuki9g2tkty6","1012":"42m4mbk8zdn4o2j8fa9w289srb2dvoqd4yfgi3rckqf9mqjtv7837y0yxq7xz724e9m9a2lx2hnp7i3u0q0lf6ayxpv907h4xrk0","1013":"zfjrgz2q90kef7u2is3um75lecidx0nmon113vu6pntbq91rra2xhskacs1mymcwo5bdie8pmtn3cxjlpbiljfv3iooqw0obb1ay","1014":"fwm46ndy0sj3ox2jnyljw9j88s0ghn0kaovetyhhlgnc0ye00knv8gv6dmpc2j77bcbo2goftdik0f0z94zoki7bigs3zqf6qa2n","1015":"fjnjn550gxegqz13lafp5vcdibzh33zx137nhdbyamu4jop2z207vzfltkwwd4oumwjvo3higbr4soyxsf0lo861embklviwopq0","1016":"mloj5g4hetidzvwb0spn20ijzfcn6muqk8cpspirn2eo03faamegv4nnikly93byot8o8454wh9ilwqpiw6elapp3mi0z9g2q4gp","1017":"kdpkoai3yqhhdo9zazh1ek6jl6wcq1eq0c2pfp0r786cajohiod73kxei81lhn6488as37hw8chr9m1l49bhq3v6rmaufys0dhpa","1018":"d0zj8ny7ig7c1hqta30842yz9z94aa8lsqg32ycb0fg1su29l73o9q4cloe03urozdkgkupf7p3c4yqk9iydqucfuv3ypw79wnqv","1019":"meeh7n1ie7a8hrdeuxzjhjpal1vz8nr4v269dtxt445jigfxt1qvxvy9q723b8fdhvlpqq6vwla7fxyaqcepa3nrtdo3q0fgoe3l","1020":"z2agdvx4wkpv93c929fwj2q7tsc2fw973ukcvms77vjc9ecyzzl5xc7d66amq0brw8wytpd4q9w5l77wattoa9p4hcd08hurrt53","1021":"txirvrxbc3l8huui1sk3xvwe3xbwsfqiuiivvuxqm1epf0zl1beusqhjb12cvrxba1d2zcorqw5geaeemmlpdopvsh0u78b2tfv6","1022":"eyqodex4qlozrt5bh4tl4c616hp6e05h8n4bzlrspleacfpa5k93fubgi42ki33po4gf2lfdwqr7fnr4c655faz9infp15l4pqj6","1023":"2iojlz4qcfe1ur8lq0a7mrt9l0d0bf5fubf5102e8ofo8k422vdf4omhsb0elxgif8jor8hwqdekole5ikpfftvoii6uu05472fc","1024":"kcmfk6uz12upomjoq5qmu0wz630mha2z00g3yf6mb3uxu51qh8kr90r8e607jo97y6f8psg688c0fy1jrpdxq0o95jti2bbsheki"}} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json new file mode 100644 index 0000000000..f16c3e3e5c --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json @@ -0,0 +1 @@ +{"id":"add9df95-3525-412f-b782-a0dbd5d0bada","NonSensitive":"dfb280c7-62a9-4d0b-b0d4-14f311fe0ec8","SensitiveStr":"ecb4222c-d335-4508-a051-654955d9465e","SensitiveInt":1692274068,"SensitiveDict":{"1":"9qze47js1zpiabsm6rvw8opz5uy0g71bvna2nv17yv3d61iegnsyq84oh3alpmvfep8jo2hhjavj8naad5zflzolsc18iweub7v8","2":"5tdm5pa1jl2ez2jvbgm9egtkincfgz5b3xzhxsq6gvwa7d65eyay6zymriq9r2jaosid4fxwilj7fxho58c8q48zhzr4vxuzv26p","3":"f0xmwqestj0d1h1es0g1rg9c8h9owc4sdhk5ul1ycnrmj6ic6owkl7qcgjs4ijgpc5iyujxiy7rd9l7ommu16vfd9nv2xuesgi5e","4":"847h5ttzmrm3dg7ey6bsid5cszjj73x0sno0ewdl30g09e5ebq8y1x91eoy04ua5kurxg7rqpjawbdtbbojgddkadnzyhkef00ed","5":"0xrlp48ceyvtfx9299c7595ae52kokox9qv5ecbjxedu5527kkqk7r0f9jpuw0xxxt109grqlem1xxsdapd1f1z44rvw9v4ggc10","6":"nuun71pabmusdv53vef4vygbypaewtpvfakbzqeep5fayihxjh13nzlhap0cdeyq0a8fi55j499gaecfux5bv2r3en4704lkt9xb","7":"d21yrhbguvwro2jzmon21trvbarj2bc4pwa5cvxs5gurhefvx5x5ufxy2ld4lv2jzgqiyhw6r6jl7620v0zizk954czfczng73c3","8":"lbvelwr350ujsnht9c3f7ckyadbq9mosbe9obfnxu3438yl0glgrg4cosvvl9gag8zkj4tscnx8pg0sssju6jfcbv9r71xpmgof9","9":"lixhbvlktx3s4202rssta5dvgtjaxzuhbrmfrllox39cb5nfau4cje64ag4kdiu6336bq34ci06c9xrt1sear4q7htvd2b17wzh3","10":"mz7umqc743raldw7tink26gqrncj3zai6zqzzy7azjuxmp136cge2riy8crc8bruwqemov2sha4aoa05zg1ahlfei50wfcj248ky","11":"tnrt4pwho6wosdtt7zz72vyoznxykda40atnyr5va1vgxc090t00qho0yy7iu3rgru7osm50ne9qth62v4afhhr31mv4banlbsui","12":"mpaae9i713xwtj0hatveyh3rlxm97emodpckgfffa72s7ufzt2tobh42dcr8wmhv6neojw8a2fyp6s62npb0k5gar43iwrlca0x9","13":"fdneqqcny6ubyirv96rbz1x5zknqh22afqee66vxwvy7dwc5u0hgjauzzfadd563bvmptedj181c6i1qaec6y9fr12xpgx8lqjec","14":"af5o4bdkpgno2oct8iot7fovousiwd19lsct2h03tbntppvw4qirhyjihnazuggtef5vqn46pl3jrviwrddve73co6gbr4qzv1jp","15":"zwe741if01gtebnuseeejzioe7wzi3h3y5hy5tmavagwqtdiic9i75yo0c64ezballqoh313vx7ga1k9xbixr0lgbuhfu8ksrj4p","16":"eahflfvs0x5mkvxia338mh3mm3x59jknsam3gnbas10vutod1dw0f41blupybowil5y2kn3oslsgmoeikeenwx0uap0dv0co3178","17":"rdwythux8nqx42bhbbzsy7bvjcgevelktbt5aw8djdh1vfjyghzjlyq88618txcz3qjogm6j4lfxkdrpv7yrwowsblj6ma5bj5d4","18":"39q2ndc6wpxm5769z7gsknrrkhiu3bvrxngs1qdmo59sv1joso1tq3z5ozzqs3obyuoff2g4b5aztum9eesr2i7oc6m31cbfll5f","19":"k9hzf4kdk8szu0qk0mpbcr6dll8jf1r5xlstooqzs88eejzhr8brhael2l0zgeiv6whpu85e25x642l6ps2e2n8jb0afzwr3b7wx","20":"qp7m3l10r4chn05fwomvd8udambaad20gy445xsiovuv8ewhp4z4rltul3syepqhepqad3katho1kufufeybbzjt5zt85gad17zy","21":"v97uza82ru2jq2urqft4kridnnsruk85n4lndw8otyjn8n7ct9lrewomf5depga2hrptu6yqifuym5oye4gkp0qtrgbhg1k22loa","22":"1v30p1o5a0wtuwp2mftybbzphlg3xpw2jpb60zepv49j5tupvha9o2hletqxmv7vl4l8vyyq1bc7rl0ifjrrbff91e6fcdje7wet","23":"xif7ligwdstp3a3v4egf1cgioyc0xdps9pir8i4tgngzs3xqembjmzq4xrz4odzruddse00hhmxrfuicx55ig87420pouttiiod6","24":"lovq7ainqta1ozmiyqzz01nolwny3hr6f5asr5te8l3eog4uwfqb7adrt4wfssdofci1oshae5cmrnkomvim9r46bs8wkjopoqni","25":"b11srpyex3nt1sn9p30joh4in5owklvgqtdmf9oq2se898k88zegzthkqq7l7qi6acyp6h44pdoqk5bj88l5agt796fjvsxm8169","26":"w27wt6o22x41jg38ha2iqu82oiruw37w1b1u5macmxqi0t3kdh41vfu7bgf13blr4pbd4svsralppddft7k3jdpz7dfzxa49ldmt","27":"j9f9unyawqa8ci0be4nqsbnwi5a776c204k15ons23mhys7bc7ykm7ttecsyxtrixkuntxlk26qpgao9u7s51r857aqntcxt5ktq","28":"1gpsx460e7spiuc30enkygbltmtglur3a64st2ee5d7rdf5bx9xrw0hkfqc3rq293mvob6b52b7e3lk6to7bffiylgstippypzog","29":"3c4alamoof07jbl0gw3an2e4sx0jkxl1di0s32ie1jr5yomt3etdj09ck95r0h58p8vrhqxoqetdw7apxvlttbucx8tkezq8cr9x","30":"j2pcy170lm6luc1nu01ugrlcxsggpkit3sirrp3nf4qvf470lkgv2hp6c2vqsbkmto7s06twff9glywum2y3ivl0n1k333it9du4","31":"cgcli39dzt0hj0ursaxakfkpka9irepzvbsdx07ogoyl9shwhixabk1kahlvkcpidc8k687fwj9g08fnyux72ai54q9zs1w62c23","32":"5byc3br3q6l724ituqdsk7qvvjny9c6yp77n6a45zydlfgi9fbbv17ho0xthkjz5is0bvzqkcqwdf24gd3h6jmm2pubp5rirex9p","33":"q9oo1ccklal9drk1lr4dk85alx4re21xl01p6o5ofisukfscryqklxiok34jtq53owqigimbii718qvofps7wl0ecln7cbf6jlbt","34":"5ckzpge6a1cg8z5cicknjs1x55n8b2lcy17gu8uro97tyovxxi5a130ubjz19j3zwi3l72wv3sen8lq2k87acium45kx1cu2qyof","35":"tz7qg132w5qi569inb54hn70wwy7sbufreopspbtdm01elq1ozl9qa90pq0wbg9aid03yyifr4yk711pjx7z10lotp06lupypwry","36":"73dybhdkumsgmoe95rruec80v1y3qo415zxrwq1ekt6afgch0hofkmbfrkeqqexduv0rgekc1hbtap55nollt1f5sg9clla2xbgv","37":"1aqcjgkm3dxsiqa87kjwdfynlwqoe9mdc29felg1a20pmo784tfr29uv25xoa6icelyxb3forx2dw2aegt816ped3w6z7kl8tupn","38":"t7zubumt7woqcnv920fw8v959m1vy3l8yllp9ywelm7rgfb1h6s554h754bsxpdfjglqq22cn0teuf57plfbkcrtatcn8g2f0sy8","39":"qp4y1bpe55xzscelojvqe6omjus8bevbut3haht9ufl900z5w6ddfudq9c6o264f0f91iamaye01w5u7z749fgoaeaft0s3giv40","40":"hk99cvejeyshfiw5jlmecn9a7d3viwrhlcx58w6xm1v79c1lpu39pz7kzs0ubo04tcgu2rtls6dlcv2xou7zd9zbrlygra3nzu7d","41":"jkh5uwir6epqwj0qnt8jpbj85im6iibokeaqdzpp8585o8seuoje3ioxtn5eizfx408i7p02cogrllqxanw2cz8y39q9011v94c3","42":"dtfp5kw38hfh6te0y1rrgoe8qk94pqp7cnqi5vin7a73254zcui1cjapk8t8dtr5gsklakhouqm4cp1bz25f5u80zs4n5bp8h1d1","43":"0leckg1prka3sbvbhya0dk3rpwqcak8rwejwh6pkbfmc68nh1iwgfnydyrxhz2p7ygszbel2h6atla1uu6gqyze6q1eduuhw80zh","44":"gtiktm0lqhgzz88gvgov6sns4zkmibp1q57gk3vupq6o0t0mx0urmuouoo24xhk06sjelc18eejdb3pwushfw123if9h22ecpoc9","45":"kzptbjo5rkh7p0jyhrcy296hay7t8n6zzgaiorsidh1oomfpb9e5goohvhpxw6ynaqpu6cyb6nhjexgxgxvb6gam3bnui0ii0355","46":"k3q8fkon89ppant3swis0tt9dcvmk3inyzurrdc5rhg13vn66eip0vuhq2to5bfb2j78wthudjcz6px4sdg5lxfh2myjx595biob","47":"vj32urvjr93i45nk4t9m00lkhrbjrrtwqfogn8xc277iu8bbbp7rv4enwbjtg4kg1n8svxy5jejng5tqzjw0z1nnhje1nrxrdu94","48":"0fcgvl3dlrwbjkk44r055j6vmkleqzg8cbe2ne4qxq2plrue93nh3vjd3zh08h5uyw9s95du5b6kvnk8gufzmb5q8s3bjemfy700","49":"c75714w3b5flq5he0uso6wdgybxv3b2t2kwyniajlkae1bvvhsyg1pzcvzi3draxkrwfkjdbzosw506wpwywisqe0tstt8g8ta2s","50":"sg3psireybv4etr63qpur5m4aezzeopo9w9bigh6iqiwndpx2dtij4rjgj2dithrt5cg33qfoj9f7xq16c0jl62jvwvjoarrt8xt","51":"3g9efb5t9b0ty5dwg1wc7xudwvkfngx9a845z2pgx4c5rbdvgovd2k9l77svabimpfbpznqeryii7rdnu0vkeeb1or52jm0lxcij","52":"nvy08acdgmhtiy0iczl1da8o8mrvomjd1tgf4y5eboxfvrrmcf0q60vk6f8tcvxwyoukucq1w4l2ju31yu6nfwog52rjeoizqpbs","53":"4ubre02694gvfb6dqzxut4t1scpuojcswtrn0xue0jw14hxhoh15xa91zk7yhrw4uhzlj4xsrf7m18pg3cf8x7kdx9negsqa2yd9","54":"1ia31ra8p4yumhakor8w4ck02c230s4hacyynpjhhxzrus2fqz9ke16lubyxv9i6kceayfqjblu4diaplh6lja82gs4q4q6w51hh","55":"x48ku1h3h5o4omqtw1kbbi658th9bqrb51u9ud62egu6im7kk8dt8wz1oymgf04pyk8c4qj92346ahrzptovtapmr4efby47nwkc","56":"5uotabm9n5byj9gy3rwsq2hsg2taab7by2698etbj08t9zb9u51yf4i17qiio1ng0t612seq8zcz60w7r9eyeejvucg9vgg2avsy","57":"4dfn56jtup5d3mx7johhhvbz9tefgwlouslw5gfce8xxuqsbkr2x65ykntp19j0fllnxkunk64sqj80k16kznehsx2rli79elba3","58":"dnj3du5zta5vcqsz7plwe0082wk7rn2me04xunjfkmihkemgam6gsirhfadkyvjopwvvxkhe84ianyqkf9mceur97q3mwv2dbzxa","59":"kgoj6765r8xzs0106h8cwz5vu7f1l0ddvofm0gtchldwauqu2yid1hywkeru19yxk5vnjsuv5nfjl78jzqodz92b9geptc1prw12","60":"tcaue8mbhkjsfmftnhy0qg22ccy0w1fmy8d1i4xpzm9s4pwb6ivv5yw2jh8jwj7smk3mtmh9kur1t5vsojr6odhgkr1l91d5cu5f","61":"cechvcezrij2ea41rabsmiz1avp8dvflr3givg6dnqfjs0cr7esi7pb7mft3yc5g08y73795bbsk3jjhr21iqkzh6fkkbwdh2rr0","62":"celnfzj1uipd25kjh1jl74q7npkp6gntvkzkjpn5ipax4q43m062ddf4gg8pvzs6i9g2mbqphpptygltdkc5mj305dh97tdv74tf","63":"8kxk1r9zrqsv3zb3v3bkf7zxclxfydkmvtzeuh51zt0w3c1aop3zxq15vo6w5idrd154ca2umlbdpfq5roj07hozpx2u1qn72g57","64":"ogqtkhmr1df4k29yvs57ayiqght7rodqxui67wtpsmfmva3yw9u8mfloe4xt72pxl1b620jwfresmn1f3bjuayutbcdia0cxs3xd","65":"27phu3a62d1cj9wwqyz524vyo7bxoc1qad1naob2xi0uo3dw5blx8nim5gthmh7tgrng9f3rdmi5fbwoglofum8glbhzmdvsj2th","66":"hynij3cgqcggezuszytkt7cnv0qjq97lhwbirbuv1l9vqemjmrqovkbu74w1060nx359pqwwfwgo3iyy2hl3cay7s1q8darqng8o","67":"kdnnac968jogx8scgufafvwvrj0iyk3evw17zswprn8toukdonl8mh69cx1jbgn3mqm1r3v1ydeebfqg5oni5tubuljbqp13jb35","68":"q7bqn04nirvo0ls2z3tz1v2i6k01ke81w434o708nc6ajd26qi3qhd6r4dogmsw4su1fg0de0qpvw1yck8enw05q5m6zd5gipi2n","69":"rtn7298ijs0k4kt6fwhbfs8njplfx42ugyxnbj4aa5gz9wxrla2jhr6wc2idxqbwx7t4fhtn8ljon30ayarhb46u1505zcpd0s1t","70":"9zntysfwmzq1rvsi2t3sr2zh5udpy1ar3d6bqs3j0d2cgmnt26pdudw66j9rxf1vi1ztm3lb0p4gd71tmcvbdukkzo71vdg4xvif","71":"yklqa8rzjpv4o0246hw41botq7dls78itt2o0f5ihywvdurdj8icp6ag3w21uc9umx1gyicxxb18dkc68fwwp04yfuhzjt20teu1","72":"d4av0chqi949y2dg8nscntr6dgpj2j41ypt9pjctkqjmki23w15mr1inwkup397kx2phq97hjawv0l9sxrbkx6yc3id35l4tze3k","73":"vkne4012phhsdcsn1xegy5eziyxi6qtuse0z49bn7p9dvg37qn91x6okpmbklkwqur7swhmg9b8xomfv9komk8mbkpkz2dbx3wp5","74":"72rfcv52r3bvtaymza4kiv0thyvsxun7ha4yf6f066szwf4ysic410uzf03vp40h21g9vvdgdsgmqmdde0uy5ypnlsl8fhdgrp2j","75":"b1qkpvkqyf5ylylzzoef8ps90sad8zjkbdqb2m3iqjkzn1mxq7627wxwop56q8qvd1cwyh0ltj76aemqipi5hqqggnsd80afursg","76":"d35v3cunt8s3puqys02wft6uqxty8gxo77olhqvvamm8u8bs999r9gwpnqoak7gkiqlgyk0td0uv9yasuxweovng860j5d2fmclh","77":"febksm0249z1z1ebqly3bfwprefs3uxwy95geqhdlncbkudyzoy0pg7iske1w2v6fdlp5il7u9pt06zsgn9te16pc14yi3ezafse","78":"6hiv0nk4emrswr778udshxtjar9y0im36gvm2sontq9qzr718afv1xlypu4libs5nqz5d5k6h7ixdefkoi5jvuou1i7noxzz0cna","79":"eid2wkk3jp4m8j2g2aufm3tnhdesisj5mroqonuxmhovuj9cc6215h1bgxmla2ezf8khis0fpkma9oleyqh56zyppp2a4of2hwye","80":"u09hgdntqkycqckwhe0ftxgxfoqvqmla0ssvarygkfmedjdjbfuyc39ol6g56x4lbsklbm0a0g8bjqbql396xd5cb233vo0edkb2","81":"yob5q3d14pe8k4ix7dsd9x4dw0riiy84jk0694mdics2dgvhpen9x947okfh51l8adibi24q42n49mba7yczlumuadqspqhudp3n","82":"twhpcolc0nvgfkwlmyaedbj49gqzadvtoczodupznap3wgmdbkaaf98326awb6pmb6msoz6q668m6lzqn85hin1i9u743f8g8isu","83":"92v8gj43iy18bpps98ohky9we9yazlp4k4bk1z9a2rh3r7e0oa4igvth88el7bx0sur4tf8dqza4ufy3b2cnazejyj3ndqcobgrs","84":"3uwqalut4diz6xjpryaaw9aetvc1g0ajzxnypchk619zxj6kgrapjhefcsx6qoqr2elhxu418h3yf5sygzi4yh2n0xqzwy7nc9se","85":"tn2kfq5r48e27oh5g3gcnmqhwswip11a1obt5df83kuphgb157ae7qagahemhfngq5rozlwtvkwc5loh96pfrqbv70th63skbdqt","86":"s3p0q74e5f2fjwryhw6vdtm3vke54q58juq7c6bwu5m5siippv9t52nq0y5i31u1wkvmeywop7omvvatestblya5mn0z0lig05vp","87":"zdk0fm2p6xvfwmh1l6mv59om77x0bcpsocfe2hvyh4c6t8w3r44qjamiau20ccbm0jvvhk44hgf568fr9tvkabedefdne33oov9b","88":"p3idf0iajvfvck7az0ddjogayn89hblmaqo8b95rqdxvx5rbsnm4txoeno3lvtb4ckzpyen3k3cs8o6jzxreh1qb3706sr9ly5g0","89":"3fmuct4aupjbuclx22phlz7b33ew82whxntcirgcn5r8sgmc708ibjn80gtih3yfu5l7vyxbb5htk3svps5jrfmakqvurp2vnq8p","90":"25516v2ifoxqvivlpd1x6zrlzfojxyg957g8svamggl6p132sbvjirapnyf2cyekqyotmypattyk34n61072ov6sd94lhr33emg1","91":"xqf0i8yokyw98i3p1ef1ucyv9g96v6vhwx4ztkso6irzcll54t20kho4wrhpt2xont9d0l42kl5m7xuapu1s6aq0mqmq3bl8ubo2","92":"fohdadlq4fyzxxq0vp33uetzxnlxi5bdl3ihfk9grugjykt3netq4hbr0uxw1zfenjfq8b5r8egas7b3q9bdyih1o6bddebp3gys","93":"g9tobhvstwkcat294bbqtzwq0kpb2xuvxgyzow1014oe8b57o63v3l20u36aug1hdy2tqgk5uy94d7i9yv95r5wtulsqzvp97uzg","94":"snqo2jyc5dye8ejyutpw06di5tovsv25chak49ws4vzeudzmf1ux0vwp85lmr0moj5j0190ohjfdpba9exsoiiwjlicd05ak5icf","95":"9egy7ndw8sr49hl9rt8sh0b8qlapngix6pf6ysjcwrs37o0l8n3zlqxfnjzepnaxjgf6n1qu4morw8tmpavd0j1nck58lk8i1hdy","96":"g7vz9wc75rupaeqdev0gcd31nmc7iv0nf2rljrexslbis6dbm7p2tdoq2fufb7aic8mh30917mrzbsst461y0ozeqxmv03f4xz4m","97":"biiprt4jdw10dyhlxdmayxkksz30lwlt3kzw8a5nd6tp5d1v8ljdlfo3gkd8n07oxcfi2mkd58frjiwomnlju2lhgx2r8nsbntpi","98":"n5zd3mpm7bv2y8klzlq3lb5ma03j1hrwsjif51y80f60r2tk8pbk79mycz77frrljt5ddh2nbe093h7fj2ad86y35h1qvltk8lc0","99":"3krxtcd90i41pb66820rodjdopeparzaouoja8g6229gn1m18wco63gq1nxtiv6xt1bv6rs81ogui7y5zukn7eosr9tevdxwvtip","100":"6iorwaxm06cd6l6bstchrg3brbtac7ofaods2rstpya29t0a2nvnjjlnuvarwx2cj76fawfh5ukc3b4ksu5ifnkjspxcb8k68c4t","101":"l5t2x74kpznm9fy8ix1zrbthnudoafu67t9op6r975gws6f022xq1utd69p9vd8k9bh29d09e45vxogtlmff68h72po4ljeanki3","102":"1oxb7e6s0ecx3lpsntxmi5qis6l10a6japco2r82bzi3d6g5p83wcfeog057m496tlz42bd5c73z7opji12kr42a6c0mb6tdaki6"}} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json new file mode 100644 index 0000000000..0a6c195ae8 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json @@ -0,0 +1 @@ +{"id":"f5b1a667-0e39-4177-986c-d05d38e1eac8","NonSensitive":"80382481-0cd4-410f-a5dc-44fad3a935ee","SensitiveStr":"8a1506f9-7b15-4f8e-96dd-50033e540260","SensitiveInt":1737683325,"SensitiveDict":{"1":"7oockvrrhrreznj9lgq38e82g96608slltnab3hco7w5alehsmf1yp4r68kojs159ycuqq9syfqvsw3gjmprt0yhbtzoityuro5b","2":"ofoz73i909m7pqhg93o2a05ozoq9x1n1k3817t2vwgym1y3nbzf3d2njgej3iqutlo4xcdyri4fyjfirflpslwg2zfvv159vujex","3":"6371pofp39fuye4mz99vdkc19wo4l1ze7ocgbb12rnwozqgg9g1zqt6fnayhlgjmdjdab4xwgycpjkr8jyn4bp2kr18lo625bznp","4":"hweoq8ktlyi81z1h6j702cted8oxrya1dsh0rm5c5by1b69vmw5czcxqx6aqlmyf20yiurbzgqylns57si31qmo76qyvx2vggynb","5":"tuqwaitw9u230otpniu6txocfi1l3k6mm62f4tkak84wuf5c979qhd2wksx84zieutzblqnpv2n8ub7uivc92kpz7pfk9huulrhd","6":"l8q06w89xgyt23pk0pizu012tjz45ptfpozh8enn3z1ipg00di1n1oot14pjqluh5hlys9s7hkkkzhwv29ob5vvmj2saqptdi4iv","7":"edjikhq40zza6rl4cnn0y92madl59ek7qdiyeszvbkbzcjyqzbjw1dsuni1qspumqj56iza25j839czjt0p0yyl6n7pg7ekfj548","8":"2nqoipmh21552ymx1wvfef4t629tmb33cjom6orbjaa1sb4b7f5xagonllk0q25vqz0murnaseha01s99bktvjcltl0d1892qd8e","9":"gag5o0xxrdhwq7x6m9r0eyp9jg8f6oarcoizpr4j7hlj3ry8n7e5s7yseh8g7n5dweuqcoulbo0gb71hng5c6p73qfddhdfadz1i","10":"wxb3ykip4tehje4uikysdru15n9jvyh5fgcsl1m802lfx6c892vxt1ryyfrlknfp97a0antnfs3vpjref21j0lystuno14t7izg6"}} \ No newline at end of file From 38a198a22128c17980f424672633a4e9934edd03 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 11:11:13 +0200 Subject: [PATCH 04/49] Add non-allocating APIs to encryptors --- .../src/AeadAes256CbcHmac256Algorithm.cs | 19 ++++++++ .../src/CosmosEncryptor.cs | 30 +++++++++++++ .../src/DataEncryptionKey.cs | 22 ++++++++++ .../src/Encryptor.cs | 44 +++++++++++++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 21 ++++++--- ...soft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../EmulatorTests/LegacyEncryptionTests.cs | 30 +++++++++++++ .../EmulatorTests/MdeCustomEncryptionTests.cs | 30 +++++++++++++ 8 files changed, 192 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index df416c3efd..73049fd9ca 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -137,6 +137,20 @@ public override byte[] EncryptData(byte[] plainText) return this.EncryptData(plainText, hasAuthenticationTag: true); } + /// + /// Encryption Algorithm + /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits + /// cell_ciphertext = AES-CBC-256(enc_key, cell_iv, cell_data) with PKCS7 padding. + /// cell_tag = HMAC_SHA-2-256(mac_key, versionbyte + cell_iv + cell_ciphertext + versionbyte_length) + /// cell_blob = versionbyte + cell_tag + cell_iv + cell_ciphertext + /// + /// Plaintext data to be encrypted + /// Returns the ciphertext corresponding to the plaintext. + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } + /// /// Encryption Algorithm /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits @@ -418,5 +432,10 @@ private byte[] PrepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset Buffer.BlockCopy(computedHash, 0, authenticationTag, 0, authenticationTag.Length); return authenticationTag; } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 462bd56a1f..20dda27ee5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -48,6 +48,21 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + /// public override async Task EncryptAsync( byte[] plainText, @@ -67,5 +82,20 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index bcee51d0e1..c505199dd6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -29,6 +29,17 @@ public abstract class DataEncryptionKey /// Encrypted value. public abstract byte[] EncryptData(byte[] plainText); + /// + /// Encrypts the plainText with a data encryption key. + /// + /// Plain text value to be encrypted. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Encrypted value. + public abstract int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset); + /// /// Decrypts the cipherText with a data encryption key. /// @@ -36,6 +47,17 @@ public abstract class DataEncryptionKey /// Plain text. public abstract byte[] DecryptData(byte[] cipherText); + /// + /// Decrypts the cipherText with a data encryption key. + /// + /// Ciphertext value to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Plain text. + public abstract int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset); + /// /// Generates raw data encryption key bytes suitable for use with the provided encryption algorithm. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 1e7f272ce5..73eeaa0890 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -27,6 +27,28 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Encrypts the plainText using the key and algorithm provided. + /// + /// Plain text. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Cipher text. + public abstract Task EncryptAsync( + byte[] plainText, + int plainTextOffset, + int plainTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + /// /// Decrypts the cipherText using the key and algorithm provided. /// @@ -40,5 +62,27 @@ public abstract Task DecryptAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + + /// + /// Decrypts the cipherText using the key and algorithm provided. + /// + /// Ciphertext to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Plain text. + public abstract Task DecryptAsync( + byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 68d863114e..0377c4257f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -12,6 +12,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal sealed class MdeEncryptionAlgorithm : DataEncryptionKey { + private const byte Version = 1; + private readonly AeadAes256CbcHmac256EncryptionAlgorithm mdeAeadAes256CbcHmac256EncryptionAlgorithm; private readonly byte[] unwrapKey; @@ -65,7 +67,8 @@ public MdeEncryptionAlgorithm( dekProperties.WrappedDataEncryptionKey); this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( protectedDataEncryptionKey, - encryptionType); + encryptionType, + Version); } else { @@ -80,11 +83,9 @@ public MdeEncryptionAlgorithm( this.RawKey = rawKey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( plaintextDataEncryptionKey, - encryptionType); - + encryptionType, + Version); } - - } /// @@ -125,5 +126,15 @@ public override byte[] DecryptData(byte[] cipherText) { return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText); } + + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Encrypt(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index ed4cca0596..ef53408b4a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index fd83ef528d..525b6b8d7f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1763,6 +1763,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -1776,6 +1796,16 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 85b7bf3f36..5526551480 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2248,6 +2248,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -2261,6 +2281,16 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } } From e0bb8bf39a41454e7ee8b89426c2a433bf4b6073 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 14:24:46 +0200 Subject: [PATCH 05/49] WIP --- .../src/AeadAes256CbcHmac256Algorithm.cs | 26 +++++++++++ .../src/CosmosEncryptor.cs | 10 +++++ .../src/DataEncryptionKey.cs | 14 ++++++ .../src/EncryptionProcessor.cs | 21 ++++++--- .../src/Encryptor.cs | 28 ++++++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 13 +++++- .../EmulatorTests/LegacyEncryptionTests.cs | 10 +++++ .../EmulatorTests/MdeCustomEncryptionTests.cs | 20 +++++++++ .../AeadAes256CbcHmac256AlgorithmTests.cs | 43 +++++++++++++++++++ 9 files changed, 178 insertions(+), 7 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index 73049fd9ca..a51e09726b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -26,11 +26,21 @@ internal class AeadAes256CbcHmac256Algorithm : DataEncryptionKey /// private const int KeySizeInBytes = AeadAes256CbcHmac256EncryptionKey.KeySize / 8; + /// + /// Authentication tag size in bytes + /// + private const int AuthenticationTagSizeInBytes = KeySizeInBytes; + /// /// Block size in bytes. AES uses 16 byte blocks. /// private const int BlockSizeInBytes = 16; + /// + /// Size of Initialization Vector in bytes. + /// + private const int IvSizeInBytes = 16; + /// /// Minimum Length of cipherText without authentication tag. This value is 1 (version byte) + 16 (IV) + 16 (minimum of 1 block of cipher Text) /// @@ -437,5 +447,21 @@ public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cip { throw new NotImplementedException(); } + + public override int GetEncryptByteCount(int plainTextLength) + { + // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. + return sizeof(byte) + AuthenticationTagSizeInBytes + IvSizeInBytes + GetCipherTextLength(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + throw new NotImplementedException(); + } + + private static int GetCipherTextLength(int inputSize) + { + return ((inputSize / BlockSizeInBytes) + 1) * BlockSizeInBytes; + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 20dda27ee5..1a0f093176 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -97,5 +97,15 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } + + public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index c505199dd6..65890ec941 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -40,6 +40,13 @@ public abstract class DataEncryptionKey /// Encrypted value. public abstract int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset); + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Size of input when encrypted. + public abstract int GetEncryptByteCount(int plainTextLength); + /// /// Decrypts the cipherText with a data encryption key. /// @@ -58,6 +65,13 @@ public abstract class DataEncryptionKey /// Plain text. public abstract int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset); + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Upper bound size of the input when decrypted. + public abstract int GetDecryptByteCount(int cipherTextLength); + /// /// Generates raw data encryption key bytes suitable for use with the provided encryption algorithm. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index af10de5e11..bb5beda440 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -99,19 +99,28 @@ public static async Task EncryptAsync( (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); - cipherText = await encryptor.EncryptAsync( + int cipherTextLength = await encryptor.GetEncryptBytesCount( + plainText.Length, + encryptionOptions.DataEncryptionKeyId, + encryptionOptions.EncryptionAlgorithm); + + byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; + cipherTextWithTypeMarker[0] = (byte)typeMarker; + + int encryptedBytesCount = await encryptor.EncryptAsync( plainText, + plainTextOffset: 0, + plainTextLength: plainText.Length, + cipherTextWithTypeMarker, + outputOffset: 1, encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - if (cipherText == null) + if (encryptedBytesCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - - byte[] cipherTextWithTypeMarker = new byte[cipherText.Length + 1]; - cipherTextWithTypeMarker[0] = (byte)typeMarker; - Buffer.BlockCopy(cipherText, 0, cipherTextWithTypeMarker, 1, cipherText.Length); + itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 73eeaa0890..9362c6f2a0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -49,6 +49,20 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Size of input when encrypted. + public abstract Task GetEncryptBytesCount( + int plainTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + /// /// Decrypts the cipherText using the key and algorithm provided. /// @@ -84,5 +98,19 @@ public abstract Task DecryptAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Upper bound size of the input when decrypted. + public abstract Task GetDecryptBytesCount( + int cipherTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 0377c4257f..ba606e9412 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -104,7 +104,8 @@ public MdeEncryptionAlgorithm( this.RawKey = rawkey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( dataEncryptionKey, - encryptionType); + encryptionType, + Version); } /// @@ -136,5 +137,15 @@ public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cip { return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); } + + public override int GetEncryptByteCount(int plainTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetEncryptByteCount(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetDecryptByteCount(cipherTextLength); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 525b6b8d7f..7b8d85e878 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1806,6 +1806,16 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } + + public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 5526551480..69924c0a7a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2291,6 +2291,26 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } + + public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs new file mode 100644 index 0000000000..329e06e7c5 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -0,0 +1,43 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Tests +{ + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using System.Linq; + using System.Text; + + [TestClass] + public class AeadAes256CbcHmac256AlgorithmTests + { + private static readonly byte[] RootKey = new byte[32]; + + private static AeadAes256CbcHmac256EncryptionKey key; + private static AeadAes256CbcHmac256Algorithm algorithm; + + [ClassInitialize] + public static void ClassInitialize(TestContext testContext) + { + AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); + AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); + } + + [TestMethod] + public void EncryptUsingBufferDecryptsSuccessfully() + { + byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 } ; + + int cipherTextLength = algorithm.GetEncryptByteCount(plainTextBytes.Length); + byte[] cipherTextBytes = new byte[cipherTextLength]; + + int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); + Assert.Equals(encrypted, cipherTextLength); + + byte[] decrypted = algorithm.DecryptData(cipherTextBytes); + + Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); + } + } +} From 9c3c276e389db6bd2971d73bd4ad722e7ab88737 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 15:38:12 +0200 Subject: [PATCH 06/49] Revert solution update --- Microsoft.Azure.Cosmos.sln | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index 6fa5e4c3f2..b1d77052bf 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.12.35209.166 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29123.88 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject From f33538c684911a9f8d472c800477f0f54d036f9e Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Tue, 17 Sep 2024 10:14:19 +0200 Subject: [PATCH 07/49] Add implementation, fix tests --- .../src/CosmosEncryptor.cs | 28 ++++++++++++++++--- .../src/EncryptionProcessor.cs | 2 +- .../EmulatorTests/LegacyEncryptionTests.cs | 18 +++++++++--- .../Readme.md | 14 +++++----- .../MdeEncryptionProcessorTests.cs | 15 +++++++++- .../TestCommon.cs | 17 ++++++++++- 6 files changed, 76 insertions(+), 18 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 1a0f093176..34c616b227 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -98,14 +98,34 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetEncryptByteCount(plainTextLength); } - public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetDecryptByteCount(cipherTextLength); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index bb5beda440..3d93d0be8a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -120,7 +120,7 @@ public static async Task EncryptAsync( { throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - + itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 7b8d85e878..d7f8c5a22b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1807,14 +1807,24 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); } - public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index bb4836273b..44aae54dd0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -2,7 +2,7 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=9.0.100-preview.7.24407.12 +.NET SDK=9.0.100-rc.1.24452.12 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **60.05 μs** | **1.537 μs** | **2.300 μs** | **5.0659** | **1.2817** | **-** | **62.65 KB** | -| Decrypt | 1 | 70.76 μs | 0.812 μs | 1.164 μs | 5.7373 | 1.4648 | - | 71.22 KB | -| **Encrypt** | **10** | **165.23 μs** | **3.741 μs** | **5.365 μs** | **21.2402** | **3.6621** | **-** | **262.38 KB** | -| Decrypt | 10 | 231.32 μs | 4.627 μs | 6.635 μs | 29.5410 | 3.4180 | - | 363.84 KB | -| **Encrypt** | **100** | **2,572.40 μs** | **242.163 μs** | **362.458 μs** | **201.1719** | **126.9531** | **125.0000** | **2466.27 KB** | -| Decrypt | 100 | 2,952.48 μs | 397.387 μs | 557.081 μs | 255.8594 | 210.9375 | 160.1563 | 3412.88 KB | +| **Encrypt** | **1** | **81.01 μs** | **2.595 μs** | **3.804 μs** | **4.7607** | **1.5869** | **-** | **58.87 KB** | +| Decrypt | 1 | 68.96 μs | 1.627 μs | 2.333 μs | 5.0049 | 1.2207 | - | 61.57 KB | +| **Encrypt** | **10** | **190.05 μs** | **5.716 μs** | **8.556 μs** | **16.1133** | **3.9063** | **-** | **197.66 KB** | +| Decrypt | 10 | 215.60 μs | 4.424 μs | 6.484 μs | 24.6582 | 4.8828 | - | 303.41 KB | +| **Encrypt** | **100** | **2,261.05 μs** | **212.952 μs** | **298.529 μs** | **148.4375** | **78.1250** | **74.2188** | **1785.47 KB** | +| Decrypt | 100 | 3,139.31 μs | 316.105 μs | 473.131 μs | 224.6094 | 175.7813 | 130.8594 | 3066.66 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 1396e7e06e..562cc0f0a7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -24,7 +24,7 @@ public class MdeEncryptionProcessorTests private const string dekId = "dekId"; [ClassInitialize] - public static void ClassInitilize(TestContext testContext) + public static void ClassInitialize(TestContext testContext) { _ = testContext; MdeEncryptionProcessorTests.encryptionOptions = new EncryptionOptions() @@ -38,9 +38,22 @@ public static void ClassInitilize(TestContext testContext) MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int plainTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? plainTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] plainText, int plainTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index 63968723e6..a0ecdab14e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -27,11 +27,26 @@ internal static byte[] EncryptData(byte[] plainText) return plainText.Select(b => (byte)(b + 1)).ToArray(); } + internal static int EncryptData(byte[] plainText, int inputOffset, byte[] output, int outputOffset) + { + byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset).ToArray()); + Buffer.BlockCopy(cipherText, 0, output, outputOffset, plainText.Length); + + return cipherText.Length; + } + internal static byte[] DecryptData(byte[] cipherText) { return cipherText.Select(b => (byte)(b - 1)).ToArray(); } - + + internal static int DecryptData(byte[] cipherText, int inputOffset, byte[] output, int outputOffset) + { + byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset).ToArray()); + Buffer.BlockCopy(plainText, 0, output, outputOffset, plainText.Length); + return plainText.Length; + } + internal static Stream ToStream(T input) { string s = JsonConvert.SerializeObject(input); From 6923fd2fdb29a831e6eadd01e26463c2e312d254 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Tue, 17 Sep 2024 18:17:17 +0200 Subject: [PATCH 08/49] Switch to randomized encryption for benchmarks --- .../EncryptionBenchmark.cs | 2 +- ...smos.Encryption.Custom.Performance.Tests.csproj | 6 ------ .../Readme.md | 14 +++++++------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index d8e4c7d8a2..3655043648 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -35,7 +35,7 @@ public async Task Setup() Mock keyProvider = new(); keyProvider .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); + .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Randomized, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); this.encryptionOptions = CreateEncryptionOptions(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index e32a42c36c..4deb4d0edf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -8,12 +8,6 @@ enable - - - - - - diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index bb4836273b..2807f8c809 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -2,7 +2,7 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=9.0.100-preview.7.24407.12 +.NET SDK=9.0.100-rc.1.24452.12 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **60.05 μs** | **1.537 μs** | **2.300 μs** | **5.0659** | **1.2817** | **-** | **62.65 KB** | -| Decrypt | 1 | 70.76 μs | 0.812 μs | 1.164 μs | 5.7373 | 1.4648 | - | 71.22 KB | -| **Encrypt** | **10** | **165.23 μs** | **3.741 μs** | **5.365 μs** | **21.2402** | **3.6621** | **-** | **262.38 KB** | -| Decrypt | 10 | 231.32 μs | 4.627 μs | 6.635 μs | 29.5410 | 3.4180 | - | 363.84 KB | -| **Encrypt** | **100** | **2,572.40 μs** | **242.163 μs** | **362.458 μs** | **201.1719** | **126.9531** | **125.0000** | **2466.27 KB** | -| Decrypt | 100 | 2,952.48 μs | 397.387 μs | 557.081 μs | 255.8594 | 210.9375 | 160.1563 | 3412.88 KB | +| **Encrypt** | **1** | **61.45 μs** | **1.676 μs** | **2.457 μs** | **4.9438** | **1.2207** | **-** | **61.25 KB** | +| Decrypt | 1 | 77.89 μs | 1.959 μs | 2.933 μs | 5.7373 | 1.4648 | - | 71.22 KB | +| **Encrypt** | **10** | **171.64 μs** | **3.341 μs** | **4.791 μs** | **21.2402** | **3.6621** | **-** | **260.97 KB** | +| Decrypt | 10 | 255.57 μs | 7.833 μs | 11.724 μs | 29.2969 | 4.3945 | - | 363.84 KB | +| **Encrypt** | **100** | **2,601.33 μs** | **215.481 μs** | **322.522 μs** | **199.2188** | **125.0000** | **123.0469** | **2464.88 KB** | +| Decrypt | 100 | 3,156.06 μs | 321.419 μs | 481.084 μs | 355.4688 | 300.7813 | 261.7188 | 3413.05 KB | From 03ef682ba696e192985b8ddd0632176882bedcd5 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Tue, 17 Sep 2024 18:05:07 +0200 Subject: [PATCH 09/49] Some more array pooling --- Directory.Build.props | 1 + .../src/ArrayPoolManager.cs | 47 ++++++ .../src/CosmosEncryptor.cs | 8 +- .../src/EncryptionProcessor.cs | 147 +++++++++++++----- .../src/Encryptor.cs | 4 +- .../src/Mirrored/UnixDateTimeConverter.cs | 2 +- .../EmulatorTests/LegacyEncryptionTests.cs | 4 +- .../EmulatorTests/MdeCustomEncryptionTests.cs | 4 +- .../EncryptionBenchmark.cs | 2 +- .../Readme.md | 16 +- .../MdeEncryptionProcessorTests.cs | 12 +- .../TestCommon.cs | 10 +- Microsoft.Azure.Cosmos.lutconfig | 6 + 13 files changed, 196 insertions(+), 67 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs create mode 100644 Microsoft.Azure.Cosmos.lutconfig diff --git a/Directory.Build.props b/Directory.Build.props index 04fc0bc918..7b9401e44a 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,6 +12,7 @@ 10.0 $([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../')) $(DefineConstants);PREVIEW;ENCRYPTIONPREVIEW + NU1903 diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs new file mode 100644 index 0000000000..b09a919a5f --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -0,0 +1,47 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Buffers; + using System.Collections.Generic; + + internal class ArrayPoolManager : IDisposable + { + private List rentedBuffers = new List(); + private bool disposedValue; + + public byte[] Rent(int minimumLength) + { + byte[] buffer = ArrayPool.Shared.Rent(minimumLength); + this.rentedBuffers.Add(buffer); + return buffer; + } + + protected virtual void Dispose(bool disposing) + { + if (!this.disposedValue) + { + if (disposing) + { + foreach (byte[] buffer in this.rentedBuffers) + { + ArrayPool.Shared.Return(buffer); + } + } + + this.rentedBuffers = null; + this.disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 34c616b227..0288ee79db 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -48,6 +48,7 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + /// public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( @@ -83,6 +84,7 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + /// public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( @@ -98,7 +100,8 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + /// + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, @@ -113,7 +116,8 @@ public override async Task GetEncryptBytesCount(int plainTextLength, string return dek.GetEncryptByteCount(plainTextLength); } - public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + /// + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 3d93d0be8a..e7afe78c2a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; + using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -25,6 +26,9 @@ internal static class EncryptionProcessor // UTF-8 encoding. private static readonly SqlVarCharSerializer SqlVarCharSerializer = new SqlVarCharSerializer(size: -1, codePageCharacterEncoding: 65001); + private static readonly SqlBitSerializer SqlBoolSerializer = new SqlBitSerializer(); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new SqlFloatSerializer(); + private static readonly SqlBigIntSerializer SqlLongSerializer = new SqlBigIntSerializer(); private static readonly JsonSerializerSettings JsonSerializerSettings = new JsonSerializerSettings() { @@ -80,6 +84,8 @@ public static async Task EncryptAsync( byte[] cipherText = null; TypeMarker typeMarker; + using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + switch (encryptionOptions.EncryptionAlgorithm) { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: @@ -97,20 +103,26 @@ public static async Task EncryptAsync( continue; } - (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); + (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); + + if (plainText == null) + { + continue; + } - int cipherTextLength = await encryptor.GetEncryptBytesCount( + int cipherTextLength = await encryptor.GetEncryptBytesCountAsync( plainText.Length, encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; + byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); + cipherTextWithTypeMarker[0] = (byte)typeMarker; int encryptedBytesCount = await encryptor.EncryptAsync( plainText, plainTextOffset: 0, - plainTextLength: plainText.Length, + plainTextLength, cipherTextWithTypeMarker, outputOffset: 1, encryptionOptions.DataEncryptionKeyId, @@ -121,16 +133,16 @@ public static async Task EncryptAsync( throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - itemJObj[propertyName] = cipherTextWithTypeMarker; + itemJObj[propertyName] = cipherTextWithTypeMarker.AsSpan(0, encryptedBytesCount + 1).ToArray(); pathsEncrypted.Add(pathToEncrypt); } encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted); + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); break; case CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized: @@ -166,11 +178,11 @@ public static async Task EncryptAsync( } encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 2, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: cipherText, - encryptionOptions.PathsToEncrypt); + encryptionFormatVersion: 2, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: cipherText, + encryptionOptions.PathsToEncrypt); break; default: @@ -278,6 +290,8 @@ private static async Task MdeEncAlgoDecryptObjectAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + JObject plainTextJObj = new JObject(); foreach (string path in encryptionProperties.EncryptedPaths) { @@ -288,25 +302,31 @@ private static async Task MdeEncAlgoDecryptObjectAsync( } byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); - if (cipherTextWithTypeMarker == null) { continue; } - byte[] cipherText = new byte[cipherTextWithTypeMarker.Length - 1]; - Buffer.BlockCopy(cipherTextWithTypeMarker, 1, cipherText, 0, cipherTextWithTypeMarker.Length - 1); + int plainTextLength = await encryptor.GetDecryptBytesCountAsync( + cipherTextWithTypeMarker.Length - 1, + encryptionProperties.DataEncryptionKeyId, + encryptionProperties.EncryptionAlgorithm); + + byte[] plainText = arrayPoolManager.Rent(plainTextLength); - byte[] plainText = await EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( + int decryptedCount = await EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( encryptionProperties, - cipherText, + cipherTextWithTypeMarker, + cipherTextOffset: 1, + cipherTextWithTypeMarker.Length - 1, + plainText, encryptor, diagnosticsContext, cancellationToken); EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], - plainText, + plainText.AsSpan(0, decryptedCount), plainTextJObj, propertyName); } @@ -340,9 +360,12 @@ private static DecryptionContext CreateDecryptionContext( return decryptionContext; } - private static async Task MdeEncAlgoDecryptPropertyAsync( + private static async Task MdeEncAlgoDecryptPropertyAsync( EncryptionProperties encryptionProperties, byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] buffer, Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) @@ -352,18 +375,22 @@ private static async Task MdeEncAlgoDecryptPropertyAsync( throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } - byte[] plainText = await encryptor.DecryptAsync( + int decryptedCount = await encryptor.DecryptAsync( cipherText, + cipherTextOffset, + cipherTextLength, + buffer, + outputOffset: 0, encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - if (plainText == null) + if (decryptedCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); } - return plainText; + return decryptedCount; } private static async Task LegacyEncAlgoDecryptContentAsync( @@ -483,49 +510,71 @@ private static JObject RetrieveEncryptionProperties( return encryptionPropertiesJObj; } - private static (TypeMarker, byte[]) Serialize(JToken propertyValue) + private static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) { + byte[] buffer; + int length; switch (propertyValue.Type) { case JTokenType.Undefined: Debug.Assert(false, "Undefined value cannot be in the JSON"); - return (default, null); + return (default, null, -1); case JTokenType.Null: Debug.Assert(false, "Null type should have been handled by caller"); - return (TypeMarker.Null, null); + return (TypeMarker.Null, null, -1); case JTokenType.Boolean: - return (TypeMarker.Boolean, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeFixed(SqlBoolSerializer); + return (TypeMarker.Boolean, buffer, length); case JTokenType.Float: - return (TypeMarker.Double, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeFixed(SqlDoubleSerializer); + return (TypeMarker.Double, buffer, length); case JTokenType.Integer: - return (TypeMarker.Long, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeFixed(SqlLongSerializer); + return (TypeMarker.Long, buffer, length); case JTokenType.String: - return (TypeMarker.String, SqlVarCharSerializer.Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeString(propertyValue.ToObject()); + return (TypeMarker.String, buffer, length); case JTokenType.Array: - return (TypeMarker.Array, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Array, buffer, length); case JTokenType.Object: - return (TypeMarker.Object, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Object, buffer, length); default: throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); } + + (byte[] bytes, int length) SerializeFixed(IFixedSizeSerializer serializer) + { + byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); + int bytes = serializer.Serialize(propertyValue.ToObject(), buffer); + return (buffer, bytes); + } + + (byte[] bytes, int length) SerializeString(string value) + { + byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); + int bytes = SqlVarCharSerializer.Serialize(value, buffer); + return (buffer, bytes); + } } private static void DeserializeAndAddProperty( TypeMarker typeMarker, - byte[] serializedBytes, + ReadOnlySpan serializedBytes, JObject jObject, string key) { switch (typeMarker) { case TypeMarker.Boolean: - jObject.Add(key, SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes)); + jObject.Add(key, SqlBoolSerializer.Deserialize(serializedBytes)); break; case TypeMarker.Double: - jObject.Add(key, SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes)); + jObject.Add(key, SqlDoubleSerializer.Deserialize(serializedBytes)); break; case TypeMarker.Long: - jObject.Add(key, SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes)); + jObject.Add(key, SqlLongSerializer.Deserialize(serializedBytes)); break; case TypeMarker.String: jObject.Add(key, SqlVarCharSerializer.Deserialize(serializedBytes)); @@ -587,5 +636,27 @@ await EncryptionProcessor.DecryptAsync( // and corresponding decrypted properties are added back in the documents. return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); } + + internal static int GetOriginalBase64Length(string base64string) + { + if (string.IsNullOrEmpty(base64string)) + { + return 0; + } + + int paddingCount = 0; + int characterCount = base64string.Length; + if (base64string[characterCount - 1] == '=') + { + paddingCount++; + } + + if (base64string[characterCount - 2] == '=') + { + paddingCount++; + } + + return (3 * (characterCount / 4)) - paddingCount; + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 9362c6f2a0..469d069313 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -57,7 +57,7 @@ public abstract Task EncryptAsync( /// Identifier for the encryption algorithm. /// Token for cancellation. /// Size of input when encrypted. - public abstract Task GetEncryptBytesCount( + public abstract Task GetEncryptBytesCountAsync( int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, @@ -107,7 +107,7 @@ public abstract Task DecryptAsync( /// Identifier for the encryption algorithm. /// Token for cancellation. /// Upper bound size of the input when decrypted. - public abstract Task GetDecryptBytesCount( + public abstract Task GetDecryptBytesCountAsync( int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs index 9fffa1e3cb..30c23a3a92 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs @@ -59,7 +59,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist try { - totalSeconds = Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); + totalSeconds = System.Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); } catch { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index d7f8c5a22b..9150a739cf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1807,7 +1807,7 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, @@ -1817,7 +1817,7 @@ public override async Task GetEncryptBytesCount(int plainTextLength, string return dek.GetEncryptByteCount(plainTextLength); } - public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 69924c0a7a..9f63b3545b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2292,7 +2292,7 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, @@ -2302,7 +2302,7 @@ public override async Task GetEncryptBytesCount(int plainTextLength, string return dek.GetEncryptByteCount(plainTextLength); } - public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index d8e4c7d8a2..3655043648 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -35,7 +35,7 @@ public async Task Setup() Mock keyProvider = new(); keyProvider .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); + .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Randomized, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); this.encryptionOptions = CreateEncryptionOptions(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 44aae54dd0..74a360b0bf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **81.01 μs** | **2.595 μs** | **3.804 μs** | **4.7607** | **1.5869** | **-** | **58.87 KB** | -| Decrypt | 1 | 68.96 μs | 1.627 μs | 2.333 μs | 5.0049 | 1.2207 | - | 61.57 KB | -| **Encrypt** | **10** | **190.05 μs** | **5.716 μs** | **8.556 μs** | **16.1133** | **3.9063** | **-** | **197.66 KB** | -| Decrypt | 10 | 215.60 μs | 4.424 μs | 6.484 μs | 24.6582 | 4.8828 | - | 303.41 KB | -| **Encrypt** | **100** | **2,261.05 μs** | **212.952 μs** | **298.529 μs** | **148.4375** | **78.1250** | **74.2188** | **1785.47 KB** | -| Decrypt | 100 | 3,139.31 μs | 316.105 μs | 473.131 μs | 224.6094 | 175.7813 | 130.8594 | 3066.66 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |-----------:|----------:|----------:|-----------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **108.1 μs** | **6.13 μs** | **8.18 μs** | **105.1 μs** | **4.3945** | **1.4648** | **-** | **56.71 KB** | +| Decrypt | 1 | 131.0 μs | 7.17 μs | 10.51 μs | 127.6 μs | 5.3711 | 1.8311 | - | 66.1 KB | +| **Encrypt** | **10** | **277.8 μs** | **13.18 μs** | **18.91 μs** | **282.8 μs** | **14.6484** | **3.9063** | **-** | **185.33 KB** | +| Decrypt | 10 | 486.4 μs | 57.69 μs | 86.34 μs | 438.9 μs | 23.4375 | 5.8594 | - | 287.62 KB | +| **Encrypt** | **100** | **3,187.7 μs** | **309.58 μs** | **443.99 μs** | **3,045.0 μs** | **136.7188** | **70.3125** | **62.5000** | **1670.54 KB** | +| Decrypt | 100 | 4,828.9 μs | 313.12 μs | 468.67 μs | 4,833.8 μs | 218.7500 | 167.9688 | 125.0000 | 2845.11 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 562cc0f0a7..462bc4122c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -38,22 +38,22 @@ public static void ClassInitialize(TestContext testContext) MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((int plainTextLength, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? plainTextLength : throw new InvalidOperationException("DEK not found.")); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] plainText, int plainTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + .ReturnsAsync((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index a0ecdab14e..cf043ac178 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -27,10 +27,10 @@ internal static byte[] EncryptData(byte[] plainText) return plainText.Select(b => (byte)(b + 1)).ToArray(); } - internal static int EncryptData(byte[] plainText, int inputOffset, byte[] output, int outputOffset) + internal static int EncryptData(byte[] plainText, int inputOffset, int inputLength, byte[] output, int outputOffset) { - byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset).ToArray()); - Buffer.BlockCopy(cipherText, 0, output, outputOffset, plainText.Length); + byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset, inputLength).ToArray()); + Buffer.BlockCopy(cipherText, 0, output, outputOffset, cipherText.Length); return cipherText.Length; } @@ -40,9 +40,9 @@ internal static byte[] DecryptData(byte[] cipherText) return cipherText.Select(b => (byte)(b - 1)).ToArray(); } - internal static int DecryptData(byte[] cipherText, int inputOffset, byte[] output, int outputOffset) + internal static int DecryptData(byte[] cipherText, int inputOffset, int inputLength, byte[] output, int outputOffset) { - byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset).ToArray()); + byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset, inputLength).ToArray()); Buffer.BlockCopy(plainText, 0, output, outputOffset, plainText.Length); return plainText.Length; } diff --git a/Microsoft.Azure.Cosmos.lutconfig b/Microsoft.Azure.Cosmos.lutconfig new file mode 100644 index 0000000000..596a860301 --- /dev/null +++ b/Microsoft.Azure.Cosmos.lutconfig @@ -0,0 +1,6 @@ + + + true + true + 180000 + \ No newline at end of file From 1044a89aa79e98ba08d4a40dd8fa5c39f47d76e7 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 19 Sep 2024 16:24:54 +0200 Subject: [PATCH 10/49] Streaming deserialization --- .../src/ArrayPoolManager.cs | 16 +- .../src/EncryptionProcessor.cs | 32 +++- .../src/MemoryTextReader.cs | 161 ++++++++++++++++++ ...soft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../EmulatorTests/MdeCustomEncryptionTests.cs | 2 +- .../Readme.md | 16 +- .../AeadAes256CbcHmac256AlgorithmTests.cs | 43 ----- .../TestCommon.cs | 36 ++-- 8 files changed, 229 insertions(+), 79 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs index b09a919a5f..8cafd07c5d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -8,14 +8,14 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Buffers; using System.Collections.Generic; - internal class ArrayPoolManager : IDisposable + internal class ArrayPoolManager : IDisposable { - private List rentedBuffers = new List(); + private List rentedBuffers = new List(); private bool disposedValue; - public byte[] Rent(int minimumLength) + public T[] Rent(int minimumLength) { - byte[] buffer = ArrayPool.Shared.Rent(minimumLength); + T[] buffer = ArrayPool.Shared.Rent(minimumLength); this.rentedBuffers.Add(buffer); return buffer; } @@ -26,9 +26,9 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - foreach (byte[] buffer in this.rentedBuffers) + foreach (T[] buffer in this.rentedBuffers) { - ArrayPool.Shared.Return(buffer); + ArrayPool.Shared.Return(buffer, clearArray: true); } } @@ -44,4 +44,8 @@ public void Dispose() GC.SuppressFinalize(this); } } + + internal class ArrayPoolManager : ArrayPoolManager + { + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index e7afe78c2a..53478d915d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -544,18 +544,18 @@ private static (TypeMarker typeMarker, byte[] serializedBytes, int serializedByt throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); } - (byte[] bytes, int length) SerializeFixed(IFixedSizeSerializer serializer) + (byte[], int) SerializeFixed(IFixedSizeSerializer serializer) { byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); - int bytes = serializer.Serialize(propertyValue.ToObject(), buffer); - return (buffer, bytes); + int length = serializer.Serialize(propertyValue.ToObject(), buffer); + return (buffer, length); } - (byte[] bytes, int length) SerializeString(string value) + (byte[], int) SerializeString(string value) { byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); - int bytes = SqlVarCharSerializer.Serialize(value, buffer); - return (buffer, bytes); + int length = SqlVarCharSerializer.Serialize(value, buffer); + return (buffer, length); } } @@ -580,15 +580,31 @@ private static void DeserializeAndAddProperty( jObject.Add(key, SqlVarCharSerializer.Deserialize(serializedBytes)); break; case TypeMarker.Array: - jObject.Add(key, JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings)); + DeserializeAndAddProperty(serializedBytes); break; case TypeMarker.Object: - jObject.Add(key, JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings)); + DeserializeAndAddProperty(serializedBytes); break; default: Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); break; } + + void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) + where T : JToken + { + using ArrayPoolManager manager = new ArrayPoolManager(); + + char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); + int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); + + JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); + + using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); + using JsonTextReader reader = new JsonTextReader(memoryTextReader); + + jObject.Add(key, serializer.Deserialize(reader)); + } } private enum TypeMarker : byte diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs new file mode 100644 index 0000000000..b8996ca802 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs @@ -0,0 +1,161 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Diagnostics.Contracts; + using System.IO; + + /// + /// Adjusted implementation of .Net StringReader reading from a Memory instead of a string. + /// + internal class MemoryTextReader : TextReader + { + private Memory chars; + private int length; + private int pos; + private bool closed; + + public MemoryTextReader(Memory chars) + { + this.chars = chars; + this.length = chars.Length; + } + + public override void Close() + { + this.Dispose(true); + } + + protected override void Dispose(bool disposing) + { + this.chars = null; + this.pos = 0; + this.length = 0; + this.closed = true; + base.Dispose(disposing); + } + + [Pure] + public override int Peek() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + if (this.pos == this.length) + { + return -1; + } + + return this.chars.Span[this.pos]; + } + + public override int Read() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + if (this.pos == this.length) + { + return -1; + } + + return this.chars.Span[this.pos++]; + } + + public override int Read(char[] buffer, int index, int count) + { + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } + + if (index < 0) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + if (buffer.Length - index < count) + { + throw new ArgumentOutOfRangeException(); + } + + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + int n = this.length - this.pos; + if (n > 0) + { + if (n > count) + { + n = count; + } + + this.chars.Span.Slice(this.pos, n).CopyTo(buffer.AsSpan(index, n)); + this.pos += n; + } + + return n; + } + + public override string ReadToEnd() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + this.pos = this.length; + return new string(this.chars.Slice(this.pos, this.length - this.pos).ToArray()); + } + + public override string ReadLine() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + int i = this.pos; + while (i < this.length) + { + char ch = this.chars.Span[i]; + if (ch == '\r' || ch == '\n') + { + string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + this.pos = i + 1; + if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') + { + this.pos++; + } + + return result; + } + + i++; + } + + if (i > this.pos) + { + string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + this.pos = i; + return result; + } + + return null; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index ef53408b4a..d940744205 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 9f63b3545b..94bcd21c46 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(32, unwrapcount); + Assert.AreEqual(64, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 74a360b0bf..5d231e7843 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |-----------:|----------:|----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **108.1 μs** | **6.13 μs** | **8.18 μs** | **105.1 μs** | **4.3945** | **1.4648** | **-** | **56.71 KB** | -| Decrypt | 1 | 131.0 μs | 7.17 μs | 10.51 μs | 127.6 μs | 5.3711 | 1.8311 | - | 66.1 KB | -| **Encrypt** | **10** | **277.8 μs** | **13.18 μs** | **18.91 μs** | **282.8 μs** | **14.6484** | **3.9063** | **-** | **185.33 KB** | -| Decrypt | 10 | 486.4 μs | 57.69 μs | 86.34 μs | 438.9 μs | 23.4375 | 5.8594 | - | 287.62 KB | -| **Encrypt** | **100** | **3,187.7 μs** | **309.58 μs** | **443.99 μs** | **3,045.0 μs** | **136.7188** | **70.3125** | **62.5000** | **1670.54 KB** | -| Decrypt | 100 | 4,828.9 μs | 313.12 μs | 468.67 μs | 4,833.8 μs | 218.7500 | 167.9688 | 125.0000 | 2845.11 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |-----------:|----------:|----------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **134.2 μs** | **24.65 μs** | **36.89 μs** | **4.5166** | **1.4648** | **-** | **56.71 KB** | +| Decrypt | 1 | 122.4 μs | 2.57 μs | 3.77 μs | 5.1270 | 1.5869 | - | 64.01 KB | +| **Encrypt** | **10** | **309.7 μs** | **40.78 μs** | **61.04 μs** | **14.6484** | **3.9063** | **-** | **185.33 KB** | +| Decrypt | 10 | 435.5 μs | 28.40 μs | 40.73 μs | 21.4844 | 5.3711 | - | 265.22 KB | +| **Encrypt** | **100** | **3,666.4 μs** | **285.40 μs** | **418.34 μs** | **136.7188** | **70.3125** | **62.5000** | **1670.51 KB** | +| Decrypt | 100 | 4,928.2 μs | 441.88 μs | 661.39 μs | 195.3125 | 121.0938 | 101.5625 | 2617.38 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs deleted file mode 100644 index 329e06e7c5..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ /dev/null @@ -1,43 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Encryption.Tests -{ - using Microsoft.Azure.Cosmos.Encryption.Custom; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using System.Linq; - using System.Text; - - [TestClass] - public class AeadAes256CbcHmac256AlgorithmTests - { - private static readonly byte[] RootKey = new byte[32]; - - private static AeadAes256CbcHmac256EncryptionKey key; - private static AeadAes256CbcHmac256Algorithm algorithm; - - [ClassInitialize] - public static void ClassInitialize(TestContext testContext) - { - AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); - AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); - } - - [TestMethod] - public void EncryptUsingBufferDecryptsSuccessfully() - { - byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 } ; - - int cipherTextLength = algorithm.GetEncryptByteCount(plainTextBytes.Length); - byte[] cipherTextBytes = new byte[cipherTextLength]; - - int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); - Assert.Equals(encrypted, cipherTextLength); - - byte[] decrypted = algorithm.DecryptData(cipherTextBytes); - - Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); - } - } -} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index cf043ac178..81d8683e57 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -70,7 +70,7 @@ private static JObject ParseStream(Stream stream) internal class TestDoc { - public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt" }; + public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt", "/SensitiveArr", "/SensitiveDict" }; [JsonProperty("id")] public string Id { get; set; } @@ -83,17 +83,12 @@ internal class TestDoc public int SensitiveInt { get; set; } - public TestDoc() - { - } + public string[] SensitiveArr { get; set; } - public TestDoc(TestDoc other) + public Dictionary SensitiveDict { get; set; } + + public TestDoc() { - this.Id = other.Id; - this.PK = other.PK; - this.NonSensitive = other.NonSensitive; - this.SensitiveStr = other.SensitiveStr; - this.SensitiveInt = other.SensitiveInt; } public override bool Equals(object obj) @@ -103,7 +98,9 @@ public override bool Equals(object obj) && this.PK == doc.PK && this.NonSensitive == doc.NonSensitive && this.SensitiveInt == doc.SensitiveInt - && this.SensitiveStr == this.SensitiveStr; + && this.SensitiveStr == doc.SensitiveStr + && this.SensitiveArr?.Equals(doc.SensitiveArr) == true + && this.SensitiveDict?.Equals(doc.SensitiveDict) == true; } public override int GetHashCode() @@ -114,6 +111,8 @@ public override int GetHashCode() hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.NonSensitive); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveStr); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveInt); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveArr); + hashCode = (hashCode * -1521134295) + EqualityComparer>.Default.GetHashCode(this.SensitiveDict); return hashCode; } @@ -125,7 +124,20 @@ public static TestDoc Create(string partitionKey = null) PK = partitionKey ?? Guid.NewGuid().ToString(), NonSensitive = Guid.NewGuid().ToString(), SensitiveStr = Guid.NewGuid().ToString(), - SensitiveInt = new Random().Next() + SensitiveInt = new Random().Next(), + SensitiveArr = new string[] + { + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + }, + SensitiveDict = new Dictionary + { + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + } }; } From 05bfc508daa7f673c3da34a0887ccbb88f77b8eb Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Fri, 20 Sep 2024 09:12:17 +0200 Subject: [PATCH 11/49] Cleanup --- .../src/Mirrored/UnixDateTimeConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs index 30c23a3a92..9fffa1e3cb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs @@ -59,7 +59,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist try { - totalSeconds = System.Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); + totalSeconds = Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); } catch { From 5629f74a2a12074ad1fcf8968eb175e35a5a54da Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Fri, 20 Sep 2024 10:06:31 +0200 Subject: [PATCH 12/49] Update MDE and rerun benchmarks --- .../Microsoft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../Readme.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index ed4cca0596..ef53408b4a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 2807f8c809..2d388b505d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **61.45 μs** | **1.676 μs** | **2.457 μs** | **4.9438** | **1.2207** | **-** | **61.25 KB** | -| Decrypt | 1 | 77.89 μs | 1.959 μs | 2.933 μs | 5.7373 | 1.4648 | - | 71.22 KB | -| **Encrypt** | **10** | **171.64 μs** | **3.341 μs** | **4.791 μs** | **21.2402** | **3.6621** | **-** | **260.97 KB** | -| Decrypt | 10 | 255.57 μs | 7.833 μs | 11.724 μs | 29.2969 | 4.3945 | - | 363.84 KB | -| **Encrypt** | **100** | **2,601.33 μs** | **215.481 μs** | **322.522 μs** | **199.2188** | **125.0000** | **123.0469** | **2464.88 KB** | -| Decrypt | 100 | 3,156.06 μs | 321.419 μs | 481.084 μs | 355.4688 | 300.7813 | 261.7188 | 3413.05 KB | +| **Encrypt** | **1** | **44.46 μs** | **0.661 μs** | **0.969 μs** | **4.2114** | **1.0376** | **-** | **51.96 KB** | +| Decrypt | 1 | 56.00 μs | 1.062 μs | 1.589 μs | 5.0049 | 1.2817 | - | 61.57 KB | +| **Encrypt** | **10** | **131.08 μs** | **0.893 μs** | **1.309 μs** | **16.3574** | **3.1738** | **-** | **200.9 KB** | +| Decrypt | 10 | 174.88 μs | 3.443 μs | 4.938 μs | 24.6582 | 4.8828 | - | 303.41 KB | +| **Encrypt** | **100** | **2,052.43 μs** | **230.487 μs** | **344.982 μs** | **160.1563** | **107.4219** | **83.9844** | **1891.44 KB** | +| Decrypt | 100 | 2,791.54 μs | 284.376 μs | 425.641 μs | 234.3750 | 166.0156 | 140.6250 | 3066.91 KB | From 495d2c4d317c18f03c94f31fb6792d1f947fb8d0 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 11:11:13 +0200 Subject: [PATCH 13/49] Add non-allocating APIs to encryptors --- .../src/AeadAes256CbcHmac256Algorithm.cs | 45 ++++++++++++ .../src/CosmosEncryptor.cs | 64 +++++++++++++++++ .../src/DataEncryptionKey.cs | 36 ++++++++++ .../src/EncryptionProcessor.cs | 19 +++-- .../src/Encryptor.cs | 72 +++++++++++++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 34 +++++++-- .../EmulatorTests/LegacyEncryptionTests.cs | 50 +++++++++++++ .../EmulatorTests/MdeCustomEncryptionTests.cs | 52 +++++++++++++- .../Readme.md | 12 ++-- .../AeadAes256CbcHmac256AlgorithmTests.cs | 43 +++++++++++ .../MdeEncryptionProcessorTests.cs | 19 ++++- .../TestCommon.cs | 58 ++++++++++----- 12 files changed, 465 insertions(+), 39 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index df416c3efd..a51e09726b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -26,11 +26,21 @@ internal class AeadAes256CbcHmac256Algorithm : DataEncryptionKey /// private const int KeySizeInBytes = AeadAes256CbcHmac256EncryptionKey.KeySize / 8; + /// + /// Authentication tag size in bytes + /// + private const int AuthenticationTagSizeInBytes = KeySizeInBytes; + /// /// Block size in bytes. AES uses 16 byte blocks. /// private const int BlockSizeInBytes = 16; + /// + /// Size of Initialization Vector in bytes. + /// + private const int IvSizeInBytes = 16; + /// /// Minimum Length of cipherText without authentication tag. This value is 1 (version byte) + 16 (IV) + 16 (minimum of 1 block of cipher Text) /// @@ -137,6 +147,20 @@ public override byte[] EncryptData(byte[] plainText) return this.EncryptData(plainText, hasAuthenticationTag: true); } + /// + /// Encryption Algorithm + /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits + /// cell_ciphertext = AES-CBC-256(enc_key, cell_iv, cell_data) with PKCS7 padding. + /// cell_tag = HMAC_SHA-2-256(mac_key, versionbyte + cell_iv + cell_ciphertext + versionbyte_length) + /// cell_blob = versionbyte + cell_tag + cell_iv + cell_ciphertext + /// + /// Plaintext data to be encrypted + /// Returns the ciphertext corresponding to the plaintext. + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } + /// /// Encryption Algorithm /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits @@ -418,5 +442,26 @@ private byte[] PrepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset Buffer.BlockCopy(computedHash, 0, authenticationTag, 0, authenticationTag.Length); return authenticationTag; } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } + + public override int GetEncryptByteCount(int plainTextLength) + { + // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. + return sizeof(byte) + AuthenticationTagSizeInBytes + IvSizeInBytes + GetCipherTextLength(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + throw new NotImplementedException(); + } + + private static int GetCipherTextLength(int inputSize) + { + return ((inputSize / BlockSizeInBytes) + 1) * BlockSizeInBytes; + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 462bd56a1f..0288ee79db 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -48,6 +48,22 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + /// + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + /// public override async Task EncryptAsync( byte[] plainText, @@ -67,5 +83,53 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + /// + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + /// + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetEncryptByteCount(plainTextLength); + } + + /// + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetDecryptByteCount(cipherTextLength); + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index bcee51d0e1..65890ec941 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -29,6 +29,24 @@ public abstract class DataEncryptionKey /// Encrypted value. public abstract byte[] EncryptData(byte[] plainText); + /// + /// Encrypts the plainText with a data encryption key. + /// + /// Plain text value to be encrypted. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Encrypted value. + public abstract int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset); + + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Size of input when encrypted. + public abstract int GetEncryptByteCount(int plainTextLength); + /// /// Decrypts the cipherText with a data encryption key. /// @@ -36,6 +54,24 @@ public abstract class DataEncryptionKey /// Plain text. public abstract byte[] DecryptData(byte[] cipherText); + /// + /// Decrypts the cipherText with a data encryption key. + /// + /// Ciphertext value to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Plain text. + public abstract int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset); + + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Upper bound size of the input when decrypted. + public abstract int GetDecryptByteCount(int cipherTextLength); + /// /// Generates raw data encryption key bytes suitable for use with the provided encryption algorithm. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index af10de5e11..8b4da86a93 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -99,19 +99,28 @@ public static async Task EncryptAsync( (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); - cipherText = await encryptor.EncryptAsync( + int cipherTextLength = await encryptor.GetEncryptBytesCountAsync( + plainText.Length, + encryptionOptions.DataEncryptionKeyId, + encryptionOptions.EncryptionAlgorithm); + + byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; + cipherTextWithTypeMarker[0] = (byte)typeMarker; + + int encryptedBytesCount = await encryptor.EncryptAsync( plainText, + plainTextOffset: 0, + plainTextLength: plainText.Length, + cipherTextWithTypeMarker, + outputOffset: 1, encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - if (cipherText == null) + if (encryptedBytesCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - byte[] cipherTextWithTypeMarker = new byte[cipherText.Length + 1]; - cipherTextWithTypeMarker[0] = (byte)typeMarker; - Buffer.BlockCopy(cipherText, 0, cipherTextWithTypeMarker, 1, cipherText.Length); itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 1e7f272ce5..469d069313 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -27,6 +27,42 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Encrypts the plainText using the key and algorithm provided. + /// + /// Plain text. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Cipher text. + public abstract Task EncryptAsync( + byte[] plainText, + int plainTextOffset, + int plainTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Size of input when encrypted. + public abstract Task GetEncryptBytesCountAsync( + int plainTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + /// /// Decrypts the cipherText using the key and algorithm provided. /// @@ -40,5 +76,41 @@ public abstract Task DecryptAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + + /// + /// Decrypts the cipherText using the key and algorithm provided. + /// + /// Ciphertext to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Plain text. + public abstract Task DecryptAsync( + byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Upper bound size of the input when decrypted. + public abstract Task GetDecryptBytesCountAsync( + int cipherTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 68d863114e..ba606e9412 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -12,6 +12,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal sealed class MdeEncryptionAlgorithm : DataEncryptionKey { + private const byte Version = 1; + private readonly AeadAes256CbcHmac256EncryptionAlgorithm mdeAeadAes256CbcHmac256EncryptionAlgorithm; private readonly byte[] unwrapKey; @@ -65,7 +67,8 @@ public MdeEncryptionAlgorithm( dekProperties.WrappedDataEncryptionKey); this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( protectedDataEncryptionKey, - encryptionType); + encryptionType, + Version); } else { @@ -80,11 +83,9 @@ public MdeEncryptionAlgorithm( this.RawKey = rawKey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( plaintextDataEncryptionKey, - encryptionType); - + encryptionType, + Version); } - - } /// @@ -103,7 +104,8 @@ public MdeEncryptionAlgorithm( this.RawKey = rawkey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( dataEncryptionKey, - encryptionType); + encryptionType, + Version); } /// @@ -125,5 +127,25 @@ public override byte[] DecryptData(byte[] cipherText) { return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText); } + + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Encrypt(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + + public override int GetEncryptByteCount(int plainTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetEncryptByteCount(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetDecryptByteCount(cipherTextLength); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index fd83ef528d..9150a739cf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1763,6 +1763,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -1776,6 +1796,36 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 85b7bf3f36..cb20c71c58 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(32, unwrapcount); + Assert.AreEqual(48, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); @@ -2248,6 +2248,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -2261,6 +2281,36 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 2d388b505d..a4921705bc 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **44.46 μs** | **0.661 μs** | **0.969 μs** | **4.2114** | **1.0376** | **-** | **51.96 KB** | -| Decrypt | 1 | 56.00 μs | 1.062 μs | 1.589 μs | 5.0049 | 1.2817 | - | 61.57 KB | -| **Encrypt** | **10** | **131.08 μs** | **0.893 μs** | **1.309 μs** | **16.3574** | **3.1738** | **-** | **200.9 KB** | -| Decrypt | 10 | 174.88 μs | 3.443 μs | 4.938 μs | 24.6582 | 4.8828 | - | 303.41 KB | -| **Encrypt** | **100** | **2,052.43 μs** | **230.487 μs** | **344.982 μs** | **160.1563** | **107.4219** | **83.9844** | **1891.44 KB** | -| Decrypt | 100 | 2,791.54 μs | 284.376 μs | 425.641 μs | 234.3750 | 166.0156 | 140.6250 | 3066.91 KB | +| **Encrypt** | **1** | **60.61 μs** | **0.554 μs** | **0.758 μs** | **4.6997** | **1.5869** | **-** | **57.72 KB** | +| Decrypt | 1 | 61.02 μs | 0.984 μs | 1.473 μs | 5.0049 | 1.2817 | - | 61.57 KB | +| **Encrypt** | **10** | **158.70 μs** | **3.114 μs** | **4.565 μs** | **15.8691** | **3.9063** | **-** | **196.51 KB** | +| Decrypt | 10 | 189.60 μs | 1.248 μs | 1.829 μs | 24.6582 | 4.8828 | - | 303.41 KB | +| **Encrypt** | **100** | **1,773.67 μs** | **135.590 μs** | **202.944 μs** | **117.1875** | **50.7813** | **41.0156** | **1784.35 KB** | +| Decrypt | 100 | 2,787.37 μs | 264.329 μs | 395.636 μs | 230.4688 | 158.2031 | 136.7188 | 3067.03 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs new file mode 100644 index 0000000000..329e06e7c5 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -0,0 +1,43 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Tests +{ + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using System.Linq; + using System.Text; + + [TestClass] + public class AeadAes256CbcHmac256AlgorithmTests + { + private static readonly byte[] RootKey = new byte[32]; + + private static AeadAes256CbcHmac256EncryptionKey key; + private static AeadAes256CbcHmac256Algorithm algorithm; + + [ClassInitialize] + public static void ClassInitialize(TestContext testContext) + { + AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); + AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); + } + + [TestMethod] + public void EncryptUsingBufferDecryptsSuccessfully() + { + byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 } ; + + int cipherTextLength = algorithm.GetEncryptByteCount(plainTextBytes.Length); + byte[] cipherTextBytes = new byte[cipherTextLength]; + + int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); + Assert.Equals(encrypted, cipherTextLength); + + byte[] decrypted = algorithm.DecryptData(cipherTextBytes); + + Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 1396e7e06e..95a5b66d39 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -24,7 +24,7 @@ public class MdeEncryptionProcessorTests private const string dekId = "dekId"; [ClassInitialize] - public static void ClassInitilize(TestContext testContext) + public static void ClassInitialize(TestContext testContext) { _ = testContext; MdeEncryptionProcessorTests.encryptionOptions = new EncryptionOptions() @@ -38,9 +38,22 @@ public static void ClassInitilize(TestContext testContext) MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int plainTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? plainTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => + .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] @@ -108,7 +121,7 @@ await EncryptionProcessor.EncryptAsync( [TestMethod] public async Task EncryptDecryptPropertyWithNullValue() - { + { TestDoc testDoc = TestDoc.Create(); testDoc.SensitiveStr = null; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index 63968723e6..1d90625bc9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -27,11 +27,26 @@ internal static byte[] EncryptData(byte[] plainText) return plainText.Select(b => (byte)(b + 1)).ToArray(); } + internal static int EncryptData(byte[] plainText, int inputOffset, int inputLength, byte[] output, int outputOffset) + { + byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset, inputLength).ToArray()); + Buffer.BlockCopy(cipherText, 0, output, outputOffset, cipherText.Length); + + return cipherText.Length; + } + internal static byte[] DecryptData(byte[] cipherText) { return cipherText.Select(b => (byte)(b - 1)).ToArray(); } - + + internal static int DecryptData(byte[] cipherText, int inputOffset, int inputLength, byte[] output, int outputOffset) + { + byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset, inputLength).ToArray()); + Buffer.BlockCopy(plainText, 0, output, outputOffset, plainText.Length); + return plainText.Length; + } + internal static Stream ToStream(T input) { string s = JsonConvert.SerializeObject(input); @@ -48,14 +63,9 @@ internal static T FromStream(Stream stream) } } - private static JObject ParseStream(Stream stream) - { - return JObject.Load(new JsonTextReader(new StreamReader(stream))); - } - internal class TestDoc { - public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt" }; + public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt", "/SensitiveArr", "/SensitiveDict" }; [JsonProperty("id")] public string Id { get; set; } @@ -68,17 +78,12 @@ internal class TestDoc public int SensitiveInt { get; set; } - public TestDoc() - { - } + public string[] SensitiveArr { get; set; } - public TestDoc(TestDoc other) + public Dictionary SensitiveDict { get; set; } + + public TestDoc() { - this.Id = other.Id; - this.PK = other.PK; - this.NonSensitive = other.NonSensitive; - this.SensitiveStr = other.SensitiveStr; - this.SensitiveInt = other.SensitiveInt; } public override bool Equals(object obj) @@ -88,7 +93,9 @@ public override bool Equals(object obj) && this.PK == doc.PK && this.NonSensitive == doc.NonSensitive && this.SensitiveInt == doc.SensitiveInt - && this.SensitiveStr == this.SensitiveStr; + && this.SensitiveStr == doc.SensitiveStr + && this.SensitiveArr?.Equals(doc.SensitiveArr) == true + && this.SensitiveDict?.Equals(doc.SensitiveDict) == true; } public override int GetHashCode() @@ -99,6 +106,8 @@ public override int GetHashCode() hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.NonSensitive); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveStr); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveInt); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveArr); + hashCode = (hashCode * -1521134295) + EqualityComparer>.Default.GetHashCode(this.SensitiveDict); return hashCode; } @@ -110,7 +119,20 @@ public static TestDoc Create(string partitionKey = null) PK = partitionKey ?? Guid.NewGuid().ToString(), NonSensitive = Guid.NewGuid().ToString(), SensitiveStr = Guid.NewGuid().ToString(), - SensitiveInt = new Random().Next() + SensitiveInt = new Random().Next(), + SensitiveArr = new string[] + { + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + }, + SensitiveDict = new Dictionary + { + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + } }; } From 14bce371afb7208d4381bd0312e01154ca154313 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 12:16:05 +0200 Subject: [PATCH 14/49] ~ drop repeated DEK calls ~ reduce validations overhead --- .../src/CosmosEncryptor.cs | 104 +++++++++--------- .../src/EncryptionProcessor.cs | 19 ++-- .../src/Encryptor.cs | 10 ++ 3 files changed, 71 insertions(+), 62 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 0288ee79db..2ded90893b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -29,8 +29,7 @@ public CosmosEncryptor(DataEncryptionKeyProvider dataEncryptionKeyProvider) } /// - public override async Task DecryptAsync( - byte[] cipherText, + public override async Task GetEncryptionKeyAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) @@ -45,91 +44,94 @@ public override async Task DecryptAsync( throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); } - return dek.DecryptData(cipherText); + return dek; } /// - public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.")] + public override async Task DecryptAsync( + byte[] cipherText, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + return dek.DecryptData(cipherText); + } + + /// + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.")] + public override async Task DecryptAsync( + byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); } /// + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.")] public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.EncryptData(plainText); } /// - public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.")] + public override async Task EncryptAsync( + byte[] plainText, + int plainTextOffset, + int plainTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } /// - public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.")] + public override async Task GetEncryptBytesCountAsync( + int plainTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.GetEncryptByteCount(plainTextLength); } /// - public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.")] + public override async Task GetDecryptBytesCountAsync( + int cipherTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.GetDecryptByteCount(cipherTextLength); } + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index a0ea6fa213..a964a04d6d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -55,19 +55,19 @@ public static async Task EncryptAsync( return input; } - if (!encryptionOptions.PathsToEncrypt.Distinct().SequenceEqual(encryptionOptions.PathsToEncrypt)) + if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) { throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); } foreach (string path in encryptionOptions.PathsToEncrypt) { - if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf('/') != 0) + if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf(',', 1) != -1) { throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); } - if (string.Equals(path.Substring(1), "id")) + if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.InvariantCulture)) { throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); } @@ -99,22 +99,19 @@ public static async Task EncryptAsync( (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); - int cipherTextLength = await encryptor.GetEncryptBytesCountAsync( - plainText.Length, - encryptionOptions.DataEncryptionKeyId, - encryptionOptions.EncryptionAlgorithm); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); + + int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; cipherTextWithTypeMarker[0] = (byte)typeMarker; - int encryptedBytesCount = await encryptor.EncryptAsync( + int encryptedBytesCount = encryptionKey.EncryptData( plainText, plainTextOffset: 0, plainTextLength: plainText.Length, cipherTextWithTypeMarker, - outputOffset: 1, - encryptionOptions.DataEncryptionKeyId, - encryptionOptions.EncryptionAlgorithm); + outputOffset: 1); if (encryptedBytesCount < 0) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 469d069313..2fd857eeb4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -49,12 +49,22 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Retrieve Data Encryption Key. + /// + /// Identifier of the data encryption key. + /// Identifier of the encryption algorithm. + /// Token for cancellation. + /// Data Encryption Key + public abstract Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// /// Calculate size of input after encryption. /// /// Input data size. /// Identifier of the data encryption key. /// Identifier for the encryption algorithm. + /// Data Encryption Key used. /// Token for cancellation. /// Size of input when encrypted. public abstract Task GetEncryptBytesCountAsync( From 4ff1601869dbc2801c6f57e24573d00a1dde4ab8 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 12:22:48 +0200 Subject: [PATCH 15/49] ! typo --- Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 2ded90893b..7e705453ce 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -121,7 +121,7 @@ public override async Task GetEncryptBytesCountAsync( } /// - [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.")] + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.")] public override async Task GetDecryptBytesCountAsync( int cipherTextLength, string dataEncryptionKeyId, From d8a345cc26c744a07ef68f22c69e0668679f8d1c Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 12:24:47 +0200 Subject: [PATCH 16/49] ~ update benchmark --- .../Readme.md | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index f0166dd8b8..dd87b34c36 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -1,20 +1,19 @@ -``` ini +``` ini BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=8.0.108 +.NET SDK=9.0.100-rc.1.24452.12 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **36.44 μs** | **0.518 μs** | **0.759 μs** | **36.40 μs** | **4.0894** | **1.0376** | **-** | **50.81 KB** | -| Decrypt | 1 | 44.51 μs | 0.900 μs | 1.291 μs | 43.83 μs | 4.8828 | 1.2207 | - | 60.1 KB | -| **Encrypt** | **10** | **114.63 μs** | **3.234 μs** | **4.841 μs** | **113.80 μs** | **16.2354** | **3.2959** | **-** | **199.75 KB** | -| Decrypt | 10 | 146.56 μs | 1.205 μs | 1.766 μs | 146.08 μs | 24.4141 | 4.6387 | - | 301.94 KB | -| **Encrypt** | **100** | **1,784.59 μs** | **180.928 μs** | **270.805 μs** | **1,743.86 μs** | **158.2031** | **95.7031** | **82.0313** | **1890.27 KB** | -| Decrypt | 100 | 2,772.68 μs | 261.923 μs | 392.035 μs | 2,648.81 μs | 236.3281 | 158.2031 | 142.5781 | 3064.91 KB | - +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|--------:|-----------:| +| **Encrypt** | **1** | **37.87 μs** | **0.699 μs** | **0.979 μs** | **3.7842** | **0.9766** | **-** | **47.03 KB** | +| Decrypt | 1 | 47.27 μs | 1.148 μs | 1.683 μs | 4.3945 | 1.0986 | - | 54.17 KB | +| **Encrypt** | **10** | **113.49 μs** | **1.380 μs** | **1.934 μs** | **15.1367** | **3.0518** | **-** | **185.81 KB** | +| Decrypt | 10 | 146.93 μs | 1.724 μs | 2.581 μs | 19.5313 | 2.1973 | - | 239.94 KB | +| **Encrypt** | **100** | **1,610.57 μs** | **161.738 μs** | **242.081 μs** | **150.3906** | **107.4219** | **74.2188** | **1773.64 KB** | +| Decrypt | 100 | 2,058.97 μs | 202.464 μs | 303.039 μs | 160.1563 | 107.4219 | 76.1719 | 2042.61 KB | From 03c06e05ce457d3a0fc5c24d09d0d5ec53cc8c79 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 16:22:22 +0200 Subject: [PATCH 17/49] ~ fix tests ~ update api file --- .../src/AeadAes256CbcHmac256Algorithm.cs | 10 +- .../EmulatorTests/LegacyEncryptionTests.cs | 39 ++++- .../EmulatorTests/MdeCustomEncryptionTests.cs | 104 ++++++------- .../AeadAes256CbcHmac256AlgorithmTests.cs | 2 +- .../DotNetSDKEncryptionCustomAPI.json | 139 +++++++++++++++++- .../MdeEncryptionProcessorTests.cs | 13 ++ .../Contracts/ContractEnforcement.cs | 2 +- 7 files changed, 235 insertions(+), 74 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index a51e09726b..903e620207 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -158,7 +158,15 @@ public override byte[] EncryptData(byte[] plainText) /// Returns the ciphertext corresponding to the plaintext. public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) { - throw new NotImplementedException(); + var buffer = this.EncryptData(plainText.AsSpan(plainTextOffset, plainTextLength).ToArray()); + + if (buffer.Length > output.Length-outputOffset) + { + throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes"); + } + + buffer.CopyTo(output, outputOffset); + return buffer.Length; } /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 9150a739cf..6f5b798ed3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -922,7 +922,7 @@ public async Task EncryptionTransactionalBatchWithCustomSerializer() await LegacyEncryptionTests.VerifyItemByReadAsync(LegacyEncryptionTests.itemContainer, doc1ToReplace); } - [TestMethod] + [TestMethod] public async Task VerifyDekOperationWithSystemTextSerializer() { System.Text.Json.JsonSerializerOptions jsonSerializerOptions = new System.Text.Json.JsonSerializerOptions() @@ -1037,6 +1037,23 @@ await LegacyEncryptionTests.IterateDekFeedAsync( isResultOrderExpected: false, "SELECT * from c", itemCountInPage: 3); + + + //clean up + FeedIterator iterator = containerWithCosmosSystemTextJsonSerializer.GetItemQueryIterator(); + + while (iterator.HasMoreResults) + { + FeedResponse feedResponse = await iterator.ReadNextAsync(); + foreach (TestDocSystemText testDoc in feedResponse) + { + if (testDoc.Id == null) + { + continue; + } + await containerWithCosmosSystemTextJsonSerializer.DeleteItemAsync(testDoc.Id, new PartitionKey(testDoc.PartitionKey)); + } + } } [TestMethod] @@ -1750,7 +1767,7 @@ public override async Task DecryptAsync( throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); } - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1770,7 +1787,7 @@ public override async Task DecryptAsync(byte[] cipherText, int cipherTextOf throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); } - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1789,7 +1806,7 @@ public override async Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1799,7 +1816,7 @@ public override async Task EncryptAsync( public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1809,7 +1826,7 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1819,13 +1836,21 @@ public override async Task GetEncryptBytesCountAsync(int plainTextLength, s public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.GetDecryptByteCount(cipherTextLength); } + + public override async Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + return await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index cb20c71c58..d5f27926fd 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -69,9 +69,9 @@ public static async Task ClassInitialize(TestContext context) MdeCustomEncryptionTests.encryptor = new TestEncryptor(MdeCustomEncryptionTests.dekProvider); MdeCustomEncryptionTests.encryptionContainer = MdeCustomEncryptionTests.itemContainer.WithEncryptor(encryptor); MdeCustomEncryptionTests.encryptionContainerForChangeFeed = MdeCustomEncryptionTests.itemContainerForChangeFeed.WithEncryptor(encryptor); + await MdeCustomEncryptionTests.dekProvider.InitializeAsync(MdeCustomEncryptionTests.database, MdeCustomEncryptionTests.keyContainer.Id); MdeCustomEncryptionTests.dekProperties = await MdeCustomEncryptionTests.CreateDekAsync(MdeCustomEncryptionTests.dekProvider, MdeCustomEncryptionTests.dekId); - } [ClassCleanup] @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(48, unwrapcount); + Assert.AreEqual(32, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); @@ -1153,6 +1153,22 @@ await MdeCustomEncryptionTests.IterateDekFeedAsync( isResultOrderExpected: false, "SELECT * from c", itemCountInPage: 3); + + // cleanup + FeedIterator iterator = containerWithCosmosSystemTextJsonSerializer.GetItemQueryIterator(); + + while (iterator.HasMoreResults) + { + FeedResponse feedResponse = await iterator.ReadNextAsync(); + foreach (TestDocSystemText testDoc in feedResponse) + { + if (testDoc.Id == null) + { + continue; + } + await containerWithCosmosSystemTextJsonSerializer.DeleteItemAsync(testDoc.Id, new PartitionKey(testDoc.PartitionKey)); + } + } } [TestMethod] @@ -2218,54 +2234,36 @@ private class TestEncryptor : Encryptor public DataEncryptionKeyProvider DataEncryptionKeyProvider { get; } public bool FailDecryption { get; set; } + private readonly CosmosEncryptor encryptor; + public TestEncryptor(DataEncryptionKeyProvider dataEncryptionKeyProvider) { - this.DataEncryptionKeyProvider = dataEncryptionKeyProvider; + this.encryptor = new CosmosEncryptor(dataEncryptionKeyProvider); this.FailDecryption = false; } - public override async Task DecryptAsync( - byte[] cipherText, - string dataEncryptionKeyId, - string encryptionAlgorithm, - CancellationToken cancellationToken = default) + private void ThrowIfFail(string dataEncryptionKeyId) { if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) { throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); } + } - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText); + public override async Task DecryptAsync( + byte[] cipherText, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) + { + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.DecryptAsync(cipherText, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); - } - - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.DecryptAsync(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task EncryptAsync( @@ -2274,42 +2272,32 @@ public override async Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.EncryptData(plainText); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.EncryptAsync(plainText, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.EncryptAsync(plainText, plainTextOffset, plainTextLength, output, outputOffset, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.GetEncryptByteCount(plainTextLength); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.GetEncryptBytesCountAsync(plainTextLength, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.GetDecryptBytesCountAsync(cipherTextLength, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); + } - return dek.GetDecryptByteCount(cipherTextLength); + public override async Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs index 329e06e7c5..9c2125f903 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -33,7 +33,7 @@ public void EncryptUsingBufferDecryptsSuccessfully() byte[] cipherTextBytes = new byte[cipherTextLength]; int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); - Assert.Equals(encrypted, cipherTextLength); + Assert.AreEqual(encrypted, cipherTextLength); byte[] decrypted = algorithm.DecryptData(cipherTextBytes); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json index 1912d16407..ccb4b56fff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json @@ -113,20 +113,61 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider get_DataEncryptionKeyProvider();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { "Type": "Method", "Attributes": [ "AsyncStateMachineAttribute" ], + "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -163,6 +204,26 @@ "Attributes": [], "MethodInfo": "Byte[] RawKey;CanRead:True;CanWrite:False;Byte[] get_RawKey();IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "Int32 DecryptData(Byte[], Int32, Int32, Byte[], Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 DecryptData(Byte[], Int32, Int32, Byte[], Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Int32 EncryptData(Byte[], Int32, Int32, Byte[], Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 EncryptData(Byte[], Int32, Int32, Byte[], Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Int32 GetDecryptByteCount(Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 GetDecryptByteCount(Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Int32 GetEncryptByteCount(Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 GetEncryptByteCount(Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey Create(Byte[], System.String)": { "Type": "Method", "Attributes": [], @@ -1064,20 +1125,61 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider get_DataEncryptionKeyProvider();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { "Type": "Method", "Attributes": [ "AsyncStateMachineAttribute" ], + "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -1088,6 +1190,11 @@ } }, "Members": { + "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)": { "Type": "Method", "Attributes": [], @@ -1097,6 +1204,26 @@ "Type": "Method", "Attributes": [], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" } }, "NestedTypes": {} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 95a5b66d39..aec838a413 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -34,7 +34,20 @@ public static void ClassInitialize(TestContext testContext) PathsToEncrypt = TestDoc.PathsToEncrypt }; + var DekMock = new Mock(); + DekMock.Setup(m => m.EncryptData(It.IsAny())) + .Returns((byte[] plainText) => TestCommon.EncryptData(plainText)); + DekMock.Setup(m => m.GetEncryptByteCount(It.IsAny())) + .Returns((int plainTextLength) => plainTextLength); + DekMock.Setup(m => m.EncryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + + MdeEncryptionProcessorTests.mockEncryptor = new Mock(); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((string dekId, string algorithm, CancellationToken token) => + dekId == MdeEncryptionProcessorTests.dekId ? DekMock.Object : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs index f0e127495a..c58109d65b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs @@ -200,7 +200,7 @@ public static void ValidateContractContainBreakingChanges( File.WriteAllText($"Contracts/{breakingChangesPath}", localJson); string baselineJson = GetBaselineContract(baselinePath); - ContractEnforcement.ValidateJsonAreSame(localJson, baselineJson); + ContractEnforcement.ValidateJsonAreSame(baselineJson, localJson); } public static void ValidateTelemetryContractContainBreakingChanges( From 3bf77c89a6c571095d6b95b40e8649afee7796b6 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 11:27:54 +0200 Subject: [PATCH 18/49] ~ cleanup --- .../src/AeadAes256CbcHmac256Algorithm.cs | 24 +++++++++++++++---- .../src/EncryptionProcessor.cs | 6 ++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index 903e620207..e50c60e8fd 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -158,11 +158,11 @@ public override byte[] EncryptData(byte[] plainText) /// Returns the ciphertext corresponding to the plaintext. public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) { - var buffer = this.EncryptData(plainText.AsSpan(plainTextOffset, plainTextLength).ToArray()); + byte[] buffer = this.EncryptData(plainText.AsSpan(plainTextOffset, plainTextLength).ToArray()); - if (buffer.Length > output.Length-outputOffset) + if (buffer.Length > output.Length - outputOffset) { - throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes"); + throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes."); } buffer.CopyTo(output, outputOffset); @@ -453,7 +453,15 @@ private byte[] PrepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) { - throw new NotImplementedException(); + byte[] buffer = this.DecryptData(cipherText.AsSpan(cipherTextOffset, cipherTextLength).ToArray(), true); + + if (buffer.Length > output.Length - outputOffset) + { + throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes"); + } + + buffer.CopyTo(output, outputOffset); + return buffer.Length; } public override int GetEncryptByteCount(int plainTextLength) @@ -464,7 +472,13 @@ public override int GetEncryptByteCount(int plainTextLength) public override int GetDecryptByteCount(int cipherTextLength) { - throw new NotImplementedException(); + int value = cipherTextLength - (sizeof(byte) + AuthenticationTagSizeInBytes + IvSizeInBytes); + if (value < BlockSizeInBytes) + { + throw new ArgumentOutOfRangeException(nameof(cipherTextLength), $"Cipher text length is too short."); + } + + return value; } private static int GetCipherTextLength(int inputSize) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index a964a04d6d..a6ee23b4e5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -55,19 +55,19 @@ public static async Task EncryptAsync( return input; } - if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) + if (!encryptionOptions.PathsToEncrypt.Distinct().SequenceEqual(encryptionOptions.PathsToEncrypt)) { throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); } foreach (string path in encryptionOptions.PathsToEncrypt) { - if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf(',', 1) != -1) + if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf('/') != 0) { throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); } - if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.InvariantCulture)) + if (string.Equals(path.Substring(1), "id")) { throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); } From ceaa8b5ee346a3ad779a79521cf95da90176f4d9 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 11:39:52 +0200 Subject: [PATCH 19/49] + refresh benchmark --- .../Readme.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index dd87b34c36..64a686244c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -2,7 +2,7 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=9.0.100-rc.1.24452.12 +.NET SDK=8.0.400 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|--------:|-----------:| -| **Encrypt** | **1** | **37.87 μs** | **0.699 μs** | **0.979 μs** | **3.7842** | **0.9766** | **-** | **47.03 KB** | -| Decrypt | 1 | 47.27 μs | 1.148 μs | 1.683 μs | 4.3945 | 1.0986 | - | 54.17 KB | -| **Encrypt** | **10** | **113.49 μs** | **1.380 μs** | **1.934 μs** | **15.1367** | **3.0518** | **-** | **185.81 KB** | -| Decrypt | 10 | 146.93 μs | 1.724 μs | 2.581 μs | 19.5313 | 2.1973 | - | 239.94 KB | -| **Encrypt** | **100** | **1,610.57 μs** | **161.738 μs** | **242.081 μs** | **150.3906** | **107.4219** | **74.2188** | **1773.64 KB** | -| Decrypt | 100 | 2,058.97 μs | 202.464 μs | 303.039 μs | 160.1563 | 107.4219 | 76.1719 | 2042.61 KB | +| **Encrypt** | **1** | **37.15 μs** | **0.683 μs** | **1.002 μs** | **3.8452** | **0.9766** | **-** | **47.28 KB** | +| Decrypt | 1 | 45.29 μs | 0.757 μs | 1.062 μs | 4.3945 | 1.0986 | - | 54.17 KB | +| **Encrypt** | **10** | **111.83 μs** | **1.252 μs** | **1.874 μs** | **15.1367** | **3.0518** | **-** | **186.07 KB** | +| Decrypt | 10 | 151.46 μs | 2.259 μs | 3.311 μs | 19.5313 | 2.1973 | - | 239.94 KB | +| **Encrypt** | **100** | **1,567.24 μs** | **153.944 μs** | **230.416 μs** | **152.3438** | **109.3750** | **76.1719** | **1773.95 KB** | +| Decrypt | 100 | 2,088.77 μs | 232.084 μs | 347.372 μs | 160.1563 | 113.2813 | 76.1719 | 2042.61 KB | From 611b3ac485adc698df5b3a7fe062cf8c4e42af0e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 11:55:26 +0200 Subject: [PATCH 20/49] + unit test --- .../AeadAes256CbcHmac256AlgorithmTests.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs index 9c2125f903..900fac6798 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests { using Microsoft.Azure.Cosmos.Encryption.Custom; using Microsoft.VisualStudio.TestTools.UnitTesting; + using System; using System.Linq; using System.Text; @@ -20,6 +21,8 @@ public class AeadAes256CbcHmac256AlgorithmTests [ClassInitialize] public static void ClassInitialize(TestContext testContext) { + _ = testContext; + AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); } @@ -39,5 +42,20 @@ public void EncryptUsingBufferDecryptsSuccessfully() Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); } + + [TestMethod] + public void DecryptUsingBufferDecryptsSuccessfully() + { + byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 }; + byte[] encrypted = algorithm.EncryptData(plainTextBytes); + + int plainTextMaxLength = algorithm.GetDecryptByteCount(encrypted.Length); + byte[] decrypted = new byte[plainTextMaxLength]; + + int decryptedBytes = algorithm.DecryptData(encrypted, 0, encrypted.Length, decrypted, 0); + + Assert.AreEqual(plainTextBytes.Length, decryptedBytes); + Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted.AsSpan(0, decryptedBytes).ToArray())); + } } } From 8a78fe87cd516919f88782d686e1b2b1e5f85e4b Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 14:35:47 +0200 Subject: [PATCH 21/49] ~ merge fixes and initial cleanup --- .../src/EncryptionProcessor.cs | 50 +++++++++---------- .../src/Encryptor.cs | 36 ------------- .../Readme.md | 16 +++--- 3 files changed, 31 insertions(+), 71 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index b657bf40ed..245e99b69d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -49,6 +49,8 @@ public static async Task EncryptAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + _ = diagnosticsContext; + EncryptionProcessor.ValidateInputForEncrypt( input, encryptor, @@ -118,7 +120,7 @@ public static async Task EncryptAsync( cipherTextWithTypeMarker[0] = (byte)typeMarker; - int encryptedBytesCount = await encryptionKey.EncryptAsync( + int encryptedBytesCount = encryptionKey.EncryptData( plainText, plainTextOffset: 0, plainTextLength, @@ -287,6 +289,13 @@ private static async Task MdeEncAlgoDecryptObjectAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); JObject plainTextJObj = new JObject(); @@ -304,22 +313,18 @@ private static async Task MdeEncAlgoDecryptObjectAsync( continue; } - int plainTextLength = await encryptor.GetDecryptBytesCountAsync( - cipherTextWithTypeMarker.Length - 1, - encryptionProperties.DataEncryptionKeyId, - encryptionProperties.EncryptionAlgorithm); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + int plainTextLength = encryptionKey.GetDecryptByteCount(cipherTextWithTypeMarker.Length - 1); byte[] plainText = arrayPoolManager.Rent(plainTextLength); - int decryptedCount = await EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( - encryptionProperties, + int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( + encryptionKey, cipherTextWithTypeMarker, cipherTextOffset: 1, cipherTextWithTypeMarker.Length - 1, - plainText, - encryptor, - diagnosticsContext, - cancellationToken); + plainText); EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], @@ -357,30 +362,19 @@ private static DecryptionContext CreateDecryptionContext( return decryptionContext; } - private static async Task MdeEncAlgoDecryptPropertyAsync( - EncryptionProperties encryptionProperties, + private static int MdeEncAlgoDecryptPropertyAsync( + DataEncryptionKey encryptionKey, byte[] cipherText, int cipherTextOffset, int cipherTextLength, - byte[] buffer, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) + byte[] buffer) { - if (encryptionProperties.EncryptionFormatVersion != 3) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - int decryptedCount = await encryptor.DecryptAsync( + int decryptedCount = encryptionKey.DecryptData( cipherText, cipherTextOffset, cipherTextLength, buffer, - outputOffset: 0, - encryptionProperties.DataEncryptionKeyId, - encryptionProperties.EncryptionAlgorithm, - cancellationToken); + outputOffset: 0); if (decryptedCount < 0) { @@ -397,6 +391,8 @@ private static async Task LegacyEncAlgoDecryptContentAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + _ = diagnosticsContext; + if (encryptionProperties.EncryptionFormatVersion != 2) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 51ce14a7cd..2fd857eeb4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -122,41 +122,5 @@ public abstract Task GetDecryptBytesCountAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); - - /// - /// Decrypts the cipherText using the key and algorithm provided. - /// - /// Ciphertext to be decrypted. - /// Offset in the cipherText array at which to begin using data from. - /// Number of bytes in the cipherText array to use as input. - /// Output buffer to write the decrypted data to. - /// Offset in the output array at which to begin writing data to. - /// Identifier of the data encryption key. - /// Identifier for the encryption algorithm. - /// Token for cancellation. - /// Plain text. - public abstract Task DecryptAsync( - byte[] cipherText, - int cipherTextOffset, - int cipherTextLength, - byte[] output, - int outputOffset, - string dataEncryptionKeyId, - string encryptionAlgorithm, - CancellationToken cancellationToken = default); - - /// - /// Calculate upper bound size of the input after decryption. - /// - /// Input data size. - /// Identifier of the data encryption key. - /// Identifier for the encryption algorithm. - /// Token for cancellation. - /// Upper bound size of the input when decrypted. - public abstract Task GetDecryptBytesCountAsync( - int cipherTextLength, - string dataEncryptionKeyId, - string encryptionAlgorithm, - CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 64a686244c..01e1e3a41a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|--------:|-----------:| -| **Encrypt** | **1** | **37.15 μs** | **0.683 μs** | **1.002 μs** | **3.8452** | **0.9766** | **-** | **47.28 KB** | -| Decrypt | 1 | 45.29 μs | 0.757 μs | 1.062 μs | 4.3945 | 1.0986 | - | 54.17 KB | -| **Encrypt** | **10** | **111.83 μs** | **1.252 μs** | **1.874 μs** | **15.1367** | **3.0518** | **-** | **186.07 KB** | -| Decrypt | 10 | 151.46 μs | 2.259 μs | 3.311 μs | 19.5313 | 2.1973 | - | 239.94 KB | -| **Encrypt** | **100** | **1,567.24 μs** | **153.944 μs** | **230.416 μs** | **152.3438** | **109.3750** | **76.1719** | **1773.95 KB** | -| Decrypt | 100 | 2,088.77 μs | 232.084 μs | 347.372 μs | 160.1563 | 113.2813 | 76.1719 | 2042.61 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **37.10 μs** | **0.527 μs** | **0.788 μs** | **36.82 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | +| Decrypt | 1 | 43.43 μs | 0.686 μs | 1.027 μs | 43.06 μs | 3.9673 | 1.0376 | - | 49.16 KB | +| **Encrypt** | **10** | **115.97 μs** | **2.256 μs** | **3.377 μs** | **117.36 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | +| Decrypt | 10 | 144.93 μs | 2.238 μs | 3.350 μs | 146.64 μs | 15.6250 | 3.1738 | - | 194.3 KB | +| **Encrypt** | **100** | **1,604.78 μs** | **150.864 μs** | **225.806 μs** | **1,557.92 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | +| Decrypt | 100 | 1,943.81 μs | 193.233 μs | 289.223 μs | 1,872.14 μs | 126.9531 | 78.1250 | 42.9688 | 1586.28 KB | From 8ed21357a28d393f6148462dac9ae37ff437e70d Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 15:06:56 +0200 Subject: [PATCH 22/49] ~ write directly to output document instead of copying --- .../src/EncryptionProcessor.cs | 21 +++----- .../EmulatorTests/MdeCustomEncryptionTests.cs | 50 ------------------- .../Readme.md | 16 +++--- 3 files changed, 16 insertions(+), 71 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 245e99b69d..cd95f1540b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -298,7 +298,7 @@ private static async Task MdeEncAlgoDecryptObjectAsync( using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); - JObject plainTextJObj = new JObject(); + List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { string propertyName = path.Substring(1); @@ -329,15 +329,10 @@ private static async Task MdeEncAlgoDecryptObjectAsync( EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), - plainTextJObj, + document, propertyName); - } - List pathsDecrypted = new List(); - foreach (JProperty property in plainTextJObj.Properties()) - { - document[property.Name] = property.Value; - pathsDecrypted.Add("/" + property.Name); + pathsDecrypted.Add(path); } DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( @@ -557,16 +552,16 @@ private static void DeserializeAndAddProperty( switch (typeMarker) { case TypeMarker.Boolean: - jObject.Add(key, SqlBoolSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlBoolSerializer.Deserialize(serializedBytes); break; case TypeMarker.Double: - jObject.Add(key, SqlDoubleSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlDoubleSerializer.Deserialize(serializedBytes); break; case TypeMarker.Long: - jObject.Add(key, SqlLongSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlLongSerializer.Deserialize(serializedBytes); break; case TypeMarker.String: - jObject.Add(key, SqlVarCharSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); break; case TypeMarker.Array: DeserializeAndAddProperty(serializedBytes); @@ -592,7 +587,7 @@ void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); using JsonTextReader reader = new JsonTextReader(memoryTextReader); - jObject.Add(key, serializer.Deserialize(reader)); + jObject[key] = serializer.Deserialize(reader); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 277bacbe89..df51ab0cc5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2266,26 +2266,6 @@ public override async Task DecryptAsync(byte[] cipherText, int cipherTextOf return await this.encryptor.DecryptAsync(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } - public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); - } - - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); - } - public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -2319,36 +2299,6 @@ public override async Task GetEncryptionKeyAsync(string dataE this.ThrowIfFail(dataEncryptionKeyId); return await this.encryptor.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } - - public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); - } - - public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.GetEncryptByteCount(plainTextLength); - } - - public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.GetDecryptByteCount(cipherTextLength); - } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 01e1e3a41a..7ffdbdcd60 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **37.10 μs** | **0.527 μs** | **0.788 μs** | **36.82 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | -| Decrypt | 1 | 43.43 μs | 0.686 μs | 1.027 μs | 43.06 μs | 3.9673 | 1.0376 | - | 49.16 KB | -| **Encrypt** | **10** | **115.97 μs** | **2.256 μs** | **3.377 μs** | **117.36 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | -| Decrypt | 10 | 144.93 μs | 2.238 μs | 3.350 μs | 146.64 μs | 15.6250 | 3.1738 | - | 194.3 KB | -| **Encrypt** | **100** | **1,604.78 μs** | **150.864 μs** | **225.806 μs** | **1,557.92 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | -| Decrypt | 100 | 1,943.81 μs | 193.233 μs | 289.223 μs | 1,872.14 μs | 126.9531 | 78.1250 | 42.9688 | 1586.28 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|---------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **37.28 μs** | **0.676 μs** | **0.991 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | +| Decrypt | 1 | 43.12 μs | 1.753 μs | 2.623 μs | 3.6011 | 1.1597 | - | 44.63 KB | +| **Encrypt** | **10** | **115.55 μs** | **1.717 μs** | **2.570 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | +| Decrypt | 10 | 121.81 μs | 2.127 μs | 3.050 μs | 12.9395 | 3.1738 | - | 159.56 KB | +| **Encrypt** | **100** | **1,571.06 μs** | **128.737 μs** | **192.687 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | +| Decrypt | 100 | 1,687.55 μs | 143.998 μs | 215.529 μs | 101.5625 | 62.5000 | 44.9219 | 1253.19 KB | From bbe98458081b72b5a53b8e70d2ca9ba873610b2e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 15:22:54 +0200 Subject: [PATCH 23/49] ! tests --- .../EmulatorTests/LegacyEncryptionTests.cs | 74 ++++++++++++++----- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index c39441fb80..d63f24755c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -103,7 +103,7 @@ public async Task EncryptionCreateDekWithMdeAlgorithmFails() { Assert.AreEqual("For use of 'MdeAeadAes256CbcHmac256Randomized' algorithm, Encryptor or CosmosDataEncryptionKeyProvider needs to be initialized with EncryptionKeyStoreProvider.", ex.Message); } - } + } [TestMethod] public async Task EncryptionRewrapDek() @@ -217,7 +217,7 @@ await LegacyEncryptionTests.IterateDekFeedAsync( QueryDefinition queryDefinition = new QueryDefinition("SELECT * from c where c.id >= @startId and c.id <= @endId ORDER BY c.id ASC") .WithParameter("@startId", "Contoso_v000") .WithParameter("@endId", "Contoso_v999"); - + await LegacyEncryptionTests.IterateDekFeedAsync( dekProvider, new List { contosoV1, contosoV2 }, @@ -405,7 +405,7 @@ public async Task EncryptionChangeFeedDecryptionSuccessful() TestDoc testDoc1 = await LegacyEncryptionTests.CreateItemAsync(LegacyEncryptionTests.encryptionContainer, LegacyEncryptionTests.dekId, TestDoc.PathsToEncrypt); TestDoc testDoc2 = await LegacyEncryptionTests.CreateItemAsync(LegacyEncryptionTests.encryptionContainer, dek2, TestDoc.PathsToEncrypt); - + // change feed iterator await this.ValidateChangeFeedIteratorResponse(LegacyEncryptionTests.encryptionContainer, testDoc1, testDoc2); @@ -597,7 +597,7 @@ public async Task EncryptionRudItemLazyDecryption() { TestDoc testDoc = TestDoc.Create(); // Upsert (item doesn't exist) - ItemResponse > upsertResponse = await LegacyEncryptionTests.encryptionContainer.UpsertItemAsync( + ItemResponse> upsertResponse = await LegacyEncryptionTests.encryptionContainer.UpsertItemAsync( new EncryptableItem(testDoc), new PartitionKey(testDoc.PK), LegacyEncryptionTests.GetRequestOptions(LegacyEncryptionTests.dekId, TestDoc.PathsToEncrypt)); @@ -660,7 +660,7 @@ public async Task EncryptionResourceTokenAuthRestricted() restrictedUserPermission.Token); Database databaseForRestrictedUser = clientForRestrictedUser.GetDatabase(LegacyEncryptionTests.database.Id); - Container containerForRestrictedUser = databaseForRestrictedUser.GetContainer(LegacyEncryptionTests.itemContainer.Id); + Container containerForRestrictedUser = databaseForRestrictedUser.GetContainer(LegacyEncryptionTests.itemContainer.Id); Container encryptionContainerForRestrictedUser = containerForRestrictedUser.WithEncryptor(encryptor); await LegacyEncryptionTests.PerformForbiddenOperationAsync(() => @@ -939,15 +939,15 @@ public async Task VerifyDekOperationWithSystemTextSerializer() // get database and container Database databaseWithCosmosSystemTextJsonSerializer = clientWithCosmosSystemTextJsonSerializer.GetDatabase(LegacyEncryptionTests.database.Id); Container containerWithCosmosSystemTextJsonSerializer = databaseWithCosmosSystemTextJsonSerializer.GetContainer(LegacyEncryptionTests.itemContainer.Id); - + // create the Dek container Container dekContainerWithCosmosSystemTextJsonSerializer = await databaseWithCosmosSystemTextJsonSerializer.CreateContainerAsync(Guid.NewGuid().ToString(), "/id", 400); - + CosmosDataEncryptionKeyProvider dekProviderWithCosmosSystemTextJsonSerializer = new CosmosDataEncryptionKeyProvider(new TestKeyWrapProvider()); await dekProviderWithCosmosSystemTextJsonSerializer.InitializeAsync(databaseWithCosmosSystemTextJsonSerializer, dekContainerWithCosmosSystemTextJsonSerializer.Id); - - TestEncryptor encryptorWithCosmosSystemTextJsonSerializer = new TestEncryptor(dekProviderWithCosmosSystemTextJsonSerializer); - + + TestEncryptor encryptorWithCosmosSystemTextJsonSerializer = new TestEncryptor(dekProviderWithCosmosSystemTextJsonSerializer); + // enable encryption on container Container encryptionContainerWithCosmosSystemTextJsonSerializer = containerWithCosmosSystemTextJsonSerializer.WithEncryptor(encryptorWithCosmosSystemTextJsonSerializer); @@ -991,7 +991,7 @@ public async Task VerifyDekOperationWithSystemTextSerializer() ItemResponse createTestDoc = await encryptionContainerWithCosmosSystemTextJsonSerializer.CreateItemAsync( testDocSystemText, new PartitionKey(testDocSystemText.PartitionKey), - LegacyEncryptionTests.GetRequestOptions(dekId, new List() { "/status"})); + LegacyEncryptionTests.GetRequestOptions(dekId, new List() { "/status" })); Assert.AreEqual(HttpStatusCode.Created, createTestDoc.StatusCode); @@ -1074,7 +1074,7 @@ public async Task EncryptionTransactionalBatchConflictResponse() Assert.AreEqual(HttpStatusCode.Conflict, batchResponse.StatusCode); Assert.AreEqual(1, batchResponse.Count); } - + private static async Task ValidateSprocResultsAsync(Container container, TestDoc expectedDoc) { string sprocId = Guid.NewGuid().ToString(); @@ -1129,7 +1129,7 @@ private static async Task ValidateQueryResultsAsync( queryResponseIterator = container.GetItemQueryIterator(queryDefinition, requestOptions: requestOptions); queryResponseIteratorForLazyDecryption = container.GetItemQueryIterator(queryDefinition, requestOptions: requestOptions); } - + FeedResponse readDocs = await queryResponseIterator.ReadNextAsync(); Assert.AreEqual(null, readDocs.ContinuationToken); @@ -1373,7 +1373,7 @@ private static async Task IterateDekFeedAsync( : dekProvider.DataEncryptionKeyContainer.GetDataEncryptionKeyQueryIterator( query, requestOptions: requestOptions); - + Assert.IsTrue(dekIterator.HasMoreResults); List readDekIds = new List(); @@ -1588,7 +1588,7 @@ private static async Task CreateDekAsync(CosmosData LegacyEncryptionTests.metadata1); Assert.AreEqual(HttpStatusCode.Created, dekResponse.StatusCode); - + return VerifyDekResponse(dekResponse, dekId); } @@ -1607,7 +1607,7 @@ private static DataEncryptionKeyProperties VerifyDekResponse( Assert.IsNotNull(dekProperties.SelfLink); Assert.IsNotNull(dekProperties.CreatedTime); Assert.IsNotNull(dekProperties.LastModified); - + return dekProperties; } @@ -1737,7 +1737,7 @@ public override Task WrapKeyAsync(byte[] key, Encryptio { this.WrapKeyCallsCount[metadata.Value]++; } - + EncryptionKeyWrapMetadata responseMetadata = new EncryptionKeyWrapMetadata(metadata.Value + LegacyEncryptionTests.metadataUpdateSuffix); int moveBy = metadata.Value == LegacyEncryptionTests.metadata1.Value ? 1 : 2; return Task.FromResult(new EncryptionKeyWrapResult(key.Select(b => (byte)(b + moveBy)).ToArray(), responseMetadata)); @@ -1813,7 +1813,45 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } - } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } + + public override async Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + return await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + } + } internal class CustomSerializer : CosmosSerializer { From a107f62b848c4a9eba5c00c94cc6acf05375d1ad Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 16:02:55 +0200 Subject: [PATCH 24/49] ~ retrieve DataEncryptionKey only once per document --- .../src/EncryptionProcessor.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index cd95f1540b..0370b94771 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -92,6 +92,8 @@ public static async Task EncryptAsync( { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { string propertyName = pathToEncrypt.Substring(1); @@ -112,8 +114,6 @@ public static async Task EncryptAsync( continue; } - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); @@ -298,6 +298,8 @@ private static async Task MdeEncAlgoDecryptObjectAsync( using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { @@ -313,8 +315,6 @@ private static async Task MdeEncAlgoDecryptObjectAsync( continue; } - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - int plainTextLength = encryptionKey.GetDecryptByteCount(cipherTextWithTypeMarker.Length - 1); byte[] plainText = arrayPoolManager.Rent(plainTextLength); From a1ad02b9095dae60db6e9ccc9df098d0962f9df7 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 16:16:27 +0200 Subject: [PATCH 25/49] ! fix tests ~ bump benchmark --- .../EmulatorTests/MdeCustomEncryptionTests.cs | 2 +- .../Readme.md | 16 ++++++++-------- .../MdeEncryptionProcessorTests.cs | 5 ++++- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index df51ab0cc5..ec98b39567 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(64, unwrapcount); + Assert.AreEqual(4, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 7ffdbdcd60..4df982a29d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|---------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **37.28 μs** | **0.676 μs** | **0.991 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | -| Decrypt | 1 | 43.12 μs | 1.753 μs | 2.623 μs | 3.6011 | 1.1597 | - | 44.63 KB | -| **Encrypt** | **10** | **115.55 μs** | **1.717 μs** | **2.570 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | -| Decrypt | 10 | 121.81 μs | 2.127 μs | 3.050 μs | 12.9395 | 3.1738 | - | 159.56 KB | -| **Encrypt** | **100** | **1,571.06 μs** | **128.737 μs** | **192.687 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | -| Decrypt | 100 | 1,687.55 μs | 143.998 μs | 215.529 μs | 101.5625 | 62.5000 | 44.9219 | 1253.19 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **29.20 μs** | **0.607 μs** | **0.871 μs** | **29.26 μs** | **3.3875** | **2.5940** | **-** | **41.51 KB** | +| Decrypt | 1 | 32.23 μs | 0.528 μs | 0.790 μs | 32.78 μs | 3.2349 | 0.7935 | - | 39.71 KB | +| **Encrypt** | **10** | **107.83 μs** | **1.975 μs** | **2.956 μs** | **107.93 μs** | **13.7939** | **0.6104** | **-** | **170.14 KB** | +| Decrypt | 10 | 116.25 μs | 1.537 μs | 2.301 μs | 117.15 μs | 12.5732 | 1.2207 | - | 154.63 KB | +| **Encrypt** | **100** | **1,493.01 μs** | **314.932 μs** | **471.376 μs** | **1,482.68 μs** | **214.8438** | **173.8281** | **140.6250** | **1655.51 KB** | +| Decrypt | 100 | 1,406.56 μs | 126.286 μs | 189.019 μs | 1,408.25 μs | 138.6719 | 103.5156 | 82.0313 | 1248.58 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index aec838a413..3e626a7453 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -41,7 +41,10 @@ public static void ClassInitialize(TestContext testContext) .Returns((int plainTextLength) => plainTextLength); DekMock.Setup(m => m.EncryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); - + DekMock.Setup(m => m.DecryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.DecryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + DekMock.Setup(m => m.GetDecryptByteCount(It.IsAny())) + .Returns((int cipherTextLength) => cipherTextLength); MdeEncryptionProcessorTests.mockEncryptor = new Mock(); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) From 4f2f072d2c9b3f8025203b425c1801c5302ae538 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 2 Oct 2024 11:40:26 +0200 Subject: [PATCH 26/49] ~ update Aes algorithm to reuse GetEncryptedByteCount --- .../src/AeadAes256CbcHmac256Algorithm.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index e50c60e8fd..c5ae780a1f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -201,12 +201,11 @@ protected byte[] EncryptData(byte[] plainText, bool hasAuthenticationTag) // Final blob we return = version + HMAC + iv + cipherText const int hmacStartIndex = 1; - int authenticationTagLen = hasAuthenticationTag ? KeySizeInBytes : 0; + int authenticationTagLen = hasAuthenticationTag ? AuthenticationTagSizeInBytes : 0; int ivStartIndex = hmacStartIndex + authenticationTagLen; int cipherStartIndex = ivStartIndex + BlockSizeInBytes; // this is where hmac starts. - // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. - int outputBufSize = sizeof(byte) + authenticationTagLen + iv.Length + (numBlocks * BlockSizeInBytes); + int outputBufSize = this.GetEncryptByteCount(plainText.Length) - (hasAuthenticationTag ? 0 : authenticationTagLen); byte[] outBuffer = new byte[outputBufSize]; // Store the version and IV rightaway From cbbeee2f664fd63a2db5da70c49044d1e025423f Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 4 Oct 2024 13:54:06 +0200 Subject: [PATCH 27/49] ~ refactor EncryptionProcessor --- .../src/AeAesEncryptionProcessor.cs | 116 +++++ .../src/EncryptionProcessor.cs | 458 ++---------------- .../src/MdeEncryptionProcessor.cs | 124 +++++ .../Transformation/JObjectSqlSerializer.cs | 121 +++++ .../src/Transformation/MdeEncryptor.cs | 55 +++ .../src/Transformation/TypeMarker.cs | 17 + 6 files changed, 472 insertions(+), 419 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs new file mode 100644 index 0000000000..7b7d366295 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs @@ -0,0 +1,116 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + internal static class AeAesEncryptionProcessor + { + public static async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken cancellationToken) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + List pathsEncrypted = new (); + EncryptionProperties encryptionProperties = null; + byte[] plainText = null; + byte[] cipherText = null; + + JObject toEncryptJObj = new (); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { + string propertyName = pathToEncrypt.Substring(1); + if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) + { + continue; + } + + toEncryptJObj.Add(propertyName, propertyValue.Value()); + itemJObj.Remove(propertyName); + } + + MemoryStream memoryStream = EncryptionProcessor.BaseSerializer.ToStream(toEncryptJObj); + Debug.Assert(memoryStream != null); + Debug.Assert(memoryStream.TryGetBuffer(out _)); + plainText = memoryStream.ToArray(); + + cipherText = await encryptor.EncryptAsync( + plainText, + encryptionOptions.DataEncryptionKeyId, + encryptionOptions.EncryptionAlgorithm, + cancellationToken); + + if (cipherText == null) + { + throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); + } + + encryptionProperties = new EncryptionProperties( + encryptionFormatVersion: 2, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: cipherText, + encryptionOptions.PathsToEncrypt); + + itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); + return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + } + + internal static async Task LegacyEncAlgoDecryptContentAsync( + JObject document, + EncryptionProperties encryptionProperties, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 2) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + byte[] plainText = await encryptor.DecryptAsync( + encryptionProperties.EncryptedData, + encryptionProperties.DataEncryptionKeyId, + encryptionProperties.EncryptionAlgorithm, + cancellationToken) ?? throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(EncryptionProcessor.DecryptAsync)}."); + JObject plainTextJObj; + using (MemoryStream memoryStream = new (plainText)) + using (StreamReader streamReader = new (memoryStream)) + using (JsonTextReader jsonTextReader = new (streamReader)) + { + jsonTextReader.ArrayPool = JsonArrayPool.Instance; + plainTextJObj = JObject.Load(jsonTextReader); + } + + List pathsDecrypted = new (); + foreach (JProperty property in plainTextJObj.Properties()) + { + document.Add(property.Name, property.Value); + pathsDecrypted.Add("/" + property.Name); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + + return decryptionContext; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 0370b94771..1c2c78fe24 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; - using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -13,7 +12,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Text; using System.Threading; using System.Threading.Tasks; - using Microsoft.Data.Encryption.Cryptography.Serializers; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -22,20 +20,12 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal static class EncryptionProcessor { - private static readonly SqlSerializerFactory SqlSerializerFactory = new SqlSerializerFactory(); - - // UTF-8 encoding. - private static readonly SqlVarCharSerializer SqlVarCharSerializer = new SqlVarCharSerializer(size: -1, codePageCharacterEncoding: 65001); - private static readonly SqlBitSerializer SqlBoolSerializer = new SqlBitSerializer(); - private static readonly SqlFloatSerializer SqlDoubleSerializer = new SqlFloatSerializer(); - private static readonly SqlBigIntSerializer SqlLongSerializer = new SqlBigIntSerializer(); - - private static readonly JsonSerializerSettings JsonSerializerSettings = new JsonSerializerSettings() + internal static readonly JsonSerializerSettings JsonSerializerSettings = new () { DateParseHandling = DateParseHandling.None, }; - internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new CosmosJsonDotNetSerializer(JsonSerializerSettings); + internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); /// /// If there isn't any PathsToEncrypt, input stream will be returned without any modification. @@ -51,7 +41,7 @@ public static async Task EncryptAsync( { _ = diagnosticsContext; - EncryptionProcessor.ValidateInputForEncrypt( + ValidateInputForEncrypt( input, encryptor, encryptionOptions); @@ -79,118 +69,14 @@ public static async Task EncryptAsync( } } - JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); - List pathsEncrypted = new List(); - EncryptionProperties encryptionProperties = null; - byte[] plainText = null; - byte[] cipherText = null; - TypeMarker typeMarker; - - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); - - switch (encryptionOptions.EncryptionAlgorithm) +#pragma warning disable CS0618 // Type or member is obsolete + return encryptionOptions.EncryptionAlgorithm switch { - case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - - foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) - { - string propertyName = pathToEncrypt.Substring(1); - if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) - { - continue; - } - - if (propertyValue.Type == JTokenType.Null) - { - continue; - } - - (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); - - if (plainText == null) - { - continue; - } - - int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); - - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); - - cipherTextWithTypeMarker[0] = (byte)typeMarker; - - int encryptedBytesCount = encryptionKey.EncryptData( - plainText, - plainTextOffset: 0, - plainTextLength, - cipherTextWithTypeMarker, - outputOffset: 1); - - if (encryptedBytesCount < 0) - { - throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); - } - - itemJObj[propertyName] = cipherTextWithTypeMarker.AsSpan(0, encryptedBytesCount + 1).ToArray(); - pathsEncrypted.Add(pathToEncrypt); - } - - encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted); - break; - - case CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized: - - JObject toEncryptJObj = new JObject(); - - foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) - { - string propertyName = pathToEncrypt.Substring(1); - if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) - { - continue; - } - - toEncryptJObj.Add(propertyName, propertyValue.Value()); - itemJObj.Remove(propertyName); - } - - MemoryStream memoryStream = EncryptionProcessor.BaseSerializer.ToStream(toEncryptJObj); - Debug.Assert(memoryStream != null); - Debug.Assert(memoryStream.TryGetBuffer(out _)); - plainText = memoryStream.ToArray(); - - cipherText = await encryptor.EncryptAsync( - plainText, - encryptionOptions.DataEncryptionKeyId, - encryptionOptions.EncryptionAlgorithm, - cancellationToken); - - if (cipherText == null) - { - throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); - } - - encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 2, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: cipherText, - encryptionOptions.PathsToEncrypt); - break; - - default: - throw new NotSupportedException($"Encryption Algorithm : {encryptionOptions.EncryptionAlgorithm} is not supported."); - } - - itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); - input.Dispose(); - return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, cancellationToken), + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, cancellationToken), + _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionOptions.EncryptionAlgorithm} is not supported."), + }; +#pragma warning restore CS0618 // Type or member is obsolete } /// @@ -213,8 +99,8 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); Debug.Assert(diagnosticsContext != null); - JObject itemJObj = EncryptionProcessor.RetrieveItem(input); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(itemJObj); + JObject itemJObj = RetrieveItem(input); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(itemJObj); if (encryptionPropertiesJObj == null) { @@ -222,26 +108,9 @@ public static async Task EncryptAsync( return (input, null); } - EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject(); - DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch - { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( - itemJObj, - encryptor, - encryptionProperties, - diagnosticsContext, - cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( - itemJObj, - encryptionProperties, - encryptor, - diagnosticsContext, - cancellationToken), - _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."), - }; - + DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, itemJObj, encryptionPropertiesJObj, cancellationToken); input.Dispose(); - return (EncryptionProcessor.BaseSerializer.ToStream(itemJObj), decryptionContext); + return (BaseSerializer.ToStream(itemJObj), decryptionContext); } public static async Task<(JObject, DecryptionContext)> DecryptAsync( @@ -254,175 +123,56 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(document); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(document); if (encryptionPropertiesJObj == null) { return (document, null); } + DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, document, encryptionPropertiesJObj, cancellationToken); + + return (document, decryptionContext); + } + + private static async Task DecryptInternalAsync(Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, JObject itemJObj, JObject encryptionPropertiesJObj, CancellationToken cancellationToken) + { EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject(); +#pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( - document, + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( - document, + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + itemJObj, encryptionProperties, encryptor, diagnosticsContext, cancellationToken), _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."), }; - - return (document, decryptionContext); - } - - private static async Task MdeEncAlgoDecryptObjectAsync( - JObject document, - Encryptor encryptor, - EncryptionProperties encryptionProperties, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - _ = diagnosticsContext; - - if (encryptionProperties.EncryptionFormatVersion != 3) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - - List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); - foreach (string path in encryptionProperties.EncryptedPaths) - { - string propertyName = path.Substring(1); - if (!document.TryGetValue(propertyName, out JToken propertyValue)) - { - continue; - } - - byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); - if (cipherTextWithTypeMarker == null) - { - continue; - } - - int plainTextLength = encryptionKey.GetDecryptByteCount(cipherTextWithTypeMarker.Length - 1); - - byte[] plainText = arrayPoolManager.Rent(plainTextLength); - - int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( - encryptionKey, - cipherTextWithTypeMarker, - cipherTextOffset: 1, - cipherTextWithTypeMarker.Length - 1, - plainText); - - EncryptionProcessor.DeserializeAndAddProperty( - (TypeMarker)cipherTextWithTypeMarker[0], - plainText.AsSpan(0, decryptedCount), - document, - propertyName); - - pathsDecrypted.Add(path); - } - - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( - pathsDecrypted, - encryptionProperties.DataEncryptionKeyId); - - document.Remove(Constants.EncryptedInfo); +#pragma warning restore CS0618 // Type or member is obsolete return decryptionContext; } - private static DecryptionContext CreateDecryptionContext( + internal static DecryptionContext CreateDecryptionContext( List pathsDecrypted, string dataEncryptionKeyId) { - DecryptionInfo decryptionInfo = new DecryptionInfo( + DecryptionInfo decryptionInfo = new ( pathsDecrypted, dataEncryptionKeyId); - DecryptionContext decryptionContext = new DecryptionContext( + DecryptionContext decryptionContext = new ( new List() { decryptionInfo }); return decryptionContext; } - private static int MdeEncAlgoDecryptPropertyAsync( - DataEncryptionKey encryptionKey, - byte[] cipherText, - int cipherTextOffset, - int cipherTextLength, - byte[] buffer) - { - int decryptedCount = encryptionKey.DecryptData( - cipherText, - cipherTextOffset, - cipherTextLength, - buffer, - outputOffset: 0); - - if (decryptedCount < 0) - { - throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); - } - - return decryptedCount; - } - - private static async Task LegacyEncAlgoDecryptContentAsync( - JObject document, - EncryptionProperties encryptionProperties, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - _ = diagnosticsContext; - - if (encryptionProperties.EncryptionFormatVersion != 2) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - byte[] plainText = await encryptor.DecryptAsync( - encryptionProperties.EncryptedData, - encryptionProperties.DataEncryptionKeyId, - encryptionProperties.EncryptionAlgorithm, - cancellationToken) ?? throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); - JObject plainTextJObj; - using (MemoryStream memoryStream = new MemoryStream(plainText)) - using (StreamReader streamReader = new StreamReader(memoryStream)) - using (JsonTextReader jsonTextReader = new JsonTextReader(streamReader)) - { - jsonTextReader.ArrayPool = JsonArrayPool.Instance; - plainTextJObj = JObject.Load(jsonTextReader); - } - - List pathsDecrypted = new List(); - foreach (JProperty property in plainTextJObj.Properties()) - { - document.Add(property.Name, property.Value); - pathsDecrypted.Add("/" + property.Name); - } - - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( - pathsDecrypted, - encryptionProperties.DataEncryptionKeyId); - - document.Remove(Constants.EncryptedInfo); - - return decryptionContext; - } - private static void ValidateInputForEncrypt( Stream input, Encryptor encryptor, @@ -465,11 +215,11 @@ private static JObject RetrieveItem( Debug.Assert(input != null); JObject itemJObj; - using (StreamReader sr = new StreamReader(input, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 1024, leaveOpen: true)) - using (JsonTextReader jsonTextReader = new JsonTextReader(sr)) + using (StreamReader sr = new (input, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 1024, leaveOpen: true)) + using (JsonTextReader jsonTextReader = new (sr)) { jsonTextReader.ArrayPool = JsonArrayPool.Instance; - JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings() + JsonSerializerSettings jsonSerializerSettings = new () { DateParseHandling = DateParseHandling.None, MaxDepth = 64, // https://github.com/advisories/GHSA-5crp-9r3c-p9vr @@ -494,129 +244,21 @@ private static JObject RetrieveEncryptionProperties( return encryptionPropertiesJObj; } - private static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) - { - byte[] buffer; - int length; - switch (propertyValue.Type) - { - case JTokenType.Undefined: - Debug.Assert(false, "Undefined value cannot be in the JSON"); - return (default, null, -1); - case JTokenType.Null: - Debug.Assert(false, "Null type should have been handled by caller"); - return (TypeMarker.Null, null, -1); - case JTokenType.Boolean: - (buffer, length) = SerializeFixed(SqlBoolSerializer); - return (TypeMarker.Boolean, buffer, length); - case JTokenType.Float: - (buffer, length) = SerializeFixed(SqlDoubleSerializer); - return (TypeMarker.Double, buffer, length); - case JTokenType.Integer: - (buffer, length) = SerializeFixed(SqlLongSerializer); - return (TypeMarker.Long, buffer, length); - case JTokenType.String: - (buffer, length) = SerializeString(propertyValue.ToObject()); - return (TypeMarker.String, buffer, length); - case JTokenType.Array: - (buffer, length) = SerializeString(propertyValue.ToString()); - return (TypeMarker.Array, buffer, length); - case JTokenType.Object: - (buffer, length) = SerializeString(propertyValue.ToString()); - return (TypeMarker.Object, buffer, length); - default: - throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); - } - - (byte[], int) SerializeFixed(IFixedSizeSerializer serializer) - { - byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); - int length = serializer.Serialize(propertyValue.ToObject(), buffer); - return (buffer, length); - } - - (byte[], int) SerializeString(string value) - { - byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); - int length = SqlVarCharSerializer.Serialize(value, buffer); - return (buffer, length); - } - } - - private static void DeserializeAndAddProperty( - TypeMarker typeMarker, - ReadOnlySpan serializedBytes, - JObject jObject, - string key) - { - switch (typeMarker) - { - case TypeMarker.Boolean: - jObject[key] = SqlBoolSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.Double: - jObject[key] = SqlDoubleSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.Long: - jObject[key] = SqlLongSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.String: - jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.Array: - DeserializeAndAddProperty(serializedBytes); - break; - case TypeMarker.Object: - DeserializeAndAddProperty(serializedBytes); - break; - default: - Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); - break; - } - - void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) - where T : JToken - { - using ArrayPoolManager manager = new ArrayPoolManager(); - - char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); - int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); - - JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); - - using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); - using JsonTextReader reader = new JsonTextReader(memoryTextReader); - - jObject[key] = serializer.Deserialize(reader); - } - } - - private enum TypeMarker : byte - { - Null = 1, // not used - String = 2, - Double = 3, - Long = 4, - Boolean = 5, - Array = 6, - Object = 7, - } - internal static async Task DeserializeAndDecryptResponseAsync( Stream content, Encryptor encryptor, CancellationToken cancellationToken) { - JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream(content); + JObject contentJObj = BaseSerializer.FromStream(content); - if (!(contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is JArray documents)) + if (contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is not JArray documents) { throw new InvalidOperationException("Feed Response body contract was violated. Feed response did not have an array of Documents"); } foreach (JToken value in documents) { - if (!(value is JObject document)) + if (value is not JObject document) { continue; } @@ -624,7 +266,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); using (diagnosticsContext.CreateScope("EncryptionProcessor.DeserializeAndDecryptResponseAsync")) { - await EncryptionProcessor.DecryptAsync( + await DecryptAsync( document, encryptor, diagnosticsContext, @@ -634,29 +276,7 @@ await EncryptionProcessor.DecryptAsync( // the contents of contentJObj get decrypted in place for MDE algorithm model, and for legacy model _ei property is removed // and corresponding decrypted properties are added back in the documents. - return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); - } - - internal static int GetOriginalBase64Length(string base64string) - { - if (string.IsNullOrEmpty(base64string)) - { - return 0; - } - - int paddingCount = 0; - int characterCount = base64string.Length; - if (base64string[characterCount - 1] == '=') - { - paddingCount++; - } - - if (base64string[characterCount - 2] == '=') - { - paddingCount++; - } - - return (3 * (characterCount / 4)) - paddingCount; + return BaseSerializer.ToStream(contentJObj); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs new file mode 100644 index 0000000000..e15dbf88f8 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs @@ -0,0 +1,124 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; + using Newtonsoft.Json.Linq; + + internal static class MdeEncryptionProcessor + { + public static async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + List pathsEncrypted = new (); + TypeMarker typeMarker; + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { + string propertyName = pathToEncrypt.Substring(1); + if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) + { + continue; + } + + if (propertyValue.Type == JTokenType.Null) + { + continue; + } + + byte[] plainText = null; + (typeMarker, plainText, int plainTextLength) = JObjectSqlSerializer.Serialize(propertyValue, arrayPoolManager); + + if (plainText == null) + { + continue; + } + + (byte[] encryptedBytes, int encryptedLength) = MdeEncryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); + + itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + pathsEncrypted.Add(pathToEncrypt); + } + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); + + itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); + return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + } + + internal static async Task MdeEncAlgoDecryptObjectAsync( + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); + foreach (string path in encryptionProperties.EncryptedPaths) + { + string propertyName = path.Substring(1); + if (!document.TryGetValue(propertyName, out JToken propertyValue)) + { + // malformed document, such record shouldn't be there at all + continue; + } + + byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); + if (cipherTextWithTypeMarker == null) + { + continue; + } + + (byte[] plainText, int decryptedCount) = MdeEncryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + + JObjectSqlSerializer.DeserializeAndAddProperty( + (TypeMarker)cipherTextWithTypeMarker[0], + plainText.AsSpan(0, decryptedCount), + document, + propertyName); + + pathsDecrypted.Add(path); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + return decryptionContext; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs new file mode 100644 index 0000000000..5056b4645c --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs @@ -0,0 +1,121 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Diagnostics; + using Microsoft.Data.Encryption.Cryptography.Serializers; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + internal static class JObjectSqlSerializer + { + private static readonly SqlBitSerializer SqlBoolSerializer = new (); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); + private static readonly SqlBigIntSerializer SqlLongSerializer = new (); + + // UTF-8 encoding. + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); + + private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; + + internal static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) + { + byte[] buffer; + int length; + switch (propertyValue.Type) + { + case JTokenType.Undefined: + Debug.Assert(false, "Undefined value cannot be in the JSON"); + return (default, null, -1); + case JTokenType.Null: + Debug.Assert(false, "Null type should have been handled by caller"); + return (TypeMarker.Null, null, -1); + case JTokenType.Boolean: + (buffer, length) = SerializeFixed(SqlBoolSerializer); + return (TypeMarker.Boolean, buffer, length); + case JTokenType.Float: + (buffer, length) = SerializeFixed(SqlDoubleSerializer); + return (TypeMarker.Double, buffer, length); + case JTokenType.Integer: + (buffer, length) = SerializeFixed(SqlLongSerializer); + return (TypeMarker.Long, buffer, length); + case JTokenType.String: + (buffer, length) = SerializeString(propertyValue.ToObject()); + return (TypeMarker.String, buffer, length); + case JTokenType.Array: + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Array, buffer, length); + case JTokenType.Object: + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Object, buffer, length); + default: + throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); + } + + (byte[], int) SerializeFixed(IFixedSizeSerializer serializer) + { + byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); + int length = serializer.Serialize(propertyValue.ToObject(), buffer); + return (buffer, length); + } + + (byte[], int) SerializeString(string value) + { + byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); + int length = SqlVarCharSerializer.Serialize(value, buffer); + return (buffer, length); + } + } + + internal static void DeserializeAndAddProperty( + TypeMarker typeMarker, + ReadOnlySpan serializedBytes, + JObject jObject, + string key) + { + switch (typeMarker) + { + case TypeMarker.Boolean: + jObject[key] = SqlBoolSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Double: + jObject[key] = SqlDoubleSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Long: + jObject[key] = SqlLongSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.String: + jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Array: + DeserializeAndAddProperty(serializedBytes); + break; + case TypeMarker.Object: + DeserializeAndAddProperty(serializedBytes); + break; + default: + Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); + break; + } + + void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) + where T : JToken + { + using ArrayPoolManager manager = new (); + + char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); + int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); + + JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); + + using MemoryTextReader memoryTextReader = new (new Memory(buffer, 0, length)); + using JsonTextReader reader = new (memoryTextReader); + + jObject[key] = serializer.Deserialize(reader); + } + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs new file mode 100644 index 0000000000..325c1b5f03 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -0,0 +1,55 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + + internal static class MdeEncryptor + { + internal static (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) + { + int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1; + + byte[] encryptedText = arrayPoolManager.Rent(encryptedTextLength); + + encryptedText[0] = (byte)typeMarker; + + int encryptedLength = encryptionKey.EncryptData( + plainText, + plainTextOffset: 0, + plainTextLength, + encryptedText, + outputOffset: 1); + + if (encryptedLength < 0) + { + throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null cipherText from {nameof(DataEncryptionKey.EncryptData)}."); + } + + return (encryptedText, encryptedLength + 1); + } + + internal static (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) + { + int plainTextLength = encryptionKey.GetDecryptByteCount(cipherText.Length - 1); + + byte[] plainText = arrayPoolManager.Rent(plainTextLength); + + int decryptedLength = encryptionKey.DecryptData( + cipherText, + cipherTextOffset: 1, + cipherTextLength: cipherText.Length - 1, + plainText, + outputOffset: 0); + + if (decryptedLength < 0) + { + throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null plainText from {nameof(DataEncryptionKey.DecryptData)}."); + } + + return (plainText, decryptedLength); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs new file mode 100644 index 0000000000..f2f31c0927 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + internal enum TypeMarker : byte + { + Null = 1, // not used + String = 2, + Double = 3, + Long = 4, + Boolean = 5, + Array = 6, + Object = 7, + } +} From b6c851cbda5a5fd974e0f7f95d24c6cea683efd7 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 4 Oct 2024 13:59:46 +0200 Subject: [PATCH 28/49] ! names --- .../src/AeAesEncryptionProcessor.cs | 2 +- .../src/EncryptionProcessor.cs | 4 ++-- .../src/MdeEncryptionProcessor.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs index 7b7d366295..0944714aac 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs @@ -69,7 +69,7 @@ public static async Task EncryptAsync( return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } - internal static async Task LegacyEncAlgoDecryptContentAsync( + internal static async Task DecryptContentAsync( JObject document, EncryptionProperties encryptionProperties, Encryptor encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 1c2c78fe24..d274d1d899 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -141,13 +141,13 @@ private static async Task DecryptInternalAsync(Encryptor encr #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.DecryptObjectAsync( itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.DecryptContentAsync( itemJObj, encryptionProperties, encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs index e15dbf88f8..fc601a0916 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs @@ -68,7 +68,7 @@ public static async Task EncryptAsync( return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } - internal static async Task MdeEncAlgoDecryptObjectAsync( + internal static async Task DecryptObjectAsync( JObject document, Encryptor encryptor, EncryptionProperties encryptionProperties, From 72ccae7a87c9fcd49663c8ba82ccb0f1a3be5b7e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 4 Oct 2024 14:30:38 +0200 Subject: [PATCH 29/49] ~ less static --- .../src/EncryptionProcessor.cs | 2 ++ .../src/MdeEncryptionProcessor.cs | 18 +++++++++++------- .../src/Transformation/JObjectSqlSerializer.cs | 16 +++++++++------- .../src/Transformation/MdeEncryptor.cs | 6 +++--- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index d274d1d899..3a97b051a9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -27,6 +27,8 @@ internal static class EncryptionProcessor internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); + private static readonly MdeEncryptionProcessor MdeEncryptionProcessor = new (); + /// /// If there isn't any PathsToEncrypt, input stream will be returned without any modification. /// Else input stream will be disposed, and a new stream is returned. diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs index fc601a0916..571c5b4ac7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs @@ -13,9 +13,13 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; using Newtonsoft.Json.Linq; - internal static class MdeEncryptionProcessor + internal class MdeEncryptionProcessor { - public static async Task EncryptAsync( + internal JObjectSqlSerializer Serializer { get; set; } = new JObjectSqlSerializer(); + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + public async Task EncryptAsync( Stream input, Encryptor encryptor, EncryptionOptions encryptionOptions, @@ -43,14 +47,14 @@ public static async Task EncryptAsync( } byte[] plainText = null; - (typeMarker, plainText, int plainTextLength) = JObjectSqlSerializer.Serialize(propertyValue, arrayPoolManager); + (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); if (plainText == null) { continue; } - (byte[] encryptedBytes, int encryptedLength) = MdeEncryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); + (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); pathsEncrypted.Add(pathToEncrypt); @@ -68,7 +72,7 @@ public static async Task EncryptAsync( return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } - internal static async Task DecryptObjectAsync( + internal async Task DecryptObjectAsync( JObject document, Encryptor encryptor, EncryptionProperties encryptionProperties, @@ -102,9 +106,9 @@ internal static async Task DecryptObjectAsync( continue; } - (byte[] plainText, int decryptedCount) = MdeEncryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); - JObjectSqlSerializer.DeserializeAndAddProperty( + this.Serializer.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs index 5056b4645c..53ee238481 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs @@ -10,7 +10,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using Newtonsoft.Json; using Newtonsoft.Json.Linq; - internal static class JObjectSqlSerializer + internal class JObjectSqlSerializer { private static readonly SqlBitSerializer SqlBoolSerializer = new (); private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); @@ -21,7 +21,8 @@ internal static class JObjectSqlSerializer private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; - internal static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) +#pragma warning disable SA1101 // Prefix local calls with this - false positive on SerializeFixed + internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) { byte[] buffer; int length; @@ -69,8 +70,9 @@ internal static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBy return (buffer, length); } } +#pragma warning restore SA1101 // Prefix local calls with this - internal static void DeserializeAndAddProperty( + internal virtual void DeserializeAndAddProperty( TypeMarker typeMarker, ReadOnlySpan serializedBytes, JObject jObject, @@ -91,17 +93,17 @@ internal static void DeserializeAndAddProperty( jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); break; case TypeMarker.Array: - DeserializeAndAddProperty(serializedBytes); + jObject[key] = Deserialize(serializedBytes); break; case TypeMarker.Object: - DeserializeAndAddProperty(serializedBytes); + jObject[key] = Deserialize(serializedBytes); break; default: Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); break; } - void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) + static T Deserialize(ReadOnlySpan serializedBytes) where T : JToken { using ArrayPoolManager manager = new (); @@ -114,7 +116,7 @@ void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) using MemoryTextReader memoryTextReader = new (new Memory(buffer, 0, length)); using JsonTextReader reader = new (memoryTextReader); - jObject[key] = serializer.Deserialize(reader); + return serializer.Deserialize(reader); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs index 325c1b5f03..9243c632d6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -6,9 +6,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; - internal static class MdeEncryptor + internal class MdeEncryptor { - internal static (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) + internal virtual (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) { int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1; @@ -31,7 +31,7 @@ internal static (byte[] encryptedText, int encryptedLength) Encrypt(DataEncrypti return (encryptedText, encryptedLength + 1); } - internal static (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) + internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) { int plainTextLength = encryptionKey.GetDecryptByteCount(cipherText.Length - 1); From 5554aa0fa308910b501f07219af1860f2da0f3df Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 13:44:16 +0200 Subject: [PATCH 30/49] ~ merge fixes --- .../src/ArrayPoolManager.cs | 4 +- .../src/CosmosEncryptor.cs | 1 - .../src/EncryptionProcessor.cs | 73 ++++++++++--------- .../src/Encryptor.cs | 1 - .../src/MemoryTextReader.cs | 4 +- .../EmulatorTests/LegacyEncryptionTests.cs | 20 ----- .../MdeEncryptionProcessorTests.cs | 18 ++--- 7 files changed, 53 insertions(+), 68 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs index 8cafd07c5d..9679adc1c4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -8,9 +8,11 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Buffers; using System.Collections.Generic; +#pragma warning disable SA1402 // File may only contain a single type internal class ArrayPoolManager : IDisposable +#pragma warning restore SA1402 // File may only contain a single type { - private List rentedBuffers = new List(); + private List rentedBuffers = new (); private bool disposedValue; public T[] Rent(int minimumLength) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 6e6be7822d..454440f5ff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -64,6 +64,5 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } - } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 7222180f76..b36747b960 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; - using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -25,10 +24,10 @@ internal static class EncryptionProcessor private static readonly SqlSerializerFactory SqlSerializerFactory = new (); // UTF-8 encoding. - private static readonly SqlVarCharSerializer SqlVarCharSerializer = new SqlVarCharSerializer(size: -1, codePageCharacterEncoding: 65001); - private static readonly SqlBitSerializer SqlBoolSerializer = new SqlBitSerializer(); - private static readonly SqlFloatSerializer SqlDoubleSerializer = new SqlFloatSerializer(); - private static readonly SqlBigIntSerializer SqlLongSerializer = new SqlBigIntSerializer(); + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); + private static readonly SqlBitSerializer SqlBoolSerializer = new (); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); + private static readonly SqlBigIntSerializer SqlLongSerializer = new (); private static readonly JsonSerializerSettings JsonSerializerSettings = new () { @@ -79,15 +78,16 @@ public static async Task EncryptAsync( } } - JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + JObject itemJObj = BaseSerializer.FromStream(input); List pathsEncrypted = new (encryptionOptions.PathsToEncrypt.Count()); EncryptionProperties encryptionProperties = null; byte[] plainText = null; byte[] cipherText = null; TypeMarker typeMarker; - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + using ArrayPoolManager arrayPoolManager = new (); +#pragma warning disable CS0618 // Type or member is obsolete switch (encryptionOptions.EncryptionAlgorithm) { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: @@ -107,7 +107,7 @@ public static async Task EncryptAsync( continue; } - (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); + (typeMarker, plainText, int plainTextLength) = Serialize(propertyValue, arrayPoolManager); if (plainText == null) { @@ -160,7 +160,7 @@ public static async Task EncryptAsync( itemJObj.Remove(propertyName); } - MemoryStream memoryStream = EncryptionProcessor.BaseSerializer.ToStream(toEncryptJObj); + MemoryStream memoryStream = BaseSerializer.ToStream(toEncryptJObj); Debug.Assert(memoryStream != null); Debug.Assert(memoryStream.TryGetBuffer(out _)); plainText = memoryStream.ToArray(); @@ -191,7 +191,7 @@ public static async Task EncryptAsync( itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); input.Dispose(); - return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + return BaseSerializer.ToStream(itemJObj); } /// @@ -214,8 +214,8 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); Debug.Assert(diagnosticsContext != null); - JObject itemJObj = EncryptionProcessor.RetrieveItem(input); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(itemJObj); + JObject itemJObj = RetrieveItem(input); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(itemJObj); if (encryptionPropertiesJObj == null) { @@ -227,13 +227,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( itemJObj, encryptionProperties, encryptor, @@ -244,7 +244,7 @@ public static async Task EncryptAsync( #pragma warning restore CS0618 // Type or member is obsolete input.Dispose(); - return (EncryptionProcessor.BaseSerializer.ToStream(itemJObj), decryptionContext); + return (BaseSerializer.ToStream(itemJObj), decryptionContext); } public static async Task<(JObject, DecryptionContext)> DecryptAsync( @@ -257,7 +257,7 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(document); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(document); if (encryptionPropertiesJObj == null) { @@ -268,13 +268,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( document, encryptionProperties, encryptor, @@ -301,11 +301,11 @@ private static async Task MdeEncAlgoDecryptObjectAsync( throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + using ArrayPoolManager arrayPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { string propertyName = path.Substring(1); @@ -324,14 +324,14 @@ private static async Task MdeEncAlgoDecryptObjectAsync( byte[] plainText = arrayPoolManager.Rent(plainTextLength); - int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( + int decryptedCount = MdeEncAlgoDecryptProperty( encryptionKey, cipherTextWithTypeMarker, cipherTextOffset: 1, cipherTextWithTypeMarker.Length - 1, plainText); - EncryptionProcessor.DeserializeAndAddProperty( + DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, @@ -340,7 +340,7 @@ private static async Task MdeEncAlgoDecryptObjectAsync( pathsDecrypted.Add(path); } - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + DecryptionContext decryptionContext = CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -362,14 +362,21 @@ private static DecryptionContext CreateDecryptionContext( return decryptionContext; } - private static int MdeEncAlgoDecryptPropertyAsync( + private static int MdeEncAlgoDecryptProperty( DataEncryptionKey encryptionKey, byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] buffer) { - if (encryptionProperties.EncryptionFormatVersion != 3) + int decryptedCount = encryptionKey.DecryptData( + cipherText, + cipherTextOffset, + cipherTextLength, + buffer, + outputOffset: 0); + + if (decryptedCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); } @@ -412,7 +419,7 @@ private static async Task LegacyEncAlgoDecryptContentAsync( pathsDecrypted.Add("/" + property.Name); } - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + DecryptionContext decryptionContext = CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -575,15 +582,15 @@ private static void DeserializeAndAddProperty( void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) where T : JToken { - using ArrayPoolManager manager = new ArrayPoolManager(); + using ArrayPoolManager manager = new (); char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); - using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); - using JsonTextReader reader = new JsonTextReader(memoryTextReader); + using MemoryTextReader memoryTextReader = new (new Memory(buffer, 0, length)); + using JsonTextReader reader = new (memoryTextReader); jObject[key] = serializer.Deserialize(reader); } @@ -605,7 +612,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( Encryptor encryptor, CancellationToken cancellationToken) { - JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream(content); + JObject contentJObj = BaseSerializer.FromStream(content); if (contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is not JArray documents) { @@ -622,7 +629,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); using (diagnosticsContext.CreateScope("EncryptionProcessor.DeserializeAndDecryptResponseAsync")) { - await EncryptionProcessor.DecryptAsync( + await DecryptAsync( document, encryptor, diagnosticsContext, @@ -632,7 +639,7 @@ await EncryptionProcessor.DecryptAsync( // the contents of contentJObj get decrypted in place for MDE algorithm model, and for legacy model _ei property is removed // and corresponding decrypted properties are added back in the documents. - return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); + return BaseSerializer.ToStream(contentJObj); } internal static int GetOriginalBase64Length(string base64string) @@ -657,4 +664,4 @@ internal static int GetOriginalBase64Length(string base64string) return (3 * (characterCount / 4)) - paddingCount; } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index a517c5fea6..27c152d491 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -13,7 +13,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// public abstract class Encryptor { - /// /// Retrieve Data Encryption Key. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs index b8996ca802..9326fbbf0e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs @@ -135,7 +135,7 @@ public override string ReadLine() char ch = this.chars.Span[i]; if (ch == '\r' || ch == '\n') { - string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); this.pos = i + 1; if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') { @@ -150,7 +150,7 @@ public override string ReadLine() if (i > this.pos) { - string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); this.pos = i; return result; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 3672f808c6..30f6364a32 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1768,26 +1768,6 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } - public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); - } - - DataEncryptionKey dek = await this.GetEncryptionKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); - } - public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 757245a83b..69a9c7afb0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -41,26 +41,24 @@ public static void ClassInitialize(TestContext testContext) .Returns((int plainTextLength) => plainTextLength); DekMock.Setup(m => m.EncryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + DekMock.Setup(m => m.DecryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.DecryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + DekMock.Setup(m => m.GetDecryptByteCount(It.IsAny())) + .Returns((int cipherTextLength) => cipherTextLength); - MdeEncryptionProcessorTests.mockEncryptor = new Mock(); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + mockEncryptor = new Mock(); + mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((string dekId, string algorithm, CancellationToken token) => dekId == MdeEncryptionProcessorTests.dekId ? DekMock.Object : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] From 28620edf2961e70384f29d75476dd90f18e9f592 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 14:18:17 +0200 Subject: [PATCH 31/49] ~ cleanup --- .../src/EncryptionProcessor.cs | 6 ++---- .../Readme.md | 12 ++++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index b36747b960..b36619b11f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -21,8 +21,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal static class EncryptionProcessor { - private static readonly SqlSerializerFactory SqlSerializerFactory = new (); - // UTF-8 encoding. private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); private static readonly SqlBitSerializer SqlBoolSerializer = new (); @@ -92,7 +90,7 @@ public static async Task EncryptAsync( { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, cancellationToken); foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { @@ -575,7 +573,7 @@ private static void DeserializeAndAddProperty( DeserializeAndAddProperty(serializedBytes); break; default: - Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); + Debug.Fail($"Unexpected type marker {typeMarker}"); break; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 4df982a29d..f8c40c6bce 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **29.20 μs** | **0.607 μs** | **0.871 μs** | **29.26 μs** | **3.3875** | **2.5940** | **-** | **41.51 KB** | -| Decrypt | 1 | 32.23 μs | 0.528 μs | 0.790 μs | 32.78 μs | 3.2349 | 0.7935 | - | 39.71 KB | -| **Encrypt** | **10** | **107.83 μs** | **1.975 μs** | **2.956 μs** | **107.93 μs** | **13.7939** | **0.6104** | **-** | **170.14 KB** | -| Decrypt | 10 | 116.25 μs | 1.537 μs | 2.301 μs | 117.15 μs | 12.5732 | 1.2207 | - | 154.63 KB | -| **Encrypt** | **100** | **1,493.01 μs** | **314.932 μs** | **471.376 μs** | **1,482.68 μs** | **214.8438** | **173.8281** | **140.6250** | **1655.51 KB** | -| Decrypt | 100 | 1,406.56 μs | 126.286 μs | 189.019 μs | 1,408.25 μs | 138.6719 | 103.5156 | 82.0313 | 1248.58 KB | +| **Encrypt** | **1** | **29.20 μs** | **0.559 μs** | **0.836 μs** | **28.66 μs** | **3.3569** | **1.0681** | **-** | **41.24 KB** | +| Decrypt | 1 | 33.10 μs | 0.630 μs | 0.904 μs | 32.57 μs | 3.2349 | 0.7935 | - | 39.7 KB | +| **Encrypt** | **10** | **107.36 μs** | **1.942 μs** | **2.907 μs** | **107.29 μs** | **13.7939** | **0.8545** | **-** | **169.87 KB** | +| Decrypt | 10 | 117.12 μs | 1.939 μs | 2.843 μs | 118.31 μs | 12.5732 | 1.2207 | - | 154.62 KB | +| **Encrypt** | **100** | **1,489.27 μs** | **335.506 μs** | **502.170 μs** | **1,486.77 μs** | **214.8438** | **175.7813** | **140.6250** | **1655.29 KB** | +| Decrypt | 100 | 1,404.43 μs | 149.158 μs | 223.253 μs | 1,408.35 μs | 142.5781 | 107.4219 | 85.9375 | 1248.36 KB | From eb059c84917e280e22d3f7fe441ec83e518ec9c6 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 14:35:29 +0200 Subject: [PATCH 32/49] ~ unwanted changes --- Directory.Build.props | 1 - Microsoft.Azure.Cosmos.lutconfig | 6 ------ 2 files changed, 7 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.lutconfig diff --git a/Directory.Build.props b/Directory.Build.props index 3f8f26ab75..dfbe8b10fc 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,7 +12,6 @@ 10.0 $([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../')) $(DefineConstants);PREVIEW;ENCRYPTIONPREVIEW - NU1903 diff --git a/Microsoft.Azure.Cosmos.lutconfig b/Microsoft.Azure.Cosmos.lutconfig deleted file mode 100644 index 596a860301..0000000000 --- a/Microsoft.Azure.Cosmos.lutconfig +++ /dev/null @@ -1,6 +0,0 @@ - - - true - true - 180000 - \ No newline at end of file From cc2eab5cc063a446628155f3123ed80fe731377d Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 14:40:22 +0200 Subject: [PATCH 33/49] - unused method --- .../src/EncryptionProcessor.cs | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index b36619b11f..c223f44cae 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -639,27 +639,5 @@ await DecryptAsync( // and corresponding decrypted properties are added back in the documents. return BaseSerializer.ToStream(contentJObj); } - - internal static int GetOriginalBase64Length(string base64string) - { - if (string.IsNullOrEmpty(base64string)) - { - return 0; - } - - int paddingCount = 0; - int characterCount = base64string.Length; - if (base64string[characterCount - 1] == '=') - { - paddingCount++; - } - - if (base64string[characterCount - 2] == '=') - { - paddingCount++; - } - - return (3 * (characterCount / 4)) - paddingCount; - } } } \ No newline at end of file From c9ba3002002dd2f75247b1f9e5212f306fbcad75 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 19:50:25 +0200 Subject: [PATCH 34/49] ~ updates (PR) --- .../src/ArrayPoolManager.cs | 3 +- .../src/EncryptionProcessor.cs | 34 +++++++++---------- ...soft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../Readme.md | 12 +++---- .../AeadAes256CbcHmac256AlgorithmTests.cs | 1 - 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs index 9679adc1c4..3546b7155b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -32,9 +32,10 @@ protected virtual void Dispose(bool disposing) { ArrayPool.Shared.Return(buffer, clearArray: true); } + + this.rentedBuffers = null; } - this.rentedBuffers = null; this.disposedValue = true; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index c223f44cae..1c7f692fe2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -105,16 +105,16 @@ public static async Task EncryptAsync( continue; } - (typeMarker, plainText, int plainTextLength) = Serialize(propertyValue, arrayPoolManager); + (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); if (plainText == null) { continue; } - int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); + int cipherTextLength = encryptionKey.GetEncryptByteCount(plainTextLength); - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); + byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; cipherTextWithTypeMarker[0] = (byte)typeMarker; @@ -130,7 +130,7 @@ public static async Task EncryptAsync( throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - itemJObj[propertyName] = cipherTextWithTypeMarker.AsSpan(0, encryptedBytesCount + 1).ToArray(); + itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } @@ -212,8 +212,8 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); Debug.Assert(diagnosticsContext != null); - JObject itemJObj = RetrieveItem(input); - JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(itemJObj); + JObject itemJObj = EncryptionProcessor.RetrieveItem(input); + JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(itemJObj); if (encryptionPropertiesJObj == null) { @@ -225,13 +225,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( itemJObj, encryptionProperties, encryptor, @@ -255,7 +255,7 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); - JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(document); + JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(document); if (encryptionPropertiesJObj == null) { @@ -266,13 +266,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( document, encryptionProperties, encryptor, @@ -322,14 +322,14 @@ private static async Task MdeEncAlgoDecryptObjectAsync( byte[] plainText = arrayPoolManager.Rent(plainTextLength); - int decryptedCount = MdeEncAlgoDecryptProperty( + int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptProperty( encryptionKey, cipherTextWithTypeMarker, cipherTextOffset: 1, cipherTextWithTypeMarker.Length - 1, plainText); - DeserializeAndAddProperty( + EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, @@ -338,7 +338,7 @@ private static async Task MdeEncAlgoDecryptObjectAsync( pathsDecrypted.Add(path); } - DecryptionContext decryptionContext = CreateDecryptionContext( + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -417,7 +417,7 @@ private static async Task LegacyEncAlgoDecryptContentAsync( pathsDecrypted.Add("/" + property.Name); } - DecryptionContext decryptionContext = CreateDecryptionContext( + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -627,7 +627,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); using (diagnosticsContext.CreateScope("EncryptionProcessor.DeserializeAndDecryptResponseAsync")) { - await DecryptAsync( + await EncryptionProcessor.DecryptAsync( document, encryptor, diagnosticsContext, @@ -637,7 +637,7 @@ await DecryptAsync( // the contents of contentJObj get decrypted in place for MDE algorithm model, and for legacy model _ei property is removed // and corresponding decrypted properties are added back in the documents. - return BaseSerializer.ToStream(contentJObj); + return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index 4c0fae78f1..8769abdb89 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index f8c40c6bce..ced6fc95ed 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **29.20 μs** | **0.559 μs** | **0.836 μs** | **28.66 μs** | **3.3569** | **1.0681** | **-** | **41.24 KB** | -| Decrypt | 1 | 33.10 μs | 0.630 μs | 0.904 μs | 32.57 μs | 3.2349 | 0.7935 | - | 39.7 KB | -| **Encrypt** | **10** | **107.36 μs** | **1.942 μs** | **2.907 μs** | **107.29 μs** | **13.7939** | **0.8545** | **-** | **169.87 KB** | -| Decrypt | 10 | 117.12 μs | 1.939 μs | 2.843 μs | 118.31 μs | 12.5732 | 1.2207 | - | 154.62 KB | -| **Encrypt** | **100** | **1,489.27 μs** | **335.506 μs** | **502.170 μs** | **1,486.77 μs** | **214.8438** | **175.7813** | **140.6250** | **1655.29 KB** | -| Decrypt | 100 | 1,404.43 μs | 149.158 μs | 223.253 μs | 1,408.35 μs | 142.5781 | 107.4219 | 85.9375 | 1248.36 KB | +| **Encrypt** | **1** | **28.40 μs** | **0.428 μs** | **0.640 μs** | **28.40 μs** | **3.3569** | **0.8240** | **-** | **41.15 KB** | +| Decrypt | 1 | 33.19 μs | 0.532 μs | 0.779 μs | 33.54 μs | 3.2349 | 0.7935 | - | 39.7 KB | +| **Encrypt** | **10** | **105.95 μs** | **2.230 μs** | **3.337 μs** | **106.49 μs** | **13.7939** | **0.6104** | **-** | **169.78 KB** | +| Decrypt | 10 | 113.47 μs | 1.716 μs | 2.569 μs | 111.81 μs | 12.5732 | 1.2207 | - | 154.62 KB | +| **Encrypt** | **100** | **1,486.58 μs** | **389.596 μs** | **583.129 μs** | **1,487.32 μs** | **216.7969** | **177.7344** | **142.5781** | **1655.2 KB** | +| Decrypt | 100 | 1,404.48 μs | 137.824 μs | 206.288 μs | 1,409.23 μs | 144.5313 | 107.4219 | 87.8906 | 1248.31 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs index 900fac6798..c2fd0551ff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -8,7 +8,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Linq; - using System.Text; [TestClass] public class AeadAes256CbcHmac256AlgorithmTests From 9f9cbcafec0b109bffc2bfb471df34da97be3742 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 13:43:29 +0200 Subject: [PATCH 35/49] ~ add stable vs preview release duplicity --- .../{ => AeadAes}/AeAesEncryptionProcessor.cs | 0 .../AeadAes256CbcHmac256Algorithm.cs | 0 .../AeadAes256CbcHmac256EncryptionKey.cs | 0 .../src/{ => AeadAes}/SymmetricKey.cs | 0 .../src/EncryptionContainer.cs | 18 +++ .../src/EncryptionProcessor.cs | 1 + ...soft.Azure.Cosmos.Encryption.Custom.csproj | 7 +- ...zer.cs => JObjectSqlSerializer.Preview.cs} | 4 + .../JObjectSqlSerializer.Stable.cs | 85 ++++++++++++ .../MdeEncryptionProcessor.Preview.cs} | 6 +- .../MdeEncryptionProcessor.Stable.cs | 130 ++++++++++++++++++ .../{ => Transformation}/MemoryTextReader.cs | 0 12 files changed, 247 insertions(+), 4 deletions(-) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/AeAesEncryptionProcessor.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/AeadAes256CbcHmac256Algorithm.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/AeadAes256CbcHmac256EncryptionKey.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/SymmetricKey.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/{JObjectSqlSerializer.cs => JObjectSqlSerializer.Preview.cs} (99%) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{MdeEncryptionProcessor.cs => Transformation/MdeEncryptionProcessor.Preview.cs} (97%) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => Transformation}/MemoryTextReader.cs (100%) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256EncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256EncryptionKey.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256EncryptionKey.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256EncryptionKey.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/SymmetricKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/SymmetricKey.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/SymmetricKey.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/SymmetricKey.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index b898a77179..d4026cd216 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -1083,5 +1083,23 @@ private async Task> DecryptChangeFeedDocumentsAsync( return decryptItems; } + +#if IS_PREVIEW + public override Task DeleteAllItemsByPartitionKeyStreamAsync(PartitionKey partitionKey, RequestOptions requestOptions = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override Task> GetPartitionKeyRangesAsync(FeedRange feedRange, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(string processorName, ChangeFeedHandler> onChangesDelegate) + { + throw new NotImplementedException(); + } +#endif + } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 3a97b051a9..a2b4d8fb4f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -12,6 +12,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Text; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; using Newtonsoft.Json; using Newtonsoft.Json.Linq; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index 4c0fae78f1..a192203753 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -17,6 +17,7 @@ https://github.com/Azure/azure-cosmos-dotnet-v3 http://go.microsoft.com/fwlink/?LinkID=288890 microsoft;azure;cosmos;cosmosdb;documentdb;docdb;nosql;azureofficial;dotnetcore;netcore;netstandard;client;encryption;byok + IS_PREVIEW @@ -24,20 +25,22 @@ + + - + - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs similarity index 99% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs index 53ee238481..9e2e934303 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs @@ -2,6 +2,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ +#if IS_PREVIEW + namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; @@ -121,3 +123,5 @@ static T Deserialize(ReadOnlySpan serializedBytes) } } } + +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs new file mode 100644 index 0000000000..315fd82289 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs @@ -0,0 +1,85 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if !IS_PREVIEW + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Diagnostics; + using Microsoft.Data.Encryption.Cryptography.Serializers; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + internal class JObjectSqlSerializer + { + private static readonly SqlSerializerFactory SqlSerializerFactory = new(); + + // UTF-8 encoding. + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new(size: -1, codePageCharacterEncoding: 65001); + + private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; + + internal (TypeMarker, byte[]) Serialize(JToken propertyValue) + { + switch (propertyValue.Type) + { + case JTokenType.Undefined: + Debug.Assert(false, "Undefined value cannot be in the JSON"); + return (default, null); + case JTokenType.Null: + Debug.Assert(false, "Null type should have been handled by caller"); + return (TypeMarker.Null, null); + case JTokenType.Boolean: + return (TypeMarker.Boolean, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + case JTokenType.Float: + return (TypeMarker.Double, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + case JTokenType.Integer: + return (TypeMarker.Long, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + case JTokenType.String: + return (TypeMarker.String, SqlVarCharSerializer.Serialize(propertyValue.ToObject())); + case JTokenType.Array: + return (TypeMarker.Array, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + case JTokenType.Object: + return (TypeMarker.Object, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + default: + throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); + } + } + + internal virtual void DeserializeAndAddProperty( + TypeMarker typeMarker, + byte[] serializedBytes, + JObject jObject, + string key) + { + switch (typeMarker) + { + case TypeMarker.Boolean: + jObject[key] = SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes); + break; + case TypeMarker.Double: + jObject[key] = SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes); + break; + case TypeMarker.Long: + jObject[key] = SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes); + break; + case TypeMarker.String: + jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Array: + jObject[key] = JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings); + break; + case TypeMarker.Object: + jObject[key] = JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings); + break; + default: + Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); + break; + } + } + } +} + +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs similarity index 97% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 571c5b4ac7..3b34113b6a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -2,7 +2,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -namespace Microsoft.Azure.Cosmos.Encryption.Custom +#if IS_PREVIEW + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; using System.Collections.Generic; @@ -10,7 +12,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Linq; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; using Newtonsoft.Json.Linq; internal class MdeEncryptionProcessor @@ -126,3 +127,4 @@ internal async Task DecryptObjectAsync( } } } +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs new file mode 100644 index 0000000000..0eeb1f1865 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -0,0 +1,130 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if !IS_PREVIEW + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Newtonsoft.Json.Linq; + + internal class MdeEncryptionProcessor + { + internal JObjectSqlSerializer Serializer { get; set; } = new JObjectSqlSerializer(); + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + public async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + List pathsEncrypted = new (); + TypeMarker typeMarker; + + using ArrayPoolManager arrayPoolManager = new(); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { + string propertyName = pathToEncrypt.Substring(1); + if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) + { + continue; + } + + if (propertyValue.Type == JTokenType.Null) + { + continue; + } + + byte[] plainText = null; + (typeMarker, plainText) = this.Serializer.Serialize(propertyValue); + + if (plainText == null) + { + continue; + } + + (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainText.Length, arrayPoolManager); + + itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + pathsEncrypted.Add(pathToEncrypt); + } + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); + + itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); + return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + } + + internal async Task DecryptObjectAsync( + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new(); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new(encryptionProperties.EncryptedPaths.Count()); + foreach (string path in encryptionProperties.EncryptedPaths) + { + string propertyName = path.Substring(1); + if (!document.TryGetValue(propertyName, out JToken propertyValue)) + { + // malformed document, such record shouldn't be there at all + continue; + } + + byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); + if (cipherTextWithTypeMarker == null) + { + continue; + } + + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + + this.Serializer.DeserializeAndAddProperty( + (TypeMarker)cipherTextWithTypeMarker[0], + plainText.AsSpan(0, decryptedCount).ToArray(), + document, + propertyName); + + pathsDecrypted.Add(path); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + return decryptionContext; + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs From 64172b8cc616962539765e360e9c2cd14ff99887 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 14:21:37 +0200 Subject: [PATCH 36/49] ~ cleanup and parent branch merge --- .../src/Transformation/JObjectSqlSerializer.Stable.cs | 4 ++-- .../src/Transformation/MdeEncryptionProcessor.Stable.cs | 6 +++--- .../src/Transformation/MemoryTextReader.cs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs index 315fd82289..f1332baea9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs @@ -14,10 +14,10 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation internal class JObjectSqlSerializer { - private static readonly SqlSerializerFactory SqlSerializerFactory = new(); + private static readonly SqlSerializerFactory SqlSerializerFactory = new (); // UTF-8 encoding. - private static readonly SqlVarCharSerializer SqlVarCharSerializer = new(size: -1, codePageCharacterEncoding: 65001); + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index 0eeb1f1865..fc2b282d86 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -30,7 +30,7 @@ public async Task EncryptAsync( List pathsEncrypted = new (); TypeMarker typeMarker; - using ArrayPoolManager arrayPoolManager = new(); + using ArrayPoolManager arrayPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); @@ -87,11 +87,11 @@ internal async Task DecryptObjectAsync( throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } - using ArrayPoolManager arrayPoolManager = new(); + using ArrayPoolManager arrayPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - List pathsDecrypted = new(encryptionProperties.EncryptedPaths.Count()); + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { string propertyName = path.Substring(1); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs index 9326fbbf0e..ec88ccd100 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. //------------------------------------------------------------ -namespace Microsoft.Azure.Cosmos.Encryption.Custom +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; using System.Diagnostics.Contracts; From 326b1bec05e5848f3b1a11a8e15a0e062d7ac91e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 19:08:35 +0200 Subject: [PATCH 37/49] ~ master merges --- .../src/Transformation/MdeEncryptionProcessor.Preview.cs | 4 ++-- .../src/Transformation/MdeEncryptionProcessor.Stable.cs | 4 ++-- .../src/Transformation/MdeEncryptor.cs | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 3b34113b6a..a5427b015b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -55,9 +55,9 @@ public async Task EncryptAsync( continue; } - (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength); - itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + itemJObj[propertyName] = encryptedBytes; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index fc2b282d86..a861a05996 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -55,9 +55,9 @@ public async Task EncryptAsync( continue; } - (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainText.Length, arrayPoolManager); + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainText.Length); - itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + itemJObj[propertyName] = encryptedBytes.ToArray(); pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs index 9243c632d6..cf5a0ffdf6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -8,11 +8,11 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation internal class MdeEncryptor { - internal virtual (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) + internal virtual byte[] Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength) { int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1; - byte[] encryptedText = arrayPoolManager.Rent(encryptedTextLength); + byte[] encryptedText = new byte[encryptedTextLength]; encryptedText[0] = (byte)typeMarker; @@ -28,7 +28,7 @@ internal virtual (byte[] encryptedText, int encryptedLength) Encrypt(DataEncrypt throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null cipherText from {nameof(DataEncryptionKey.EncryptData)}."); } - return (encryptedText, encryptedLength + 1); + return encryptedText; } internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) From c520e16a4861aaf2d507da191500b2c436919830 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 19:15:08 +0200 Subject: [PATCH 38/49] - duplicate --- .../src/MemoryTextReader.cs | 161 ------------------ 1 file changed, 161 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs deleted file mode 100644 index 9326fbbf0e..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs +++ /dev/null @@ -1,161 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Encryption.Custom -{ - using System; - using System.Diagnostics.Contracts; - using System.IO; - - /// - /// Adjusted implementation of .Net StringReader reading from a Memory instead of a string. - /// - internal class MemoryTextReader : TextReader - { - private Memory chars; - private int length; - private int pos; - private bool closed; - - public MemoryTextReader(Memory chars) - { - this.chars = chars; - this.length = chars.Length; - } - - public override void Close() - { - this.Dispose(true); - } - - protected override void Dispose(bool disposing) - { - this.chars = null; - this.pos = 0; - this.length = 0; - this.closed = true; - base.Dispose(disposing); - } - - [Pure] - public override int Peek() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - if (this.pos == this.length) - { - return -1; - } - - return this.chars.Span[this.pos]; - } - - public override int Read() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - if (this.pos == this.length) - { - return -1; - } - - return this.chars.Span[this.pos++]; - } - - public override int Read(char[] buffer, int index, int count) - { - if (buffer == null) - { - throw new ArgumentNullException(nameof(buffer)); - } - - if (index < 0) - { - throw new ArgumentOutOfRangeException(nameof(index)); - } - - if (count < 0) - { - throw new ArgumentOutOfRangeException(nameof(count)); - } - - if (buffer.Length - index < count) - { - throw new ArgumentOutOfRangeException(); - } - - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - int n = this.length - this.pos; - if (n > 0) - { - if (n > count) - { - n = count; - } - - this.chars.Span.Slice(this.pos, n).CopyTo(buffer.AsSpan(index, n)); - this.pos += n; - } - - return n; - } - - public override string ReadToEnd() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - this.pos = this.length; - return new string(this.chars.Slice(this.pos, this.length - this.pos).ToArray()); - } - - public override string ReadLine() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - int i = this.pos; - while (i < this.length) - { - char ch = this.chars.Span[i]; - if (ch == '\r' || ch == '\n') - { - string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); - this.pos = i + 1; - if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') - { - this.pos++; - } - - return result; - } - - i++; - } - - if (i > this.pos) - { - string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); - this.pos = i; - return result; - } - - return null; - } - } -} From 5c408214623b3509ca7d4b87d380f21700e3c909 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 20:02:16 +0200 Subject: [PATCH 39/49] ~ cleanup --- .../Transformation/JObjectSqlSerializer.Preview.cs | 11 +++++------ .../Transformation/MdeEncryptionProcessor.Preview.cs | 4 +++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs index 9e2e934303..2c706aa361 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs @@ -72,13 +72,13 @@ internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedB return (buffer, length); } } -#pragma warning restore SA1101 // Prefix local calls with this internal virtual void DeserializeAndAddProperty( TypeMarker typeMarker, ReadOnlySpan serializedBytes, JObject jObject, - string key) + string key, + ArrayPoolManager arrayPoolManager) { switch (typeMarker) { @@ -105,12 +105,10 @@ internal virtual void DeserializeAndAddProperty( break; } - static T Deserialize(ReadOnlySpan serializedBytes) + T Deserialize(ReadOnlySpan serializedBytes) where T : JToken { - using ArrayPoolManager manager = new (); - - char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); + char[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); @@ -121,6 +119,7 @@ static T Deserialize(ReadOnlySpan serializedBytes) return serializer.Deserialize(reader); } } +#pragma warning restore SA1101 // Prefix local calls with this } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index a5427b015b..ccb59b0b40 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -88,6 +88,7 @@ internal async Task DecryptObjectAsync( } using ArrayPoolManager arrayPoolManager = new (); + using ArrayPoolManager charPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); @@ -113,7 +114,8 @@ internal async Task DecryptObjectAsync( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, - propertyName); + propertyName, + charPoolManager); pathsDecrypted.Add(path); } From 6dad4cdd342553def3a9bf6be12e93e4140e3de6 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 8 Oct 2024 08:06:19 +0200 Subject: [PATCH 40/49] + initial commit --- .../src/CompressionOptions.cs | 52 +++++++++++++ .../src/Constants.cs | 2 + .../src/EncryptionContainer.cs | 10 --- .../src/EncryptionOptions.cs | 5 ++ .../src/EncryptionProcessor.cs | 4 + .../src/EncryptionProperties.cs | 10 ++- ...soft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../src/Transformation/BrotliCompressor.cs | 52 +++++++++++++ .../MdeEncryptionProcessor.Preview.cs | 75 +++++++++++++++---- ...mos.Encryption.Custom.EmulatorTests.csproj | 2 +- .../EncryptionBenchmark.cs | 15 +++- ...Encryption.Custom.Performance.Tests.csproj | 2 +- .../MdeEncryptionProcessorTests.cs | 2 +- ...zure.Cosmos.Encryption.Custom.Tests.csproj | 2 +- .../Contracts/ContractEnforcement.cs | 2 + 15 files changed, 203 insertions(+), 34 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs new file mode 100644 index 0000000000..343a984c53 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs @@ -0,0 +1,52 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System.IO.Compression; + + /// + /// Options for payload compression + /// + public class CompressionOptions + { + /// + /// Supported compression algorithms + /// + /// Compression is only supported with .NET8.0+. + public enum CompressionAlgorithm + { + /// + /// No compression + /// + None = 0, +#if NET8_0_OR_GREATER + + /// + /// Brotli compression + /// + Brotli = 1, +#endif + } + + /// + /// Gets or sets compression algorithm. + /// +#if NET8_0_OR_GREATER + public CompressionAlgorithm Algorithm { get; set; } = CompressionAlgorithm.Brotli; +#else + public CompressionAlgorithm Algorithm { get; set; } = CompressionAlgorithm.None; +#endif + + /// + /// Gets or sets compression level. + /// + public CompressionLevel CompressionLevel { get; set; } = CompressionLevel.Fastest; + + /// + /// Gets or sets minimal property size for compression. + /// + public int MinimalCompressedLength { get; set; } = 128; + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs index c59c74cc67..84f4d1b79b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs @@ -13,6 +13,8 @@ internal static class Constants public const string EncryptionDekId = "_en"; public const string EncryptionFormatVersion = "_ef"; public const string EncryptedPaths = "_ep"; + public const string CompressionAlgorithm = "_ce"; + public const string CompressedEncryptedPaths = "_cp"; public const int DekPropertiesDefaultTTLInMinutes = 120; } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index d870f1b835..68edf972c6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -428,11 +428,6 @@ private async Task ReplaceItemHelperAsync( cancellationToken); } - if (partitionKey == null) - { - throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}."); - } - streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, @@ -572,11 +567,6 @@ private async Task UpsertItemHelperAsync( cancellationToken); } - if (partitionKey == null) - { - throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}."); - } - streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs index a5dff8f12b..163643f0d1 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs @@ -29,6 +29,11 @@ public sealed class EncryptionOptions /// public string EncryptionAlgorithm { get; set; } + /// + /// Gets or sets payload compression mode + /// + public CompressionOptions CompressionOptions { get; set; } = new CompressionOptions(); + /// /// Gets or sets list of JSON paths to encrypt on the payload. /// Only top level paths are supported. diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index e1e384d156..dad85d6fad 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -112,7 +112,11 @@ public static async Task EncryptAsync( } DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, itemJObj, encryptionPropertiesJObj, cancellationToken); +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else input.Dispose(); +#endif return (BaseSerializer.ToStream(itemJObj), decryptionContext); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs index 8e0ccaf84a..a369d2cea2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs @@ -24,18 +24,26 @@ internal class EncryptionProperties [JsonProperty(PropertyName = Constants.EncryptedPaths)] public IEnumerable EncryptedPaths { get; } + [JsonProperty(PropertyName = Constants.CompressionAlgorithm)] + public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; } + + [JsonProperty(PropertyName = Constants.CompressedEncryptedPaths)] + public Dictionary CompressedEncryptedPaths { get; } + public EncryptionProperties( int encryptionFormatVersion, string encryptionAlgorithm, string dataEncryptionKeyId, byte[] encryptedData, - IEnumerable encryptedPaths) + IEnumerable encryptedPaths, + Dictionary compressedEncryptedPaths = null) { this.EncryptionFormatVersion = encryptionFormatVersion; this.EncryptionAlgorithm = encryptionAlgorithm; this.DataEncryptionKeyId = dataEncryptionKeyId; this.EncryptedData = encryptedData; this.EncryptedPaths = encryptedPaths; + this.CompressedEncryptedPaths = compressedEncryptedPaths; } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index a192203753..83f2758040 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -1,6 +1,6 @@ - netstandard2.0 + netstandard2.0;net8.0 Microsoft.Azure.Cosmos.Encryption.Custom Microsoft.Azure.Cosmos.Encryption.Custom $(LangVersion) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs new file mode 100644 index 0000000000..d84ec42575 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs @@ -0,0 +1,52 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ +#if NET8_0_OR_GREATER + + using System; + using System.Diagnostics; + using System.IO.Compression; + + internal class BrotliCompressor + { + internal static int GetQualityFromCompressionLevel(CompressionLevel compressionLevel) + { + return compressionLevel switch + { + CompressionLevel.NoCompression => 0, + CompressionLevel.Fastest => 1, + CompressionLevel.Optimal => 4, + CompressionLevel.SmallestSize => 11, + _ => throw new ArgumentException("Unsupported compression level", nameof(compressionLevel)) + }; + } + + public virtual (byte[], int) Compress(EncryptionProperties properties, string path, byte[] bytes, int length, ArrayPoolManager arrayPoolManager, int compressionLevel) + { + byte[] compressedBytes = arrayPoolManager.Rent(BrotliEncoder.GetMaxCompressedLength(length)); + + if (!BrotliEncoder.TryCompress(bytes.AsSpan(0, length), compressedBytes, out int bytesWritten, compressionLevel, 22)) + { + throw new InvalidOperationException(); + } + + properties.CompressedEncryptedPaths[path] = length; + + return (compressedBytes, bytesWritten); + } + + public virtual int Decompress(byte[] inputBytes, int length, byte[] outputBytes) + { + if (!BrotliDecoder.TryDecompress(inputBytes.AsSpan(0, length), outputBytes, out int bytesWritten)) + { + throw new InvalidOperationException(); + } + + return bytesWritten; + } + } +#endif +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index ccb59b0b40..ea6dc130b5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -8,6 +8,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; using System.Collections.Generic; + using System.Diagnostics; using System.IO; using System.Linq; using System.Threading; @@ -20,6 +21,10 @@ internal class MdeEncryptionProcessor internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); +#if NET8_0_OR_GREATER + internal BrotliCompressor BrotliCompressor { get; set; } = new BrotliCompressor(); +#endif + public async Task EncryptAsync( Stream input, Encryptor encryptor, @@ -34,9 +39,27 @@ public async Task EncryptAsync( DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: 4, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted, + new Dictionary()); + +#if NET8_0_OR_GREATER + bool compress = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli; + BrotliCompressor compressor = null; + int compressionLevel = BrotliCompressor.GetQualityFromCompressionLevel(encryptionOptions.CompressionOptions.CompressionLevel); +#endif + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { +#if NET8_0_OR_GREATER + string propertyName = pathToEncrypt[1..]; +#else string propertyName = pathToEncrypt.Substring(1); +#endif if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) { continue; @@ -47,29 +70,34 @@ public async Task EncryptAsync( continue; } - byte[] plainText = null; - (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); + byte[] processedBytes = null; + (typeMarker, processedBytes, int processedBytesLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); - if (plainText == null) + if (processedBytes == null) { continue; } - byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength); +#if NET8_0_OR_GREATER + if (compress && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) + { + (processedBytes, processedBytesLength) = compressor.Compress(encryptionProperties, pathToEncrypt, processedBytes, processedBytesLength, arrayPoolManager, compressionLevel); + } +#endif + + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength); itemJObj[propertyName] = encryptedBytes; + pathsEncrypted.Add(pathToEncrypt); } - EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted); - itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else input.Dispose(); +#endif return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } @@ -82,7 +110,7 @@ internal async Task DecryptObjectAsync( { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 3) + if (encryptionProperties.EncryptionFormatVersion != 3 && encryptionProperties.EncryptionFormatVersion != 4) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } @@ -95,7 +123,11 @@ internal async Task DecryptObjectAsync( List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { +#if NET8_0_OR_GREATER + string propertyName = path[1..]; +#else string propertyName = path.Substring(1); +#endif if (!document.TryGetValue(propertyName, out JToken propertyValue)) { // malformed document, such record shouldn't be there at all @@ -108,11 +140,26 @@ internal async Task DecryptObjectAsync( continue; } - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + +#if NET8_0_OR_GREATER + if (encryptionProperties.EncryptionFormatVersion == 4) + { + if (encryptionProperties.CompressedEncryptedPaths?.TryGetValue(path, out int decompressedSize) == true) + { + byte[] buffer = arrayPoolManager.Rent(decompressedSize); + processedBytes = this.BrotliCompressor.Decompress(bytes, processedBytes, buffer); + + Debug.Assert(processedBytes == decompressedSize); + + bytes = buffer; + } + } +#endif this.Serializer.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], - plainText.AsSpan(0, decryptedCount), + bytes.AsSpan(0, processedBytes), document, propertyName, charPoolManager); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj index c5dfa8db0c..915be6c3f8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj @@ -3,7 +3,7 @@ true true AnyCPU - net6.0 + net6.0;net8.0 false false Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 638222785f..620adbcdf7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -22,9 +22,12 @@ public partial class EncryptionBenchmark private byte[]? encryptedData; private byte[]? plaintext; - [Params(1, 10, 100)] + [Params(1/*, 10, 100*/)] public int DocumentSizeInKb { get; set; } + [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)] + public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } + [GlobalSetup] public async Task Setup() { @@ -38,7 +41,7 @@ public async Task Setup() .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Randomized, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); - this.encryptionOptions = CreateEncryptionOptions(); + this.encryptionOptions = this.CreateEncryptionOptions(); this.plaintext = this.LoadTestDoc(); Stream encryptedStream = await EncryptionProcessor.EncryptAsync( @@ -74,13 +77,17 @@ await EncryptionProcessor.DecryptAsync( CancellationToken.None); } - private static EncryptionOptions CreateEncryptionOptions() + private EncryptionOptions CreateEncryptionOptions() { EncryptionOptions options = new() { DataEncryptionKeyId = "dekId", EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt + PathsToEncrypt = TestDoc.PathsToEncrypt, + CompressionOptions = new() + { + Algorithm = this.CompressionAlgorithm + } }; return options; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 4deb4d0edf..22743701c7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -3,7 +3,7 @@ Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests Exe - net6 + net8.0 enable enable diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 69a9c7afb0..6594722127 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -234,7 +234,7 @@ private static async Task VerifyEncryptionSucceeded(TestDoc testDoc) Assert.IsNotNull(encryptionProperties); Assert.AreEqual(dekId, encryptionProperties.DataEncryptionKeyId); - Assert.AreEqual(3, encryptionProperties.EncryptionFormatVersion); + Assert.AreEqual(4, encryptionProperties.EncryptionFormatVersion); Assert.IsNull(encryptionProperties.EncryptedData); Assert.IsNotNull(encryptionProperties.EncryptedPaths); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj index eee64aada9..7e1f7d48fe 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj @@ -4,7 +4,7 @@ true true AnyCPU - net6.0 + net6.0;net8.0 false false Microsoft.Azure.Cosmos.Encryption.Tests diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs index c58109d65b..7564ad3709 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs @@ -83,6 +83,7 @@ private static string GenerateNameWithClassAttributes(Type type) } } +#pragma warning disable SYSLIB0050 // Type or member is obsolete return $"{type.FullName};{baseTypeString};{nameof(type.IsAbstract)}:{(type.IsAbstract ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsSealed)}:{(type.IsSealed ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsInterface)}:{(type.IsInterface ? bool.TrueString : bool.FalseString)};" + @@ -92,6 +93,7 @@ private static string GenerateNameWithClassAttributes(Type type) $"{nameof(type.IsNested)}:{(type.IsNested ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsGenericType)}:{(type.IsGenericType ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsSerializable)}:{(type.IsSerializable ? bool.TrueString : bool.FalseString)}"; +#pragma warning restore SYSLIB0050 // Type or member is obsolete } private static string GenerateNameWithMethodAttributes(MethodInfo methodInfo) From 99c7a754d96a7f8c42adae0474b66850599982d0 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 8 Oct 2024 09:23:27 +0200 Subject: [PATCH 41/49] + Add .NET8.0 target for Cosmos.Encryption.Custom + Address new compiler complaints --- .../src/AeadAes/AeAesEncryptionProcessor.cs | 6 ++++ .../AeadAes/AeadAes256CbcHmac256Algorithm.cs | 4 +++ .../src/Common/CosmosJsonDotNetSerializer.cs | 4 +++ .../src/CosmosDataEncryptionKeyProvider.cs | 4 +++ .../src/DataEncryptionKey.cs | 4 +++ .../src/DataEncryptionKeyContainerCore.cs | 8 ++++++ .../DataEncryptionKeyContainerInlineCore.cs | 4 +++ .../src/DataEncryptionKeyFeedIterator{T}.cs | 6 ++-- .../src/DecryptableFeedResponse.cs | 4 +++ .../src/EncryptionContainer.cs | 28 ++++++++++++------- .../src/EncryptionExceptionFactory.cs | 2 ++ .../src/EncryptionFeedIterator{T}.cs | 5 ++-- .../src/EncryptionKeyWrapMetadata.cs | 13 --------- .../src/EncryptionProcessor.cs | 12 ++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 5 ++++ .../src/MdeServices/MdeKeyWrapProvider.cs | 8 ++++++ ...soft.Azure.Cosmos.Encryption.Custom.csproj | 3 +- .../src/SecurityUtility.cs | 6 ++++ .../JObjectSqlSerializer.Stable.cs | 2 +- .../MdeEncryptionProcessor.Preview.cs | 13 +++++++++ .../MdeEncryptionProcessor.Stable.cs | 12 ++++++++ .../src/Transformation/MemoryTextReader.cs | 19 +++++++++++++ .../EmulatorTests/LegacyEncryptionTests.cs | 14 ++++------ .../EmulatorTests/MdeCustomEncryptionTests.cs | 2 +- ...mos.Encryption.Custom.EmulatorTests.csproj | 2 +- ...Encryption.Custom.Performance.Tests.csproj | 2 +- ...zure.Cosmos.Encryption.Custom.Tests.csproj | 2 +- .../Contracts/ContractEnforcement.cs | 16 +++++++---- 28 files changed, 162 insertions(+), 48 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs index 0944714aac..c7eb3658d4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs @@ -13,6 +13,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using Newtonsoft.Json; using Newtonsoft.Json.Linq; +#pragma warning disable IDE0057 // Use range operator +#pragma warning disable VSTHRD103 // Call async methods when in an async method internal static class AeAesEncryptionProcessor { public static async Task EncryptAsync( @@ -65,6 +67,7 @@ public static async Task EncryptAsync( encryptionOptions.PathsToEncrypt); itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } @@ -113,4 +116,7 @@ internal static async Task DecryptContentAsync( return decryptionContext; } } + +#pragma warning restore IDE0057 // Use range operator +#pragma warning restore VSTHRD103 // Call async methods when in an async method } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs index fab34ec143..3bc0486487 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs @@ -10,6 +10,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Security.Cryptography; +#pragma warning disable SYSLIB0021 // Type or member is obsolete + /// /// This class implements authenticated encryption algorithm with associated data as described in /// http://tools.ietf.org/html/draft-mcgrew-aead-aes-cbc-hmac-sha2-05 - specifically this implements @@ -484,4 +486,6 @@ private static int GetCipherTextLength(int inputSize) return ((inputSize / BlockSizeInBytes) + 1) * BlockSizeInBytes; } } + +#pragma warning restore SYSLIB0021 // Type or member is obsolete } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs index b00c004476..9ed8056360 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs @@ -40,10 +40,14 @@ internal CosmosJsonDotNetSerializer(JsonSerializerSettings jsonSerializerSetting /// The object representing the deserialized stream public T FromStream(Stream stream) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(stream); +#else if (stream == null) { throw new ArgumentNullException(nameof(stream)); } +#endif if (typeof(Stream).IsAssignableFrom(typeof(T))) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs index cd2fe84fbc..c6f1ffb66c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs @@ -148,10 +148,14 @@ public async Task InitializeAsync( throw new InvalidOperationException($"{nameof(CosmosDataEncryptionKeyProvider)} has already been initialized."); } +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(database); +#else if (database == null) { throw new ArgumentNullException(nameof(database)); } +#endif ContainerResponse containerResponse = await database.CreateContainerIfNotExistsAsync( containerId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index b764debfd5..810227479e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -102,10 +102,14 @@ public static DataEncryptionKey Create( byte[] rawKey, string encryptionAlgorithm) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(rawKey); +#else if (rawKey == null) { throw new ArgumentNullException(nameof(rawKey)); } +#endif #pragma warning disable CS0618 // Type or member is obsolete if (!string.Equals(encryptionAlgorithm, CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized)) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs index 90cf907abc..1fcdd8c783 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs @@ -67,10 +67,14 @@ public override async Task> CreateData throw new ArgumentException(string.Format("Unsupported Encryption Algorithm {0}", encryptionAlgorithm), nameof(encryptionAlgorithm)); } +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(encryptionKeyWrapMetadata); +#else if (encryptionKeyWrapMetadata == null) { throw new ArgumentNullException(nameof(encryptionKeyWrapMetadata)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); @@ -155,10 +159,14 @@ public override async Task> RewrapData ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(newWrapMetadata); +#else if (newWrapMetadata == null) { throw new ArgumentNullException(nameof(newWrapMetadata)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs index d918d85d4a..bab5cadd62 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs @@ -78,10 +78,14 @@ public override Task> RewrapDataEncryp throw new ArgumentNullException(nameof(id)); } +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(newWrapMetadata); +#else if (newWrapMetadata == null) { throw new ArgumentNullException(nameof(newWrapMetadata)); } +#endif return TaskHelper.RunInlineIfNeededAsync(() => this.dataEncryptionKeyContainerCore.RewrapDataEncryptionKeyAsync(id, newWrapMetadata, encryptionAlgorithm, requestOptions, cancellationToken)); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs index 18e4246fdc..d01422c81b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs @@ -13,7 +13,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom internal sealed class DataEncryptionKeyFeedIterator : FeedIterator { - private readonly FeedIterator feedIterator; + private readonly DataEncryptionKeyFeedIterator feedIterator; private readonly CosmosResponseFactory responseFactory; public DataEncryptionKeyFeedIterator( @@ -57,7 +57,7 @@ public override async Task> ReadNextAsync(CancellationToken canc if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) { - dataEncryptionKeyPropertiesList = this.ConvertResponseToDataEncryptionKeyPropertiesList( + dataEncryptionKeyPropertiesList = DataEncryptionKeyFeedIterator.ConvertResponseToDataEncryptionKeyPropertiesList( responseMessage.Content); return (responseMessage, dataEncryptionKeyPropertiesList); @@ -67,7 +67,7 @@ public override async Task> ReadNextAsync(CancellationToken canc } } - private List ConvertResponseToDataEncryptionKeyPropertiesList( + private static List ConvertResponseToDataEncryptionKeyPropertiesList( Stream content) { JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream(content); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs index c16497ac20..24c07c71e4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs @@ -45,10 +45,14 @@ internal static DecryptableFeedResponse CreateResponse( ResponseMessage responseMessage, IReadOnlyCollection resource) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(responseMessage); +#else if (responseMessage == null) { throw new ArgumentNullException(nameof(responseMessage)); } +#endif using (responseMessage) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index d870f1b835..d13c1c16fe 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -122,10 +122,14 @@ public override async Task CreateItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(streamPayload); +#else if (streamPayload == null) { throw new ArgumentNullException(nameof(streamPayload)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("CreateItemStream")) @@ -304,6 +308,10 @@ public override async Task> ReplaceItemAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(id); + ArgumentNullException.ThrowIfNull(item); +#else if (id == null) { throw new ArgumentNullException(nameof(id)); @@ -313,6 +321,7 @@ public override async Task> ReplaceItemAsync( { throw new ArgumentNullException(nameof(item)); } +#endif if (requestOptions is not EncryptionItemRequestOptions encryptionItemRequestOptions || encryptionItemRequestOptions.EncryptionOptions == null) @@ -384,6 +393,10 @@ public override async Task ReplaceItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(id); + ArgumentNullException.ThrowIfNull(streamPayload); +#else if (id == null) { throw new ArgumentNullException(nameof(id)); @@ -393,6 +406,7 @@ public override async Task ReplaceItemStreamAsync( { throw new ArgumentNullException(nameof(streamPayload)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("ReplaceItemStream")) @@ -428,11 +442,6 @@ private async Task ReplaceItemHelperAsync( cancellationToken); } - if (partitionKey == null) - { - throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}."); - } - streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, @@ -536,10 +545,14 @@ public override async Task UpsertItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(streamPayload); +#else if (streamPayload == null) { throw new ArgumentNullException(nameof(streamPayload)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("UpsertItemStream")) @@ -572,11 +585,6 @@ private async Task UpsertItemHelperAsync( cancellationToken); } - if (partitionKey == null) - { - throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}."); - } - streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs index 55884f1093..3b50114cdb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs @@ -8,6 +8,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom internal static class EncryptionExceptionFactory { +#pragma warning disable CA2208 // Instantiate argument exceptions correctly internal static ArgumentException InvalidKeySize(string algorithmName, int actualKeylength, int expectedLength) { return new ArgumentException( @@ -28,6 +29,7 @@ internal static ArgumentException InvalidAlgorithmVersion(byte actual, byte expe $"Invalid encryption algorithm version; actual: {actual:X2}, expected: {expected:X2}.", "cipherText"); } +#pragma warning restore CA2208 // Instantiate argument exceptions correctly internal static ArgumentException InvalidAuthenticationTag() { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs index ee22f330ac..81f1516e1b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs @@ -11,7 +11,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom internal sealed class EncryptionFeedIterator : FeedIterator { - private readonly FeedIterator feedIterator; + private readonly EncryptionFeedIterator feedIterator; private readonly CosmosResponseFactory responseFactory; public EncryptionFeedIterator( @@ -31,8 +31,7 @@ public override async Task> ReadNextAsync(CancellationToken canc if (typeof(T) == typeof(DecryptableItem)) { IReadOnlyCollection resource; - EncryptionFeedIterator encryptionFeedIterator = this.feedIterator as EncryptionFeedIterator; - (responseMessage, resource) = await encryptionFeedIterator.ReadNextWithoutDecryptionAsync(cancellationToken); + (responseMessage, resource) = await this.feedIterator.ReadNextWithoutDecryptionAsync(cancellationToken); return DecryptableFeedResponse.CreateResponse( responseMessage, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs index e3c285531d..5855a011cf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs @@ -114,18 +114,5 @@ public bool Equals(EncryptionKeyWrapMetadata other) this.Value == other.Value && this.Name == other.Name; } - - internal string GetName(EncryptionKeyWrapMetadata encryptionKeyWrapMetadata) - { - /* A legacy DEK may not have a Name value in meta-data*/ - if (string.IsNullOrWhiteSpace(encryptionKeyWrapMetadata.Name)) - { - return encryptionKeyWrapMetadata.Value; - } - else - { - return encryptionKeyWrapMetadata.Name; - } - } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index e1e384d156..53d74390f8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -112,7 +112,11 @@ public static async Task EncryptAsync( } DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, itemJObj, encryptionPropertiesJObj, cancellationToken); +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else input.Dispose(); +#endif return (BaseSerializer.ToStream(itemJObj), decryptionContext); } @@ -181,6 +185,11 @@ private static void ValidateInputForEncrypt( Encryptor encryptor, EncryptionOptions encryptionOptions) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(encryptor); + ArgumentNullException.ThrowIfNull(encryptionOptions); +#else if (input == null) { throw new ArgumentNullException(nameof(input)); @@ -195,7 +204,9 @@ private static void ValidateInputForEncrypt( { throw new ArgumentNullException(nameof(encryptionOptions)); } +#endif +#pragma warning disable CA2208 // Instantiate argument exceptions correctly if (string.IsNullOrWhiteSpace(encryptionOptions.DataEncryptionKeyId)) { throw new ArgumentNullException(nameof(encryptionOptions.DataEncryptionKeyId)); @@ -210,6 +221,7 @@ private static void ValidateInputForEncrypt( { throw new ArgumentNullException(nameof(encryptionOptions.PathsToEncrypt)); } +#pragma warning restore CA2208 // Instantiate argument exceptions correctly } private static JObject RetrieveItem( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 963ab8705a..ff543dfc0a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -37,6 +37,10 @@ public MdeEncryptionAlgorithm( TimeSpan? cacheTimeToLive, bool withRawKey = false) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(dekProperties); + ArgumentNullException.ThrowIfNull(encryptionKeyStoreProvider); +#else if (dekProperties == null) { throw new ArgumentNullException(nameof(dekProperties)); @@ -46,6 +50,7 @@ public MdeEncryptionAlgorithm( { throw new ArgumentNullException(nameof(encryptionKeyStoreProvider)); } +#endif KeyEncryptionKey keyEncryptionKey = KeyEncryptionKey.GetOrCreate( dekProperties.EncryptionKeyWrapMetadata.Name, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs index 6520ecd67e..297a4dac7e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs @@ -28,10 +28,14 @@ public override Task UnwrapKeyAsync( EncryptionKeyWrapMetadata metadata, CancellationToken cancellationToken) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(metadata); +#else if (metadata == null) { throw new ArgumentNullException(nameof(metadata)); } +#endif KeyEncryptionKey keyEncryptionKey = KeyEncryptionKey.GetOrCreate( metadata.Name, @@ -47,10 +51,14 @@ public override Task WrapKeyAsync( EncryptionKeyWrapMetadata metadata, CancellationToken cancellationToken) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(metadata); +#else if (metadata == null) { throw new ArgumentNullException(nameof(metadata)); } +#endif KeyEncryptionKey keyEncryptionKey = KeyEncryptionKey.GetOrCreate( metadata.Name, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index a192203753..d17c1bc48b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -1,6 +1,6 @@ - netstandard2.0 + netstandard2.0;net8.0 Microsoft.Azure.Cosmos.Encryption.Custom Microsoft.Azure.Cosmos.Encryption.Custom $(LangVersion) @@ -17,6 +17,7 @@ https://github.com/Azure/azure-cosmos-dotnet-v3 http://go.microsoft.com/fwlink/?LinkID=288890 microsoft;azure;cosmos;cosmosdb;documentdb;docdb;nosql;azureofficial;dotnetcore;netcore;netstandard;client;encryption;byok + True IS_PREVIEW diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs index 632f1904d5..9a7ad82bbe 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs @@ -42,7 +42,9 @@ internal static string GetSHA256Hash(byte[] input) using (SHA256 sha256 = SHA256.Create()) { +#pragma warning disable CA1850 // Prefer static 'HashData' method over 'ComputeHash' byte[] hashValue = sha256.ComputeHash(input); +#pragma warning restore CA1850 // Prefer static 'HashData' method over 'ComputeHash' return GetHexString(hashValue); } } @@ -54,8 +56,12 @@ internal static string GetSHA256Hash(byte[] input) internal static void GenerateRandomBytes(byte[] randomBytes) { // Generate random bytes cryptographically. +#if NET8_0_OR_GREATER + RandomNumberGenerator.Fill(randomBytes); +#else using RNGCryptoServiceProvider rngCsp = new (); rngCsp.GetBytes(randomBytes); +#endif } /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs index f1332baea9..209048f907 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs @@ -21,7 +21,7 @@ internal class JObjectSqlSerializer private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; - internal (TypeMarker, byte[]) Serialize(JToken propertyValue) + internal virtual (TypeMarker, byte[]) Serialize(JToken propertyValue) { switch (propertyValue.Type) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index ccb59b0b40..b0f293a592 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -36,7 +36,11 @@ public async Task EncryptAsync( foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { +#if NET8_0_OR_GREATER + string propertyName = pathToEncrypt[1..]; +#else string propertyName = pathToEncrypt.Substring(1); +#endif if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) { continue; @@ -69,7 +73,11 @@ public async Task EncryptAsync( pathsEncrypted); itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else input.Dispose(); +#endif return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } @@ -95,7 +103,12 @@ internal async Task DecryptObjectAsync( List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { +#if NET8_0_OR_GREATER + string propertyName = path[1..]; +#else string propertyName = path.Substring(1); +#endif + if (!document.TryGetValue(propertyName, out JToken propertyValue)) { // malformed document, such record shouldn't be there at all diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index a861a05996..d4222b6f1d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -36,7 +36,11 @@ public async Task EncryptAsync( foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { +#if NET8_0_OR_GREATER + string propertyName = pathToEncrypt[1..]; +#else string propertyName = pathToEncrypt.Substring(1); +#endif if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) { continue; @@ -69,7 +73,11 @@ public async Task EncryptAsync( pathsEncrypted); itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else input.Dispose(); +#endif return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } @@ -94,7 +102,11 @@ internal async Task DecryptObjectAsync( List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { +#if NET8_0_OR_GREATER + string propertyName = path[1..]; +#else string propertyName = path.Substring(1); +#endif if (!document.TryGetValue(propertyName, out JToken propertyValue)) { // malformed document, such record shouldn't be there at all diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs index ec88ccd100..893d31b864 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs @@ -71,6 +71,12 @@ public override int Read() public override int Read(char[] buffer, int index, int count) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); + ArgumentOutOfRangeException.ThrowIfNegative(index); + ArgumentOutOfRangeException.ThrowIfNegative(count); + ArgumentOutOfRangeException.ThrowIfGreaterThan(count, buffer.Length - index); +#else if (buffer == null) { throw new ArgumentNullException(nameof(buffer)); @@ -90,6 +96,7 @@ public override int Read(char[] buffer, int index, int count) { throw new ArgumentOutOfRangeException(); } +#endif if (this.closed) { @@ -119,7 +126,11 @@ public override string ReadToEnd() } this.pos = this.length; +#if NET8_0_OR_GREATER + return new string(this.chars[this.pos..this.length].Span); +#else return new string(this.chars.Slice(this.pos, this.length - this.pos).ToArray()); +#endif } public override string ReadLine() @@ -135,7 +146,11 @@ public override string ReadLine() char ch = this.chars.Span[i]; if (ch == '\r' || ch == '\n') { +#if NET8_0_OR_GREATER + string result = new (this.chars[this.pos..i].Span); +#else string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); +#endif this.pos = i + 1; if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') { @@ -150,7 +165,11 @@ public override string ReadLine() if (i > this.pos) { +#if NET8_0_OR_GREATER + string result = new (this.chars[this.pos..i].Span); +#else string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); +#endif this.pos = i; return result; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 30f6364a32..9ee43b8b07 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1491,7 +1491,7 @@ private static EncryptionItemRequestOptions GetRequestOptions( }; } - private static TransactionalBatchItemRequestOptions GetBatchItemRequestOptions( + private static EncryptionTransactionalBatchItemRequestOptions GetBatchItemRequestOptions( string dekId, List pathsToEncrypt, string ifMatchEtag = null) @@ -1812,13 +1812,11 @@ public override Stream ToStream(T input) MemoryStream streamPayload = new(); using (StreamWriter streamWriter = new(streamPayload, encoding: Encoding.UTF8, bufferSize: 1024, leaveOpen: true)) { - using (JsonWriter writer = new JsonTextWriter(streamWriter)) - { - writer.Formatting = Formatting.None; - this.serializer.Serialize(writer, input); - writer.Flush(); - streamWriter.Flush(); - } + using JsonTextWriter writer = new (streamWriter); + writer.Formatting = Formatting.None; + this.serializer.Serialize(writer, input); + writer.Flush(); + streamWriter.Flush(); } streamPayload.Position = 0; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 661b876907..14b1915abb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -1822,7 +1822,7 @@ private static EncryptionItemRequestOptions GetRequestOptions( } } - private static TransactionalBatchItemRequestOptions GetBatchItemRequestOptions( + private static EncryptionTransactionalBatchItemRequestOptions GetBatchItemRequestOptions( string dekId, List pathsToEncrypt, string ifMatchEtag = null) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj index c5dfa8db0c..915be6c3f8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj @@ -3,7 +3,7 @@ true true AnyCPU - net6.0 + net6.0;net8.0 false false Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 4deb4d0edf..821014c014 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -3,7 +3,7 @@ Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests Exe - net6 + net8.0 enable enable diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj index eee64aada9..7e1f7d48fe 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj @@ -4,7 +4,7 @@ true true AnyCPU - net6.0 + net6.0;net8.0 false false Microsoft.Azure.Cosmos.Encryption.Tests diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs index c58109d65b..84bd8c63c1 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs @@ -14,7 +14,7 @@ public class ContractEnforcement { - private static readonly InvariantComparer invariantComparer = new InvariantComparer(); + private static readonly InvariantComparer invariantComparer = new (); private static Assembly GetAssemblyLocally(string name) { @@ -91,7 +91,10 @@ private static string GenerateNameWithClassAttributes(Type type) $"{nameof(type.IsValueType)}:{(type.IsValueType ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsNested)}:{(type.IsNested ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsGenericType)}:{(type.IsGenericType ? bool.TrueString : bool.FalseString)};" + +#pragma warning disable SYSLIB0050 // 'Type.IsSerializable' is obsolete: 'Formatter-based serialization is obsolete and should not be used. $"{nameof(type.IsSerializable)}:{(type.IsSerializable ? bool.TrueString : bool.FalseString)}"; +#pragma warning restore SYSLIB0050 // 'Type.IsSerializable' is obsolete: 'Formatter-based serialization is obsolete and should not be used. + } private static string GenerateNameWithMethodAttributes(MethodInfo methodInfo) @@ -241,7 +244,7 @@ public static void ValidatePreviewContractContainBreakingChanges( public static string GetCurrentContract(string dllName) { - TypeTree locally = new TypeTree(typeof(object)); + TypeTree locally = new (typeof(object)); Assembly assembly = ContractEnforcement.GetAssemblyLocally(dllName); Type[] exportedTypes = assembly.GetExportedTypes(); ContractEnforcement.BuildTypeTree(locally, exportedTypes); @@ -252,13 +255,13 @@ public static string GetCurrentContract(string dllName) public static string GetCurrentTelemetryContract(string dllName) { - List nonTelemetryModels = new List + List nonTelemetryModels = new() { "AzureVMMetadata", "Compute" }; - TypeTree locally = new TypeTree(typeof(object)); + TypeTree locally = new (typeof(object)); Assembly assembly = ContractEnforcement.GetAssemblyLocally(dllName); Type[] exportedTypes = assembly.GetTypes().Where(t => t!= null && @@ -328,7 +331,10 @@ public static void ValidateJsonAreSame(string baselineJson, string currentJson) private class InvariantComparer : IComparer { - public int Compare(string a, string b) => Comparer.DefaultInvariant.Compare(a, b); + public int Compare(string a, string b) + { + return Comparer.DefaultInvariant.Compare(a, b); + } } } } From 31c20e74721c88cda94b1a58633b382409fde480 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 8 Oct 2024 09:30:13 +0200 Subject: [PATCH 42/49] - remove implicit IsPreview from csproj --- .../src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index d17c1bc48b..83f2758040 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -17,7 +17,6 @@ https://github.com/Azure/azure-cosmos-dotnet-v3 http://go.microsoft.com/fwlink/?LinkID=288890 microsoft;azure;cosmos;cosmosdb;documentdb;docdb;nosql;azureofficial;dotnetcore;netcore;netstandard;client;encryption;byok - True IS_PREVIEW From 0a627bdb41a1f0f15982ca404e61aae0a1bfca8e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 8 Oct 2024 13:28:42 +0200 Subject: [PATCH 43/49] ~ fixes --- .../src/CosmosEncryptionAlgorithm.cs | 2 +- .../src/Transformation/BrotliCompressor.cs | 1 - .../MdeEncryptionProcessor.Preview.cs | 3 +-- .../EncryptionBenchmark.cs | 4 ++-- ...Encryption.Custom.Performance.Tests.csproj | 1 + .../Readme.md | 24 ++++++++++++------- 6 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs index 088965d747..574b07d687 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs @@ -21,7 +21,7 @@ public static class CosmosEncryptionAlgorithm /// MDE(Microsoft.Data.Encryption) Randomized AEAD_AES_256_CBC_HMAC_SHA256 Algorithm. /// As described here. /// - public const string MdeAeadAes256CbcHmac256Randomized = "MdeAeadAes256CbcHmac256Randomized"; + public const string MdeAeadAes256CbcHmac256Randomized = @"MdeAeadAes256CbcHmac256Randomized"; /// /// Verify if the Encryption Algorithm is supported by Cosmos. diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs index d84ec42575..d4c8fa95dd 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs @@ -7,7 +7,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation #if NET8_0_OR_GREATER using System; - using System.Diagnostics; using System.IO.Compression; internal class BrotliCompressor diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 0fe83149c5..67cce822c8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -49,7 +49,6 @@ public async Task EncryptAsync( #if NET8_0_OR_GREATER bool compress = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli; - BrotliCompressor compressor = null; int compressionLevel = BrotliCompressor.GetQualityFromCompressionLevel(encryptionOptions.CompressionOptions.CompressionLevel); #endif @@ -81,7 +80,7 @@ public async Task EncryptAsync( #if NET8_0_OR_GREATER if (compress && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) { - (processedBytes, processedBytesLength) = compressor.Compress(encryptionProperties, pathToEncrypt, processedBytes, processedBytesLength, arrayPoolManager, compressionLevel); + (processedBytes, processedBytesLength) = this.BrotliCompressor.Compress(encryptionProperties, pathToEncrypt, processedBytes, processedBytesLength, arrayPoolManager, compressionLevel); } #endif diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 620adbcdf7..2daa128663 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -22,7 +22,7 @@ public partial class EncryptionBenchmark private byte[]? encryptedData; private byte[]? plaintext; - [Params(1/*, 10, 100*/)] + [Params(1, 10, 100)] public int DocumentSizeInKb { get; set; } [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)] @@ -55,7 +55,7 @@ public async Task Setup() encryptedStream.CopyTo(memoryStream); this.encryptedData = memoryStream.ToArray(); } - + [Benchmark] public async Task Encrypt() { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 821014c014..edcdac998b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -6,6 +6,7 @@ net8.0 enable enable + true diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index ced6fc95ed..ffbccf01cd 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -3,17 +3,23 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores .NET SDK=8.0.400 - [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 + [Host] : .NET 8.0.8 (8.0.824.36612), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **28.40 μs** | **0.428 μs** | **0.640 μs** | **28.40 μs** | **3.3569** | **0.8240** | **-** | **41.15 KB** | -| Decrypt | 1 | 33.19 μs | 0.532 μs | 0.779 μs | 33.54 μs | 3.2349 | 0.7935 | - | 39.7 KB | -| **Encrypt** | **10** | **105.95 μs** | **2.230 μs** | **3.337 μs** | **106.49 μs** | **13.7939** | **0.6104** | **-** | **169.78 KB** | -| Decrypt | 10 | 113.47 μs | 1.716 μs | 2.569 μs | 111.81 μs | 12.5732 | 1.2207 | - | 154.62 KB | -| **Encrypt** | **100** | **1,486.58 μs** | **389.596 μs** | **583.129 μs** | **1,487.32 μs** | **216.7969** | **177.7344** | **142.5781** | **1655.2 KB** | -| Decrypt | 100 | 1,404.48 μs | 137.824 μs | 206.288 μs | 1,409.23 μs | 144.5313 | 107.4219 | 87.8906 | 1248.31 KB | +| Method | DocumentSizeInKb | CompressionAlgorithm | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |--------------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **None** | **23.16 μs** | **0.437 μs** | **0.655 μs** | **0.1526** | **0.0305** | **-** | **41.4 KB** | +| Decrypt | 1 | None | 27.39 μs | 0.277 μs | 0.415 μs | 0.1526 | 0.0305 | - | 40.48 KB | +| **Encrypt** | **1** | **Brotli** | **28.90 μs** | **0.302 μs** | **0.452 μs** | **0.1526** | **0.0305** | **-** | **37.63 KB** | +| Decrypt | 1 | Brotli | 31.98 μs | 0.316 μs | 0.473 μs | 0.1221 | - | - | 39.86 KB | +| **Encrypt** | **10** | **None** | **86.81 μs** | **0.494 μs** | **0.708 μs** | **0.6104** | **0.1221** | **-** | **170.02 KB** | +| Decrypt | 10 | None | 104.55 μs | 1.059 μs | 1.484 μs | 0.6104 | 0.1221 | - | 155.41 KB | +| **Encrypt** | **10** | **Brotli** | **121.61 μs** | **1.055 μs** | **1.579 μs** | **0.6104** | **0.1221** | **-** | **166.34 KB** | +| Decrypt | 10 | Brotli | 123.72 μs | 1.539 μs | 2.304 μs | 0.4883 | - | - | 140.35 KB | +| **Encrypt** | **100** | **None** | **1,106.25 μs** | **17.210 μs** | **25.227 μs** | **25.3906** | **21.4844** | **21.4844** | **1655.24 KB** | +| Decrypt | 100 | None | 1,180.62 μs | 24.202 μs | 36.224 μs | 17.5781 | 15.6250 | 15.6250 | 1248.93 KB | +| **Encrypt** | **100** | **Brotli** | **1,279.29 μs** | **12.850 μs** | **18.428 μs** | **15.6250** | **11.7188** | **11.7188** | **1355.62 KB** | +| Decrypt | 100 | Brotli | 1,208.76 μs | 39.144 μs | 53.581 μs | 11.7188 | 9.7656 | 9.7656 | 1087.69 KB | From 78da8bb0706f6977787e310020306efec36a6570 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 8 Oct 2024 14:56:39 +0200 Subject: [PATCH 44/49] + unit tests --- .../src/EncryptionOptionsExtensions.cs | 47 ++++++++++++ .../src/EncryptionProcessor.cs | 17 +---- .../src/Transformation/BrotliCompressor.cs | 4 +- .../EncryptionOptionsExtensionsTests.cs | 56 ++++++++++++++ .../MdeEncryptionProcessorTests.cs | 62 ++++++++++++---- .../Transformation/BrotliCompressionTests.cs | 74 +++++++++++++++++++ 6 files changed, 230 insertions(+), 30 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionOptionsExtensionsTests.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs new file mode 100644 index 0000000000..1297652a25 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs @@ -0,0 +1,47 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + + internal static class EncryptionOptionsExtensions + { + internal static void Validate(this EncryptionOptions options) + { + if (string.IsNullOrWhiteSpace(options.DataEncryptionKeyId)) + { +#pragma warning disable CA2208 // Instantiate argument exceptions correctly + throw new ArgumentNullException(nameof(options.DataEncryptionKeyId)); +#pragma warning restore CA2208 // Instantiate argument exceptions correctly + } + + if (string.IsNullOrWhiteSpace(options.EncryptionAlgorithm)) + { +#pragma warning disable CA2208 // Instantiate argument exceptions correctly + throw new ArgumentNullException(nameof(options.EncryptionAlgorithm)); +#pragma warning restore CA2208 // Instantiate argument exceptions correctly + } + + if (options.PathsToEncrypt == null) + { +#pragma warning disable CA2208 // Instantiate argument exceptions correctly + throw new ArgumentNullException(nameof(options.PathsToEncrypt)); +#pragma warning restore CA2208 // Instantiate argument exceptions correctly + } + + options.CompressionOptions?.Validate(); + } + + internal static void Validate(this CompressionOptions options) + { + if (options.MinimalCompressedLength < 0) + { +#pragma warning disable CA2208 // Instantiate argument exceptions correctly + throw new ArgumentOutOfRangeException(nameof(options.MinimalCompressedLength)); +#pragma warning restore CA2208 // Instantiate argument exceptions correctly + } + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 53d74390f8..72f77d6700 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -206,22 +206,7 @@ private static void ValidateInputForEncrypt( } #endif -#pragma warning disable CA2208 // Instantiate argument exceptions correctly - if (string.IsNullOrWhiteSpace(encryptionOptions.DataEncryptionKeyId)) - { - throw new ArgumentNullException(nameof(encryptionOptions.DataEncryptionKeyId)); - } - - if (string.IsNullOrWhiteSpace(encryptionOptions.EncryptionAlgorithm)) - { - throw new ArgumentNullException(nameof(encryptionOptions.EncryptionAlgorithm)); - } - - if (encryptionOptions.PathsToEncrypt == null) - { - throw new ArgumentNullException(nameof(encryptionOptions.PathsToEncrypt)); - } -#pragma warning restore CA2208 // Instantiate argument exceptions correctly + encryptionOptions.Validate(); } private static JObject RetrieveItem( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs index d4c8fa95dd..e461ce32a4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs @@ -11,6 +11,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation internal class BrotliCompressor { + private const int DefaultWindow = 22; + internal static int GetQualityFromCompressionLevel(CompressionLevel compressionLevel) { return compressionLevel switch @@ -27,7 +29,7 @@ public virtual (byte[], int) Compress(EncryptionProperties properties, string pa { byte[] compressedBytes = arrayPoolManager.Rent(BrotliEncoder.GetMaxCompressedLength(length)); - if (!BrotliEncoder.TryCompress(bytes.AsSpan(0, length), compressedBytes, out int bytesWritten, compressionLevel, 22)) + if (!BrotliEncoder.TryCompress(bytes.AsSpan(0, length), compressedBytes, out int bytesWritten, compressionLevel, BrotliCompressor.DefaultWindow)) { throw new InvalidOperationException(); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionOptionsExtensionsTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionOptionsExtensionsTests.cs new file mode 100644 index 0000000000..373a120a60 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionOptionsExtensionsTests.cs @@ -0,0 +1,56 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Tests +{ + using System; + using System.Collections.Generic; + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class EncryptionOptionsExtensionsTests + { + [TestMethod] + public void Validate_EncryptionOptions_Throws() + { + Assert.ThrowsException(() => new EncryptionOptions() + { + DataEncryptionKeyId = null, + EncryptionAlgorithm = "something", + PathsToEncrypt = new List() + }.Validate()); + + Assert.ThrowsException(() => new EncryptionOptions() + { + DataEncryptionKeyId = "something", + EncryptionAlgorithm = null, + PathsToEncrypt = new List() + }.Validate()); + + Assert.ThrowsException(() => new EncryptionOptions() + { + DataEncryptionKeyId = "something", + EncryptionAlgorithm = "something", + PathsToEncrypt = null + }.Validate()); + + Assert.ThrowsException(() => new EncryptionOptions() + { + DataEncryptionKeyId = "something", + EncryptionAlgorithm = "something", + PathsToEncrypt = new List(), + CompressionOptions = new CompressionOptions() + { + MinimalCompressedLength = -1 + } + }.Validate()); + } + + [TestMethod] + public void Validate_CompressionOptions_Throws() + { + Assert.ThrowsException(() => new CompressionOptions() + { + MinimalCompressedLength = -1 + }.Validate()); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 6594722127..cfd6a80923 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -20,19 +20,12 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests public class MdeEncryptionProcessorTests { private static Mock mockEncryptor; - private static EncryptionOptions encryptionOptions; private const string dekId = "dekId"; [ClassInitialize] public static void ClassInitialize(TestContext testContext) { _ = testContext; - encryptionOptions = new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt - }; Mock DekMock = new(); DekMock.Setup(m => m.EncryptData(It.IsAny())) @@ -125,12 +118,13 @@ await EncryptionProcessor.EncryptAsync( } [TestMethod] - public async Task EncryptDecryptPropertyWithNullValue() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task EncryptDecryptPropertyWithNullValue(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); testDoc.SensitiveStr = null; - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc); + JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -146,11 +140,12 @@ public async Task EncryptDecryptPropertyWithNullValue() } [TestMethod] - public async Task ValidateEncryptDecryptDocument() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task ValidateEncryptDecryptDocument(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc); + JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -166,7 +161,8 @@ public async Task ValidateEncryptDecryptDocument() } [TestMethod] - public async Task ValidateDecryptStream() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task ValidateDecryptStream(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); @@ -209,7 +205,7 @@ public async Task DecryptStreamWithoutEncryptedProperty() Assert.IsNull(decryptionContext); } - private static async Task VerifyEncryptionSucceeded(TestDoc testDoc) + private static async Task VerifyEncryptionSucceeded(TestDoc testDoc, EncryptionOptions encryptionOptions) { Stream encryptedStream = await EncryptionProcessor.EncryptAsync( testDoc.ToStream(), @@ -287,5 +283,45 @@ private static void VerifyDecryptionSucceeded( } } } + + public static IEnumerable EncryptionOptionsCombinations => new[] { + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.None + } + } + }, +#if NET8_0_OR_GREATER + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, + CompressionLevel = System.IO.Compression.CompressionLevel.Fastest + } + } + }, + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, + CompressionLevel = System.IO.Compression.CompressionLevel.NoCompression, + } + } + } +#endif + }; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs new file mode 100644 index 0000000000..2ac1ac4bb4 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs @@ -0,0 +1,74 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation +{ +#if NET8_0_OR_GREATER + using System; + using System.IO.Compression; + using System.Linq; + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class BrotliCompressionTests + { + [TestMethod] + [DataRow(CompressionLevel.NoCompression, 0)] + [DataRow(CompressionLevel.Fastest, 1)] + [DataRow(CompressionLevel.Optimal, 4)] + [DataRow(CompressionLevel.SmallestSize, 11)] + public void GetQuality_WorksForKnownLevels(CompressionLevel compressionLevel, int expectedQuality) + { + Assert.AreEqual(BrotliCompressor.GetQualityFromCompressionLevel(compressionLevel), expectedQuality); + } + + [TestMethod] + public void GetQuality_ThrowsForUnknownLevels() + { + Assert.ThrowsException(() => BrotliCompressor.GetQualityFromCompressionLevel((CompressionLevel)(-999))); + } + + [TestMethod] + [DataRow(CompressionLevel.NoCompression, 256)] + [DataRow(CompressionLevel.Fastest, 256)] + [DataRow(CompressionLevel.Optimal, 256)] + [DataRow(CompressionLevel.SmallestSize, 256)] + [DataRow(CompressionLevel.NoCompression, 1024)] + [DataRow(CompressionLevel.Fastest, 1024)] + [DataRow(CompressionLevel.Optimal, 1024)] + [DataRow(CompressionLevel.SmallestSize, 1024)] + [DataRow(CompressionLevel.NoCompression, 4096)] + [DataRow(CompressionLevel.Fastest, 4096)] + [DataRow(CompressionLevel.Optimal, 4096)] + [DataRow(CompressionLevel.SmallestSize, 4096)] + public void CompressAndDecompress_HasSameResult(CompressionLevel compressionLevel, int payloadSize) + { + BrotliCompressor compressor = new (); + EncryptionProperties properties = new (0, "", "", null, null, new()); + string path = "somePath"; + + byte[] bytes = new byte[payloadSize]; + bytes.AsSpan().Fill(127); + ArrayPoolManager manager = new (); + + (byte[] compressedBytes, int compressedBytesSize) = compressor.Compress(properties, path, bytes, bytes.Length, manager, BrotliCompressor.GetQualityFromCompressionLevel(compressionLevel)); + + Assert.AreNotSame(bytes, compressedBytes); + Assert.IsTrue(compressedBytesSize > 0); + Assert.IsTrue(compressedBytesSize < bytes.Length); + + Console.WriteLine($"Original: {bytes.Length} Compressed: {compressedBytesSize}"); + + Assert.IsTrue(properties.CompressedEncryptedPaths.ContainsKey(path)); + + int recordedSize = properties.CompressedEncryptedPaths["somePath"]; + Assert.AreEqual(bytes.Length, recordedSize); + + byte[] decompressedBytes = new byte[recordedSize]; + int decompressedBytesSize = compressor.Decompress(compressedBytes, compressedBytesSize, decompressedBytes); + + Assert.AreEqual(decompressedBytesSize, bytes.Length); + Assert.IsTrue(bytes.SequenceEqual(decompressedBytes)); + } + } +#endif +} From ff08a1f1cedbb6a3567c4eba37282c42b013d4c8 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 09:46:10 +0200 Subject: [PATCH 45/49] ~ post merge fixes --- .../src/Transformation/BrotliCompressor.cs | 2 +- .../MdeEncryptionProcessor.Preview.cs | 7 ----- .../Readme.md | 28 +++++++++---------- ...zure.Cosmos.Encryption.Custom.Tests.csproj | 13 +++++---- .../Transformation/BrotliCompressionTests.cs | 2 +- 5 files changed, 23 insertions(+), 29 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs index e461ce32a4..2b3f624636 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs @@ -4,7 +4,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER using System; using System.IO.Compression; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 763b40feed..6939f936c7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -91,13 +91,6 @@ public async Task EncryptAsync( pathsEncrypted.Add(pathToEncrypt); } - EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted); - itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); #if NET8_0_OR_GREATER await input.DisposeAsync(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index ffbccf01cd..933968ba15 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -1,9 +1,9 @@ ``` ini -BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) +BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4317) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores .NET SDK=8.0.400 - [Host] : .NET 8.0.8 (8.0.824.36612), X64 RyuJIT AVX2 + [Host] : .NET 8.0.10 (8.0.1024.46610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 @@ -11,15 +11,15 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | CompressionAlgorithm | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |--------------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **None** | **23.16 μs** | **0.437 μs** | **0.655 μs** | **0.1526** | **0.0305** | **-** | **41.4 KB** | -| Decrypt | 1 | None | 27.39 μs | 0.277 μs | 0.415 μs | 0.1526 | 0.0305 | - | 40.48 KB | -| **Encrypt** | **1** | **Brotli** | **28.90 μs** | **0.302 μs** | **0.452 μs** | **0.1526** | **0.0305** | **-** | **37.63 KB** | -| Decrypt | 1 | Brotli | 31.98 μs | 0.316 μs | 0.473 μs | 0.1221 | - | - | 39.86 KB | -| **Encrypt** | **10** | **None** | **86.81 μs** | **0.494 μs** | **0.708 μs** | **0.6104** | **0.1221** | **-** | **170.02 KB** | -| Decrypt | 10 | None | 104.55 μs | 1.059 μs | 1.484 μs | 0.6104 | 0.1221 | - | 155.41 KB | -| **Encrypt** | **10** | **Brotli** | **121.61 μs** | **1.055 μs** | **1.579 μs** | **0.6104** | **0.1221** | **-** | **166.34 KB** | -| Decrypt | 10 | Brotli | 123.72 μs | 1.539 μs | 2.304 μs | 0.4883 | - | - | 140.35 KB | -| **Encrypt** | **100** | **None** | **1,106.25 μs** | **17.210 μs** | **25.227 μs** | **25.3906** | **21.4844** | **21.4844** | **1655.24 KB** | -| Decrypt | 100 | None | 1,180.62 μs | 24.202 μs | 36.224 μs | 17.5781 | 15.6250 | 15.6250 | 1248.93 KB | -| **Encrypt** | **100** | **Brotli** | **1,279.29 μs** | **12.850 μs** | **18.428 μs** | **15.6250** | **11.7188** | **11.7188** | **1355.62 KB** | -| Decrypt | 100 | Brotli | 1,208.76 μs | 39.144 μs | 53.581 μs | 11.7188 | 9.7656 | 9.7656 | 1087.69 KB | +| **Encrypt** | **1** | **None** | **23.77 μs** | **0.445 μs** | **0.652 μs** | **0.1526** | **0.0305** | **-** | **41.4 KB** | +| Decrypt | 1 | None | 28.50 μs | 0.296 μs | 0.434 μs | 0.1526 | 0.0305 | - | 40.48 KB | +| **Encrypt** | **1** | **Brotli** | **30.73 μs** | **0.636 μs** | **0.912 μs** | **0.1221** | **-** | **-** | **37.63 KB** | +| Decrypt | 1 | Brotli | 33.83 μs | 0.468 μs | 0.686 μs | 0.1221 | - | - | 39.86 KB | +| **Encrypt** | **10** | **None** | **88.16 μs** | **0.688 μs** | **1.009 μs** | **0.6104** | **0.1221** | **-** | **170.02 KB** | +| Decrypt | 10 | None | 107.81 μs | 1.161 μs | 1.737 μs | 0.6104 | 0.1221 | - | 155.41 KB | +| **Encrypt** | **10** | **Brotli** | **125.53 μs** | **1.229 μs** | **1.802 μs** | **0.4883** | **-** | **-** | **166.34 KB** | +| Decrypt | 10 | Brotli | 126.47 μs | 1.007 μs | 1.507 μs | 0.4883 | - | - | 140.35 KB | +| **Encrypt** | **100** | **None** | **1,125.90 μs** | **46.668 μs** | **69.851 μs** | **25.3906** | **23.4375** | **21.4844** | **1655.17 KB** | +| Decrypt | 100 | None | 1,144.44 μs | 19.388 μs | 28.419 μs | 17.5781 | 15.6250 | 15.6250 | 1248.93 KB | +| **Encrypt** | **100** | **Brotli** | **1,281.09 μs** | **19.431 μs** | **28.481 μs** | **15.6250** | **13.6719** | **11.7188** | **1355.61 KB** | +| Decrypt | 100 | Brotli | 1,182.40 μs | 16.788 μs | 25.127 μs | 11.7188 | 9.7656 | 9.7656 | 1087.68 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj index 7e1f7d48fe..66a0c2da24 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj @@ -1,4 +1,4 @@ - + true @@ -9,6 +9,7 @@ false Microsoft.Azure.Cosmos.Encryption.Tests $(LangVersion) + $(DefineConstants);ENCRYPTION_CUSTOM_PREVIEW @@ -31,20 +32,20 @@ - - + + - + - + - + true true diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs index 2ac1ac4bb4..8e66b921ed 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs @@ -1,6 +1,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation { -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER using System; using System.IO.Compression; using System.Linq; From a058cb124e4dffd1e4a11f10f53d4eb2f3606d5b Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 19:27:12 +0200 Subject: [PATCH 46/49] ~ fix tests for version difference between stable and preview mode --- .../MdeEncryptionProcessorTests.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index cfd6a80923..2314f624d3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -230,7 +230,11 @@ private static async Task VerifyEncryptionSucceeded(TestDoc testDoc, En Assert.IsNotNull(encryptionProperties); Assert.AreEqual(dekId, encryptionProperties.DataEncryptionKeyId); +#if ENCRYPTION_CUSTOM_PREVIEW Assert.AreEqual(4, encryptionProperties.EncryptionFormatVersion); +#else + Assert.AreEqual(3, encryptionProperties.EncryptionFormatVersion); +#endif Assert.IsNull(encryptionProperties.EncryptedData); Assert.IsNotNull(encryptionProperties.EncryptedPaths); From 8630cd4017b031f497769c67396d34c533d2a48a Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 11 Oct 2024 07:06:27 +0200 Subject: [PATCH 47/49] ~ cleanup --- .../src/EncryptionContainer.cs | 18 ------------------ .../src/EncryptionProperties.cs | 4 ++-- .../EncryptionBenchmark.cs | 2 +- ...Azure.Cosmos.Encryption.Custom.Tests.csproj | 16 ++++++++-------- .../Transformation/BrotliCompressionTests.cs | 3 ++- .../Contracts/ContractEnforcement.cs | 1 - 6 files changed, 13 insertions(+), 31 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index b6970f621f..f769e1504a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -1105,23 +1105,5 @@ private async Task> DecryptChangeFeedDocumentsAsync( return decryptItems; } - -#if IS_PREVIEW - public override Task DeleteAllItemsByPartitionKeyStreamAsync(PartitionKey partitionKey, RequestOptions requestOptions = null, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public override Task> GetPartitionKeyRangesAsync(FeedRange feedRange, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(string processorName, ChangeFeedHandler> onChangesDelegate) - { - throw new NotImplementedException(); - } -#endif - } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs index a369d2cea2..99e2f5e99e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs @@ -28,7 +28,7 @@ internal class EncryptionProperties public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; } [JsonProperty(PropertyName = Constants.CompressedEncryptedPaths)] - public Dictionary CompressedEncryptedPaths { get; } + public IDictionary CompressedEncryptedPaths { get; } public EncryptionProperties( int encryptionFormatVersion, @@ -36,7 +36,7 @@ public EncryptionProperties( string dataEncryptionKeyId, byte[] encryptedData, IEnumerable encryptedPaths, - Dictionary compressedEncryptedPaths = null) + IDictionary compressedEncryptedPaths = null) { this.EncryptionFormatVersion = encryptionFormatVersion; this.EncryptionAlgorithm = encryptionAlgorithm; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 2daa128663..910817db3d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -55,7 +55,7 @@ public async Task Setup() encryptedStream.CopyTo(memoryStream); this.encryptedData = memoryStream.ToArray(); } - + [Benchmark] public async Task Encrypt() { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj index 66a0c2da24..0968e6b3dc 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj @@ -31,21 +31,21 @@ - - - + + + - + - + - + - + - + true true diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs index 8e66b921ed..529619d11a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs @@ -2,6 +2,7 @@ { #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER using System; + using System.Collections.Generic; using System.IO.Compression; using System.Linq; using Microsoft.Azure.Cosmos.Encryption.Custom; @@ -43,7 +44,7 @@ public void GetQuality_ThrowsForUnknownLevels() public void CompressAndDecompress_HasSameResult(CompressionLevel compressionLevel, int payloadSize) { BrotliCompressor compressor = new (); - EncryptionProperties properties = new (0, "", "", null, null, new()); + EncryptionProperties properties = new (0, "", "", null, null, new Dictionary()); string path = "somePath"; byte[] bytes = new byte[payloadSize]; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs index 69162a3f13..84bd8c63c1 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs @@ -83,7 +83,6 @@ private static string GenerateNameWithClassAttributes(Type type) } } -#pragma warning disable SYSLIB0050 // Type or member is obsolete return $"{type.FullName};{baseTypeString};{nameof(type.IsAbstract)}:{(type.IsAbstract ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsSealed)}:{(type.IsSealed ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsInterface)}:{(type.IsInterface ? bool.TrueString : bool.FalseString)};" + From 450df8b2ebc5e79cc81867e8ffa5983397ba18e3 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 11 Oct 2024 20:30:18 +0200 Subject: [PATCH 48/49] ~ refactor as pre PR --- .../src/EncryptionProperties.cs | 2 + .../src/Transformation/BrotliCompressor.cs | 78 ++++++++++++++++--- .../MdeEncryptionProcessor.Preview.cs | 43 +++++++--- .../Readme.md | 28 +++---- .../EncryptionPropertiesTests.cs | 54 +++++++++++++ .../Transformation/BrotliCompressionTests.cs | 8 +- 6 files changed, 172 insertions(+), 41 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs index 99e2f5e99e..70f385e064 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs @@ -36,6 +36,7 @@ public EncryptionProperties( string dataEncryptionKeyId, byte[] encryptedData, IEnumerable encryptedPaths, + CompressionOptions.CompressionAlgorithm compressionAlgorithm = CompressionOptions.CompressionAlgorithm.None, IDictionary compressedEncryptedPaths = null) { this.EncryptionFormatVersion = encryptionFormatVersion; @@ -43,6 +44,7 @@ public EncryptionProperties( this.DataEncryptionKeyId = dataEncryptionKeyId; this.EncryptedData = encryptedData; this.EncryptedPaths = encryptedPaths; + this.CompressionAlgorithm = compressionAlgorithm; this.CompressedEncryptedPaths = compressedEncryptedPaths; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs index 2b3f624636..fc293507d1 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs @@ -7,9 +7,10 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER using System; + using System.Buffers; using System.IO.Compression; - internal class BrotliCompressor + internal class BrotliCompressor : IDisposable { private const int DefaultWindow = 22; @@ -25,28 +26,83 @@ internal static int GetQualityFromCompressionLevel(CompressionLevel compressionL }; } - public virtual (byte[], int) Compress(EncryptionProperties properties, string path, byte[] bytes, int length, ArrayPoolManager arrayPoolManager, int compressionLevel) + internal static int GetMaxCompressedSize(int inputSize) { - byte[] compressedBytes = arrayPoolManager.Rent(BrotliEncoder.GetMaxCompressedLength(length)); + return BrotliEncoder.GetMaxCompressedLength(inputSize); + } - if (!BrotliEncoder.TryCompress(bytes.AsSpan(0, length), compressedBytes, out int bytesWritten, compressionLevel, BrotliCompressor.DefaultWindow)) - { - throw new InvalidOperationException(); - } + private readonly BrotliDecoder decoder; + private readonly BrotliEncoder encoder; + private bool disposedValue; + + public BrotliCompressor() + { + } + + public BrotliCompressor(CompressionLevel compressionLevel) + { + this.encoder = new BrotliEncoder(BrotliCompressor.GetQualityFromCompressionLevel(compressionLevel), DefaultWindow); + } + + public virtual int Compress(EncryptionProperties properties, string path, byte[] bytes, int length, byte[] outputBytes) + { + OperationStatus status = this.encoder.Compress(bytes.AsSpan(0, length), outputBytes, out int bytesConsumed, out int bytesWritten, true); + + ThrowIfFailure(status, length, bytesConsumed); properties.CompressedEncryptedPaths[path] = length; - return (compressedBytes, bytesWritten); + return bytesWritten; } public virtual int Decompress(byte[] inputBytes, int length, byte[] outputBytes) { - if (!BrotliDecoder.TryDecompress(inputBytes.AsSpan(0, length), outputBytes, out int bytesWritten)) + OperationStatus status = this.decoder.Decompress(inputBytes.AsSpan(0, length), outputBytes, out int bytesConsumed, out int bytesWritten); + + ThrowIfFailure(status, length, bytesConsumed); + + return bytesWritten; + } + + private static void ThrowIfFailure(OperationStatus status, int expectedLength, int processedLength) + { + if (status != OperationStatus.Done) { - throw new InvalidOperationException(); + throw new InvalidOperationException($"Brotli compressor failed : {status}"); } - return bytesWritten; + if (expectedLength != processedLength) + { + throw new InvalidOperationException($"Expected to process {expectedLength} but only processed {processedLength}"); + } + } + + protected virtual void Dispose(bool disposing) + { + if (!this.disposedValue) + { + if (disposing) + { + } + + // TODO: free unmanaged resources (unmanaged objects) and override finalizer + this.encoder.Dispose(); + + this.disposedValue = true; + } + } + + ~BrotliCompressor() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + this.Dispose(disposing: false); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + this.Dispose(disposing: true); + GC.SuppressFinalize(this); } } #endif diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 6939f936c7..0974125b0f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -21,10 +21,6 @@ internal class MdeEncryptionProcessor internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); -#if NET8_0_OR_GREATER - internal BrotliCompressor BrotliCompressor { get; set; } = new BrotliCompressor(); -#endif - public async Task EncryptAsync( Stream input, Encryptor encryptor, @@ -45,11 +41,12 @@ public async Task EncryptAsync( encryptionOptions.DataEncryptionKeyId, encryptedData: null, pathsEncrypted, + encryptionOptions.CompressionOptions.Algorithm, new Dictionary()); #if NET8_0_OR_GREATER - bool compress = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli; - int compressionLevel = BrotliCompressor.GetQualityFromCompressionLevel(encryptionOptions.CompressionOptions.CompressionLevel); + BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli + ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; #endif foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) @@ -78,9 +75,11 @@ public async Task EncryptAsync( } #if NET8_0_OR_GREATER - if (compress && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) + if (compressor != null && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) { - (processedBytes, processedBytesLength) = this.BrotliCompressor.Compress(encryptionProperties, pathToEncrypt, processedBytes, processedBytesLength, arrayPoolManager, compressionLevel); + byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(processedBytesLength)); + processedBytesLength = compressor.Compress(encryptionProperties, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes); + processedBytes = compressedBytes; } #endif @@ -91,6 +90,10 @@ public async Task EncryptAsync( pathsEncrypted.Add(pathToEncrypt); } +#if NET8_0_OR_GREATER + compressor?.Dispose(); +#endif + itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); #if NET8_0_OR_GREATER await input.DisposeAsync(); @@ -120,6 +123,24 @@ internal async Task DecryptObjectAsync( DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); + +#if NET8_0_OR_GREATER + BrotliCompressor decompressor = null; + if (encryptionProperties.EncryptionFormatVersion == 4) + { + bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true; + if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) + { + throw new NotSupportedException($"Unknown compression algorithm {encryptionProperties.CompressionAlgorithm}"); + } + + if (containsCompressed) + { + decompressor = new (); + } + } +#endif + foreach (string path in encryptionProperties.EncryptedPaths) { #if NET8_0_OR_GREATER @@ -143,14 +164,12 @@ internal async Task DecryptObjectAsync( (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); #if NET8_0_OR_GREATER - if (encryptionProperties.EncryptionFormatVersion == 4) + if (decompressor != null) { if (encryptionProperties.CompressedEncryptedPaths?.TryGetValue(path, out int decompressedSize) == true) { byte[] buffer = arrayPoolManager.Rent(decompressedSize); - processedBytes = this.BrotliCompressor.Decompress(bytes, processedBytes, buffer); - - Debug.Assert(processedBytes == decompressedSize); + processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); bytes = buffer; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 933968ba15..2913528571 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -1,8 +1,8 @@ ``` ini -BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4317) +BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.26100.2033) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=8.0.400 +.NET SDK=8.0.403 [Host] : .NET 8.0.10 (8.0.1024.46610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,15 +11,15 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | CompressionAlgorithm | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |--------------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **None** | **23.77 μs** | **0.445 μs** | **0.652 μs** | **0.1526** | **0.0305** | **-** | **41.4 KB** | -| Decrypt | 1 | None | 28.50 μs | 0.296 μs | 0.434 μs | 0.1526 | 0.0305 | - | 40.48 KB | -| **Encrypt** | **1** | **Brotli** | **30.73 μs** | **0.636 μs** | **0.912 μs** | **0.1221** | **-** | **-** | **37.63 KB** | -| Decrypt | 1 | Brotli | 33.83 μs | 0.468 μs | 0.686 μs | 0.1221 | - | - | 39.86 KB | -| **Encrypt** | **10** | **None** | **88.16 μs** | **0.688 μs** | **1.009 μs** | **0.6104** | **0.1221** | **-** | **170.02 KB** | -| Decrypt | 10 | None | 107.81 μs | 1.161 μs | 1.737 μs | 0.6104 | 0.1221 | - | 155.41 KB | -| **Encrypt** | **10** | **Brotli** | **125.53 μs** | **1.229 μs** | **1.802 μs** | **0.4883** | **-** | **-** | **166.34 KB** | -| Decrypt | 10 | Brotli | 126.47 μs | 1.007 μs | 1.507 μs | 0.4883 | - | - | 140.35 KB | -| **Encrypt** | **100** | **None** | **1,125.90 μs** | **46.668 μs** | **69.851 μs** | **25.3906** | **23.4375** | **21.4844** | **1655.17 KB** | -| Decrypt | 100 | None | 1,144.44 μs | 19.388 μs | 28.419 μs | 17.5781 | 15.6250 | 15.6250 | 1248.93 KB | -| **Encrypt** | **100** | **Brotli** | **1,281.09 μs** | **19.431 μs** | **28.481 μs** | **15.6250** | **13.6719** | **11.7188** | **1355.61 KB** | -| Decrypt | 100 | Brotli | 1,182.40 μs | 16.788 μs | 25.127 μs | 11.7188 | 9.7656 | 9.7656 | 1087.68 KB | +| **Encrypt** | **1** | **None** | **22.45 μs** | **0.447 μs** | **0.655 μs** | **0.1526** | **0.0305** | **-** | **40.94 KB** | +| Decrypt | 1 | None | 26.65 μs | 0.165 μs | 0.247 μs | 0.1526 | 0.0305 | - | 40.32 KB | +| **Encrypt** | **1** | **Brotli** | **27.92 μs** | **0.212 μs** | **0.317 μs** | **0.1526** | **0.0305** | **-** | **37.3 KB** | +| Decrypt | 1 | Brotli | 33.45 μs | 0.718 μs | 1.075 μs | 0.1221 | - | - | 39.95 KB | +| **Encrypt** | **10** | **None** | **83.56 μs** | **0.358 μs** | **0.502 μs** | **0.6104** | **0.1221** | **-** | **167.12 KB** | +| Decrypt | 10 | None | 100.88 μs | 0.437 μs | 0.627 μs | 0.6104 | 0.1221 | - | 153.59 KB | +| **Encrypt** | **10** | **Brotli** | **111.94 μs** | **0.456 μs** | **0.669 μs** | **0.6104** | **0.1221** | **-** | **164.26 KB** | +| Decrypt | 10 | Brotli | 121.06 μs | 2.794 μs | 4.182 μs | 0.4883 | - | - | 141.31 KB | +| **Encrypt** | **100** | **None** | **1,194.10 μs** | **37.744 μs** | **56.494 μs** | **23.4375** | **23.4375** | **21.4844** | **1638.78 KB** | +| Decrypt | 100 | None | 1,247.89 μs | 32.037 μs | 47.952 μs | 17.5781 | 15.6250 | 15.6250 | 1230.56 KB | +| **Encrypt** | **100** | **Brotli** | **1,199.53 μs** | **30.018 μs** | **44.930 μs** | **13.6719** | **11.7188** | **9.7656** | **1347 KB** | +| Decrypt | 100 | Brotli | 1,196.64 μs | 22.702 μs | 33.979 μs | 11.7188 | 9.7656 | 9.7656 | 1097.75 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs new file mode 100644 index 0000000000..5ce666f2b5 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs @@ -0,0 +1,54 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Tests +{ + using System; + using System.Collections.Generic; + using System.Linq; + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class EncryptionPropertiesTests + { + [TestMethod] + public void Ctor_AssignsAllMandatoryProperties() + { + EncryptionProperties properties = new EncryptionProperties( + 11, + "algorithm", + "dek-id", + new byte[] { 1, 2, 3, 4, 5 }, + new List { "a", "b" }); + + Assert.AreEqual(11, properties.EncryptionFormatVersion); + Assert.AreEqual("algorithm", properties.EncryptionAlgorithm); + Assert.AreEqual("dek-id", properties.DataEncryptionKeyId); + Assert.IsTrue(new byte[] { 1, 2, 3, 4, 5}.SequenceEqual(properties.EncryptedData)); + Assert.IsTrue(new List { "a", "b"}.SequenceEqual(properties.EncryptedPaths)); + Assert.AreEqual(CompressionOptions.CompressionAlgorithm.None, properties.CompressionAlgorithm); + Assert.IsNull(properties.CompressedEncryptedPaths); + } + +#if NET8_0_OR_GREATER + [TestMethod] + public void Ctor_AssignsAllProperties() + { + EncryptionProperties properties = new EncryptionProperties( + 11, + "algorithm", + "dek-id", + new byte[] { 1, 2, 3, 4, 5 }, + new List { "a", "b" }, + CompressionOptions.CompressionAlgorithm.Brotli, + new Dictionary { { "a", 246 } }); + + Assert.AreEqual(11, properties.EncryptionFormatVersion); + Assert.AreEqual("algorithm", properties.EncryptionAlgorithm); + Assert.AreEqual("dek-id", properties.DataEncryptionKeyId); + Assert.IsTrue(new byte[] { 1, 2, 3, 4, 5 }.SequenceEqual(properties.EncryptedData)); + Assert.IsTrue(new List { "a", "b" }.SequenceEqual(properties.EncryptedPaths)); + Assert.AreEqual(CompressionOptions.CompressionAlgorithm.Brotli, properties.CompressionAlgorithm); + Assert.IsTrue(new Dictionary { { "a", 246 } }.SequenceEqual(properties.CompressedEncryptedPaths)); + } +#endif + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs index 529619d11a..90e873ed82 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs @@ -43,15 +43,15 @@ public void GetQuality_ThrowsForUnknownLevels() [DataRow(CompressionLevel.SmallestSize, 4096)] public void CompressAndDecompress_HasSameResult(CompressionLevel compressionLevel, int payloadSize) { - BrotliCompressor compressor = new (); - EncryptionProperties properties = new (0, "", "", null, null, new Dictionary()); + BrotliCompressor compressor = new (compressionLevel); + EncryptionProperties properties = new (0, "", "", null, null, CompressionOptions.CompressionAlgorithm.Brotli, new Dictionary()); string path = "somePath"; byte[] bytes = new byte[payloadSize]; bytes.AsSpan().Fill(127); - ArrayPoolManager manager = new (); - (byte[] compressedBytes, int compressedBytesSize) = compressor.Compress(properties, path, bytes, bytes.Length, manager, BrotliCompressor.GetQualityFromCompressionLevel(compressionLevel)); + byte[] compressedBytes = new byte[BrotliCompressor.GetMaxCompressedSize(payloadSize)]; + int compressedBytesSize = compressor.Compress(properties, path, bytes, bytes.Length, compressedBytes); Assert.AreNotSame(bytes, compressedBytes); Assert.IsTrue(compressedBytesSize > 0); From 96041abec30dcdd41b623f0283c606ad0b086d0c Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 11 Oct 2024 22:11:16 +0200 Subject: [PATCH 49/49] ! fix breaking change --- .../src/CompressionOptions.cs | 4 ---- .../src/Transformation/BrotliCompressor.cs | 7 ++++--- .../MdeEncryptionProcessor.Preview.cs | 20 ++++++++++--------- .../EncryptionPropertiesTests.cs | 4 ++-- .../MdeEncryptionProcessorTests.cs | 11 +++++----- .../Transformation/BrotliCompressionTests.cs | 8 ++++---- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs index 343a984c53..0a59b34c1e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs @@ -33,11 +33,7 @@ public enum CompressionAlgorithm /// /// Gets or sets compression algorithm. /// -#if NET8_0_OR_GREATER - public CompressionAlgorithm Algorithm { get; set; } = CompressionAlgorithm.Brotli; -#else public CompressionAlgorithm Algorithm { get; set; } = CompressionAlgorithm.None; -#endif /// /// Gets or sets compression level. diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs index fc293507d1..20f16efc27 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs @@ -4,10 +4,11 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +#if NET8_0_OR_GREATER using System; using System.Buffers; + using System.Collections.Generic; using System.IO.Compression; internal class BrotliCompressor : IDisposable @@ -44,13 +45,13 @@ public BrotliCompressor(CompressionLevel compressionLevel) this.encoder = new BrotliEncoder(BrotliCompressor.GetQualityFromCompressionLevel(compressionLevel), DefaultWindow); } - public virtual int Compress(EncryptionProperties properties, string path, byte[] bytes, int length, byte[] outputBytes) + public virtual int Compress(Dictionary compressedPaths, string path, byte[] bytes, int length, byte[] outputBytes) { OperationStatus status = this.encoder.Compress(bytes.AsSpan(0, length), outputBytes, out int bytesConsumed, out int bytesWritten, true); ThrowIfFailure(status, length, bytesConsumed); - properties.CompressedEncryptedPaths[path] = length; + compressedPaths[path] = length; return bytesWritten; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 0974125b0f..4786e9f1a4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -35,19 +35,13 @@ public async Task EncryptAsync( DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); - EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: 4, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted, - encryptionOptions.CompressionOptions.Algorithm, - new Dictionary()); + bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None; #if NET8_0_OR_GREATER BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; #endif + Dictionary compressedPaths = new (); foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { @@ -78,7 +72,7 @@ public async Task EncryptAsync( if (compressor != null && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) { byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(processedBytesLength)); - processedBytesLength = compressor.Compress(encryptionProperties, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes); + processedBytesLength = compressor.Compress(compressedPaths, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes); processedBytes = compressedBytes; } #endif @@ -93,6 +87,14 @@ public async Task EncryptAsync( #if NET8_0_OR_GREATER compressor?.Dispose(); #endif + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: compressionEnabled ? 4 : 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted, + encryptionOptions.CompressionOptions.Algorithm, + compressedPaths); itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); #if NET8_0_OR_GREATER diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs index 5ce666f2b5..7d0c2562c2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs @@ -12,7 +12,7 @@ public class EncryptionPropertiesTests [TestMethod] public void Ctor_AssignsAllMandatoryProperties() { - EncryptionProperties properties = new EncryptionProperties( + EncryptionProperties properties = new ( 11, "algorithm", "dek-id", @@ -32,7 +32,7 @@ public void Ctor_AssignsAllMandatoryProperties() [TestMethod] public void Ctor_AssignsAllProperties() { - EncryptionProperties properties = new EncryptionProperties( + EncryptionProperties properties = new ( 11, "algorithm", "dek-id", diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 2314f624d3..410a9fa64e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -230,11 +230,12 @@ private static async Task VerifyEncryptionSucceeded(TestDoc testDoc, En Assert.IsNotNull(encryptionProperties); Assert.AreEqual(dekId, encryptionProperties.DataEncryptionKeyId); -#if ENCRYPTION_CUSTOM_PREVIEW - Assert.AreEqual(4, encryptionProperties.EncryptionFormatVersion); -#else - Assert.AreEqual(3, encryptionProperties.EncryptionFormatVersion); -#endif + + int expectedVersion = + (encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None) + ? 4 : 3; + Assert.AreEqual(expectedVersion, encryptionProperties.EncryptionFormatVersion); + Assert.IsNull(encryptionProperties.EncryptedData); Assert.IsNotNull(encryptionProperties.EncryptedPaths); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs index 90e873ed82..d8f5b611dd 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs @@ -1,6 +1,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +#if NET8_0_OR_GREATER using System; using System.Collections.Generic; using System.IO.Compression; @@ -44,7 +44,7 @@ public void GetQuality_ThrowsForUnknownLevels() public void CompressAndDecompress_HasSameResult(CompressionLevel compressionLevel, int payloadSize) { BrotliCompressor compressor = new (compressionLevel); - EncryptionProperties properties = new (0, "", "", null, null, CompressionOptions.CompressionAlgorithm.Brotli, new Dictionary()); + Dictionary properties = new (); string path = "somePath"; byte[] bytes = new byte[payloadSize]; @@ -59,9 +59,9 @@ public void CompressAndDecompress_HasSameResult(CompressionLevel compressionLeve Console.WriteLine($"Original: {bytes.Length} Compressed: {compressedBytesSize}"); - Assert.IsTrue(properties.CompressedEncryptedPaths.ContainsKey(path)); + Assert.IsTrue(properties.ContainsKey(path)); - int recordedSize = properties.CompressedEncryptedPaths["somePath"]; + int recordedSize = properties["somePath"]; Assert.AreEqual(bytes.Length, recordedSize); byte[] decompressedBytes = new byte[recordedSize];