Skip to content

Commit

Permalink
[Client encryption]: Add JsonNode Encryption processor (#4780)
Browse files Browse the repository at this point in the history 
# Pull Request Template

## Description

- Added support for System.Text.JsonNode DOM on encryption path
- Configuration option to select Json Processor (defaults to Newtonsoft)
- Custom byte[] coverter supporting offset+length for JsonNode
- Perf tests expanded to cover both json processors (JsonNode decryption
still uses Newtonsoft, this will be upgraded with further PRs)

To be processed after #4779 

## Type of change

Please delete options that are not relevant.

- [] New feature (non-breaking change which adds functionality)
- [] This change requires a documentation update

## Closing issues

Contributes to #4678

---------

Co-authored-by: Juraj Blazek <[email protected]>
Co-authored-by: juraj-blazek <[email protected]>
Co-authored-by: Santosh Kulkarni <[email protected]>
  • Loading branch information
@JanHyka @juraj-blazek @kr-santosh
4 people authored Oct 15, 2024
1 parent 8644014 commit 138601f
Show file tree
Hide file tree
Showing 13 changed files with 616 additions and 176 deletions.
25 changes: 25 additions & 0 deletions Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,25 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom
{
using System.Collections.Generic;

/// <summary>
/// API for JSON processing
/// </summary>
public enum JsonProcessor
{
/// <summary>
/// Newtonsoft.Json
/// </summary>
Newtonsoft,

#if NET8_0_OR_GREATER
/// <summary>
/// System.Text.Json
/// </summary>
/// <remarks>Available with .NET8.0 package only.</remarks>
SystemTextJson,
#endif
}

/// <summary>
/// Options for encryption of data.
/// </summary>
Expand Down Expand Up @@ -40,5 +59,11 @@ public sealed class EncryptionOptions
/// Example of a path specification: /sensitive
/// </summary>
public IEnumerable<string> PathsToEncrypt { get; set; }

/// <summary>
/// Gets or sets API used for Json processing
/// </summary>
/// <remarks>Setting only applies with Mde encryption is used.</remarks>
public JsonProcessor JsonProcessor { get; set; } = JsonProcessor.Newtonsoft;
}
}
8 changes: 8 additions & 0 deletions Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,29 +5,37 @@
namespace Microsoft.Azure.Cosmos.Encryption.Custom
{
using System.Collections.Generic;
using System.Text.Json.Serialization;
using Newtonsoft.Json;

internal class EncryptionProperties
{
[JsonProperty(PropertyName = Constants.EncryptionFormatVersion)]
[JsonPropertyName(Constants.EncryptionFormatVersion)]
public int EncryptionFormatVersion { get; }

[JsonProperty(PropertyName = Constants.EncryptionDekId)]
[JsonPropertyName(Constants.EncryptionDekId)]
public string DataEncryptionKeyId { get; }

[JsonProperty(PropertyName = Constants.EncryptionAlgorithm)]
[JsonPropertyName(Constants.EncryptionAlgorithm)]
public string EncryptionAlgorithm { get; }

[JsonProperty(PropertyName = Constants.EncryptedData)]
[JsonPropertyName(Constants.EncryptedData)]
public byte[] EncryptedData { get; }

[JsonProperty(PropertyName = Constants.EncryptedPaths)]
[JsonPropertyName(Constants.EncryptedPaths)]
public IEnumerable<string> EncryptedPaths { get; }

[JsonProperty(PropertyName = Constants.CompressionAlgorithm)]
[JsonPropertyName(Constants.CompressionAlgorithm)]
public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; }

[JsonProperty(PropertyName = Constants.CompressedEncryptedPaths)]
[JsonPropertyName(Constants.CompressedEncryptedPaths)]
public IDictionary<string, int> CompressedEncryptedPaths { get; }

public EncryptionProperties(
Expand Down
170 changes: 10 additions & 160 deletions ...osoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,102 +7,33 @@
namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation
{
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;

internal class MdeEncryptionProcessor
{
internal JObjectSqlSerializer Serializer { get; set; } = new JObjectSqlSerializer();
internal MdeJObjectEncryptionProcessor JObjectEncryptionProcessor { get; set; } = new MdeJObjectEncryptionProcessor();

internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor();
#if NET8_0_OR_GREATER
internal MdeJsonNodeEncryptionProcessor JsonNodeEncryptionProcessor { get; set; } = new MdeJsonNodeEncryptionProcessor();
#endif

public async Task<Stream> EncryptAsync(
Stream input,
Encryptor encryptor,
EncryptionOptions encryptionOptions,
CancellationToken token)
{
JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream<JObject>(input);
List<string> pathsEncrypted = new ();
TypeMarker typeMarker;

using ArrayPoolManager arrayPoolManager = new ();

DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token);

bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None;

#if NET8_0_OR_GREATER
BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli
? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null;
#endif
Dictionary<string, int> compressedPaths = new ();

foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt)
return encryptionOptions.JsonProcessor switch
{
JsonProcessor.Newtonsoft => await this.JObjectEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token),
#if NET8_0_OR_GREATER
string propertyName = pathToEncrypt[1..];
#else
string propertyName = pathToEncrypt.Substring(1);
JsonProcessor.SystemTextJson => await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token),
#endif
if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue))
{
continue;
}

if (propertyValue.Type == JTokenType.Null)
{
continue;
}

byte[] processedBytes = null;
(typeMarker, processedBytes, int processedBytesLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager);

if (processedBytes == null)
{
continue;
}

#if NET8_0_OR_GREATER
if (compressor != null && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength))
{
byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(processedBytesLength));
processedBytesLength = compressor.Compress(compressedPaths, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes);
processedBytes = compressedBytes;
}
#endif

byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength);

itemJObj[propertyName] = encryptedBytes;

pathsEncrypted.Add(pathToEncrypt);
}

#if NET8_0_OR_GREATER
compressor?.Dispose();
#endif
EncryptionProperties encryptionProperties = new (
encryptionFormatVersion: compressionEnabled ? 4 : 3,
encryptionOptions.EncryptionAlgorithm,
encryptionOptions.DataEncryptionKeyId,
encryptedData: null,
pathsEncrypted,
encryptionOptions.CompressionOptions.Algorithm,
compressedPaths);

itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties));
#if NET8_0_OR_GREATER
await input.DisposeAsync();
#else
input.Dispose();
#endif
return EncryptionProcessor.BaseSerializer.ToStream(itemJObj);
_ => throw new InvalidOperationException("Unsupported JsonProcessor")
};
}

internal async Task<DecryptionContext> DecryptObjectAsync(
Expand All @@ -112,88 +43,7 @@ internal async Task<DecryptionContext> DecryptObjectAsync(
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
_ = diagnosticsContext;

if (encryptionProperties.EncryptionFormatVersion != 3 && encryptionProperties.EncryptionFormatVersion != 4)
{
throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version.");
}

using ArrayPoolManager arrayPoolManager = new ();
using ArrayPoolManager<char> charPoolManager = new ();

DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken);

List<string> pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count());

#if NET8_0_OR_GREATER
BrotliCompressor decompressor = null;
if (encryptionProperties.EncryptionFormatVersion == 4)
{
bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true;
if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed)
{
throw new NotSupportedException($"Unknown compression algorithm {encryptionProperties.CompressionAlgorithm}");
}

if (containsCompressed)
{
decompressor = new ();
}
}
#endif

foreach (string path in encryptionProperties.EncryptedPaths)
{
#if NET8_0_OR_GREATER
string propertyName = path[1..];
#else
string propertyName = path.Substring(1);
#endif

if (!document.TryGetValue(propertyName, out JToken propertyValue))
{
// malformed document, such record shouldn't be there at all
continue;
}

byte[] cipherTextWithTypeMarker = propertyValue.ToObject<byte[]>();
if (cipherTextWithTypeMarker == null)
{
continue;
}

(byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager);

#if NET8_0_OR_GREATER
if (decompressor != null)
{
if (encryptionProperties.CompressedEncryptedPaths?.TryGetValue(path, out int decompressedSize) == true)
{
byte[] buffer = arrayPoolManager.Rent(decompressedSize);
processedBytes = decompressor.Decompress(bytes, processedBytes, buffer);

bytes = buffer;
}
}
#endif

this.Serializer.DeserializeAndAddProperty(
(TypeMarker)cipherTextWithTypeMarker[0],
bytes.AsSpan(0, processedBytes),
document,
propertyName,
charPoolManager);

pathsDecrypted.Add(path);
}

DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext(
pathsDecrypted,
encryptionProperties.DataEncryptionKeyId);

document.Remove(Constants.EncryptedInfo);
return decryptionContext;
return await this.JObjectEncryptionProcessor.DecryptObjectAsync(document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken);
}
}
}
Expand Down
23 changes: 23 additions & 0 deletions Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,29 @@ internal virtual byte[] Encrypt(DataEncryptionKey encryptionKey, TypeMarker type
return encryptedText;
}

internal virtual (byte[], int) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager)
{
int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1;

byte[] encryptedText = arrayPoolManager.Rent(encryptedTextLength);

encryptedText[0] = (byte)typeMarker;

int encryptedLength = encryptionKey.EncryptData(
plainText,
plainTextOffset: 0,
plainTextLength,
encryptedText,
outputOffset: 1);

if (encryptedLength < 0)
{
throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null cipherText from {nameof(DataEncryptionKey.EncryptData)}.");
}

return (encryptedText, encryptedTextLength);
}

internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager)
{
int plainTextLength = encryptionKey.GetDecryptByteCount(cipherText.Length - 1);
Expand Down
Loading

0 comments on commit 138601f

Please sign in to comment.