diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml index e423fcddfc81..c9fe1c177033 100644 --- a/eng/.docsettings.yml +++ b/eng/.docsettings.yml @@ -124,6 +124,7 @@ known_content_issues: - ['sdk/cosmos/azure-cosmos-benchmark/README.md', '#3113'] - ['sdk/cosmos/azure-cosmos-examples/README.md', '#3113'] - ['sdk/cosmos/azure-cosmos/README.md', '#3113'] + - ['sdk/cosmos/azure-cosmos-encryption/README.md', '#3113'] - ['sdk/e2e/README.md', '#3113'] - ['sdk/eventgrid/microsoft-azure-eventgrid/README.md', '#3113'] - ['sdk/eventhubs/microsoft-azure-eventhubs-eph/README.md', '#3113'] diff --git a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml index e5bbe9ac1a68..d812ae0d8f34 100755 --- a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml +++ b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml @@ -430,6 +430,7 @@ + + + + + + + + + + + + + + + + + + + + @@ -1452,6 +1471,12 @@ + + + + + + @@ -1647,6 +1672,28 @@ + + + + + + + + + + + + + + + + + + + + + + @@ -1654,6 +1701,27 @@ + + + + + + + + + + + + + + + + + + + + + @@ -1817,6 +1885,12 @@ + + + + + + diff --git a/eng/jacoco-test-coverage/pom.xml b/eng/jacoco-test-coverage/pom.xml index 27a675955041..c6f7f000e22a 100644 --- a/eng/jacoco-test-coverage/pom.xml +++ b/eng/jacoco-test-coverage/pom.xml @@ -111,6 +111,11 @@ azure-cosmos 4.3.0-beta.1 + + com.azure + azure-cosmos-encryption + 1.0.0-beta.1 + com.azure azure-data-appconfiguration diff --git a/eng/versioning/version_client.txt b/eng/versioning/version_client.txt index e1cdaf5e8af4..182724cbf40a 100644 --- a/eng/versioning/version_client.txt +++ b/eng/versioning/version_client.txt @@ -21,6 +21,7 @@ com.azure:azure-core-test;1.3.1;1.4.0-beta.1 com.azure:azure-core-tracing-opentelemetry;1.0.0-beta.5;1.0.0-beta.6 com.azure:azure-cosmos;4.2.0;4.3.0-beta.1 com.azure:azure-cosmos-benchmark;4.0.1-beta.1;4.0.1-beta.1 +com.azure:azure-cosmos-encryption;1.0.0-beta.1;1.0.0-beta.1 com.azure:azure-data-appconfiguration;1.1.3;1.2.0-beta.1 com.azure:azure-data-schemaregistry;1.0.0-beta.2;1.0.0-beta.3 com.azure:azure-data-schemaregistry-avro;1.0.0-beta.2;1.0.0-beta.3 diff --git a/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md b/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md new file mode 100644 index 000000000000..6d9dbe848599 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/CHANGELOG.md @@ -0,0 +1,6 @@ +# Release History + +# Release History +## 1.0.0-beta.1 (Unreleased) + + diff --git a/sdk/cosmos/azure-cosmos-encryption/README.md b/sdk/cosmos/azure-cosmos-encryption/README.md new file mode 100644 index 000000000000..341592e2250d --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/README.md @@ -0,0 +1,69 @@ +# Azure CosmosDB client library for Java +TODO + +## Getting started +### Include the package + +[//]: # ({x-version-update-start;com.azure:azure-cosmos-encryption;current}) +```xml + + com.azure + azure-cosmos-encryption + 1.0.0-beta.1 + +``` +[//]: # ({x-version-update-end}) + + +### Prerequisites +TODO + +## Key concepts +TODO + +## Examples +TODO + +## Troubleshooting +TODO + +## Next steps +TODO + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +[Contributor License Agreement (CLA)][cla] declaring that you have the right to, and actually do, grant us the rights +to use your contribution. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate +the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to +do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct][coc]. For more information see the [Code of Conduct FAQ][coc_faq] +or contact [opencode@microsoft.com][coc_contact] with any additional questions or comments. + + +[source_code]: src +[cosmos_introduction]: https://docs.microsoft.com/en-us/azure/cosmos-db/ +[api_documentation]: https://azuresdkdocs.blob.core.windows.net/$web/java/azure-cosmos/latest/index.html +[cosmos_docs]: https://docs.microsoft.com/en-us/azure/cosmos-db/introduction +[jdk]: https://docs.microsoft.com/java/azure/java-supported-jdk-runtime?view=azure-java-stable +[maven]: https://maven.apache.org/ +[cosmos_maven]: https://search.maven.org/artifact/com.azure/azure-cosmos +[cosmos_maven_svg]: https://img.shields.io/maven-central/v/com.azure/azure-cosmos.svg +[cla]: https://cla.microsoft.com +[coc]: https://opensource.microsoft.com/codeofconduct/ +[coc_faq]: https://opensource.microsoft.com/codeofconduct/faq/ +[coc_contact]: mailto:opencode@microsoft.com +[azure_subscription]: https://azure.microsoft.com/free/ +[samples]: https://github.com/Azure-Samples/azure-cosmos-java-sql-api-samples +[samples_readme]: https://github.com/Azure-Samples/azure-cosmos-java-sql-api-samples/blob/master/README.md +[troubleshooting]: https://docs.microsoft.com/en-us/azure/cosmos-db/troubleshoot-java-sdk-v4-sql +[perf_guide]: https://docs.microsoft.com/en-us/azure/cosmos-db/performance-tips-java-sdk-v4-sql?tabs=api-async +[sql_api_query]: https://docs.microsoft.com/en-us/azure/cosmos-db/sql-api-sql-query +[getting_started]: https://github.com/Azure-Samples/azure-cosmos-java-getting-started +[quickstart]: https://docs.microsoft.com/en-us/azure/cosmos-db/create-sql-api-java?tabs=sync +[project_reactor_schedulers]: https://projectreactor.io/docs/core/release/api/reactor/core/scheduler/Schedulers.html + +![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-java%2Fsdk%2Fcosmos%2FREADME.png) diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml new file mode 100644 index 000000000000..0467daae1538 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -0,0 +1,427 @@ + + + 4.0.0 + + com.azure + azure-client-sdk-parent + 1.7.0 + ../../parents/azure-client-sdk-parent + + + com.azure + azure-cosmos-encryption + 1.0.0-beta.1 + Microsoft Azure SDK for SQL API of Azure Cosmos DB Service + This Package contains Microsoft Azure Cosmos SDK (with Reactive Extension Reactor support) for Azure Cosmos DB SQL API + jar + https://github.com/Azure/azure-sdk-for-java + + + + azure-java-build-docs + ${site.url}/site/${project.artifactId} + + + + + scm:git:https://github.com/Azure/azure-sdk-for-java + scm:git:git@github.com:Azure/azure-sdk-for-java.git + HEAD + + + + + 0.10 + 0.10 + + + + + + com.azure + azure-cosmos + 4.3.0-beta.1 + + + + + com.google.code.findbugs + jsr305 + 3.0.2 + provided + + + + org.apache.commons + commons-collections4 + test + 4.2 + + + + org.apache.commons + commons-text + test + 1.6 + + + + org.testng + testng + 6.14.3 + test + + + + org.assertj + assertj-core + 3.11.1 + test + + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.13.0 + test + + + + org.apache.logging.log4j + log4j-api + 2.11.1 + test + + + + org.apache.logging.log4j + log4j-core + 2.11.1 + test + + + + com.google.guava + guava + 25.0-jre + test + + + + io.projectreactor + reactor-test + 3.3.5.RELEASE + test + + + + io.reactivex.rxjava2 + rxjava + 2.2.4 + test + + + + org.mockito + mockito-core + 1.10.19 + test + + + + org.bouncycastle + bcprov-jdk15on + 1.60 + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 1.8 + 1.8 + false + + + + + + + + org.revapi + revapi-maven-plugin + 0.11.2 + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M3 + + unit + + %regex[.*] + + + + surefire.testng.verbose + 2 + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.0.0-M3 + + + + + com.azure:* + com.fasterxml.jackson.core:jackson-core:[2.10.1] + com.fasterxml.jackson.core:jackson-annotations:[2.10.1] + com.fasterxml.jackson.core:jackson-databind:[2.10.1] + com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.10.1] + com.fasterxml.jackson.module:jackson-module-afterburner:[2.10.1] + com.google.code.findbugs:jsr305:[3.0.2] + io.dropwizard.metrics:metrics-core:[4.1.0] + io.micrometer:micrometer-core:[1.2.0] + io.netty:netty-codec-http:[4.1.49.Final] + io.netty:netty-codec-http2:[4.1.49.Final] + io.netty:netty-handler:[4.1.49.Final] + io.netty:netty-handler-proxy:[4.1.49.Final] + io.netty:netty-transport-native-epoll:[4.1.49.Final] + io.projectreactor:reactor-core:[3.3.5.RELEASE] + io.projectreactor.netty:reactor-netty:[0.9.7.RELEASE] + org.slf4j:slf4j-api:[1.7.28] + org.slf4j:slf4j-api:[1.7.28] + org.bouncycastle:bcprov-jdk15on:[1.60] + + + + + + + + + + + + unit + + default + unit + + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M3 + + + + + + + + + fast + + simple,cosmosv3 + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.0 + + + src/test/resources/fast-testng.xml + + + + + + + + + long + + long + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.0 + + + src/test/resources/long-testng.xml + + + + + + + + + direct + + direct + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.0 + + + src/test/resources/direct-testng.xml + + + + + + + + + multi-master + + multi-master + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.0 + + + src/test/resources/multi-master-testng.xml + + + + + + + + + examples + + + samples,examples + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.0 + + + src/test/resources/examples-testng.xml + + + + + + integration-test + verify + + + + + + + + + + emulator + + emulator + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.0 + + + src/test/resources/emulator-testng.xml + + + + + + + + + non-emulator + + non-emulator + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.0 + + + src/test/resources/non-emulator-testng.xml + + + + + + + + + e2e + + e2e + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.22.0 + + + src/test/resources/e2e-testng.xml + + + + + + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256Algorithm.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256Algorithm.java new file mode 100644 index 000000000000..5ee1be227aa1 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256Algorithm.java @@ -0,0 +1,381 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.encryption.api.CosmosEncryptionAlgorithm; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey; +import com.azure.cosmos.implementation.encryption.api.EncryptionType; + + +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * This class implements authenticated encryption algorithm with associated data as described in + * http://tools.ietf.org/html/draft-mcgrew-aead-aes-cbc-hmac-sha2-05 - specifically this implements + * AEAD_AES_256_CBC_HMAC_SHA256 algorithm. + * This (and AeadAes256CbcHmac256EncryptionKey) implementation for Cosmos DB is same as the existing + * SQL client implementation with StyleCop related changes - also, we restrict to randomized encryption to start with. + */ +class AeadAes256CbcHmac256Algorithm implements DataEncryptionKey { + + public final static String ALGORITHM_NAME = "AEAD_AES_256_CBC_HMAC_SHA256"; + + /** + * Key size in bytes + */ + private static final int KEY_SIZE_IN_BYTES = AeadAes256CbcHmac256EncryptionKey.KEY_SIZE / 8; + + /** + * Block size in bytes. AES uses 16 byte blocks. + */ + private static final int BLOCK_SIZE_IN_BYTES = 16; + + /** + * Minimum Length of cipherText without authentication tag. This value is 1 (version byte) + 16 (IV) + 16 (minimum of 1 block of cipher Text) + */ + private static final int MINIMUM_CIPHER_TEXT_LENGTH_IN_BYTES_NO_AUTHENTICATION_TAG = Bytes.ONE_BYTE_SIZE + BLOCK_SIZE_IN_BYTES + BLOCK_SIZE_IN_BYTES; + + /** + * Minimum Length of cipherText. This value is 1 (version byte) + 32 (authentication tag) + 16 (IV) + 16 (minimum of 1 block of cipher Text) + */ + private static final int MINIMUM_CIPHER_TEXT_LENGTH_IN_BYTES_WITH_AUTHENTICATION_TAG = MINIMUM_CIPHER_TEXT_LENGTH_IN_BYTES_NO_AUTHENTICATION_TAG + KEY_SIZE_IN_BYTES; + + /** + * Cipher Mode. For this algorithm, we only use CBC mode. + */ + private static final AesCryptoServiceProvider.CipherMode CIPHER_MODE = AesCryptoServiceProvider.CipherMode.CBC; + + /** + * Padding mode. This algorithm uses PKCS7. // TODO: + */ + private static final AesCryptoServiceProvider.PaddingMode PADDING_MODE = AesCryptoServiceProvider.PaddingMode.PKCS7; + + /** + * Byte array with algorithm version used for authentication tag computation. + */ + private static final byte[] VERSION = new byte[]{0x01}; + + /** + * Byte array with algorithm version size used for authentication tag computation. + */ + private static final byte[] VERSION_SIZE = new byte[]{Bytes.ONE_BYTE_SIZE}; + + /** + * Variable indicating whether this algorithm should work in Deterministic mode or Randomized mode. + * For deterministic encryption, we derive an IV from the plaintext data. + * For randomized encryption, we generate a cryptographically random IV. + */ + private final boolean isDeterministic; + + /** + * Algorithm Version. + */ + private final byte algorithmVersion; + + /** + * Data Encryption Key. This has a root key and three derived keys. + */ + private final AeadAes256CbcHmac256EncryptionKey dataEncryptionKey; + + /** + * The pool of crypto providers to use for encrypt/decrypt operations. + */ + private final ConcurrentLinkedQueue cryptoProviderPool; + + @Override + public byte[] getRawKey() { + return this.dataEncryptionKey.getRootKey(); + } + + + @Override + public String getEncryptionAlgorithm() { + return CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized; + } + + /** + * Initializes a new instance of AeadAes256CbcHmac256Algorithm algorithm with a given key and encryption type + * + * @param encryptionKey Root encryption key from which three other keys will be derived + * @param encryptionType Encryption Type, accepted values are Deterministic and Randomized. + * For Deterministic encryption, a synthetic IV will be genenrated during encryption + * For Randomized encryption, a random IV will be generated during encryption. + * @param algorithmVersion Algorithm version + */ + public AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256EncryptionKey encryptionKey, EncryptionType encryptionType, byte algorithmVersion) { + this.dataEncryptionKey = encryptionKey; + this.algorithmVersion = algorithmVersion; + + VERSION[0] = algorithmVersion; + + assert encryptionKey != null : "Null encryption key detected in AeadAes256CbcHmac256 algorithm"; + assert algorithmVersion == 0x01 : "Unknown algorithm version passed to AeadAes256CbcHmac256"; + + // Validate encryption type for this algorithm + // This algorithm can only provide randomized or deterministic encryption types. + // Right now, we support only randomized encryption for Cosmos DB client side encryption. + assert encryptionType == EncryptionType.RANDOMIZED : "Invalid Encryption Type detected in AeadAes256CbcHmac256Algorithm"; + this.isDeterministic = false; + + this.cryptoProviderPool = new ConcurrentLinkedQueue<>(); + } + + /** + * Encryption Algorithm + *

+ * cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits + * cell_ciphertext = AES-CBC-256(enc_key, cell_iv, cell_data) with PKCS7 padding. + * cell_tag = HMAC_SHA-2-256(mac_key, versionbyte + cell_iv + cell_ciphertext + versionbyte_length) + * cell_blob = versionbyte + cell_tag + cell_iv + cell_ciphertext + * + * @param plainText Plain text value to be encrypted. + * @return Returns the ciphertext corresponding to the plaintext. + */ + @Override + public byte[] encryptData(byte[] plainText) { + return this.encryptData(plainText, true); + } + + /** + * Encryption Algorithm + *

+ * cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits + * cell_ciphertext = AES-CBC-256(enc_key, cell_iv, cell_data) with PKCS7 padding. + * (optional) cell_tag = HMAC_SHA-2-256(mac_key, versionbyte + cell_iv + cell_ciphertext + versionbyte_length) + * cell_blob = versionbyte + [cell_tag] + cell_iv + cell_ciphertext + * + * @param plainText Plaintext data to be encrypted + * @param hasAuthenticationTag Does the algorithm require authentication tag. + * @return Returns the ciphertext corresponding to the plaintext. + */ + private byte[] encryptData(byte[] plainText, boolean hasAuthenticationTag) { + // Empty values get encrypted and decrypted properly for both Deterministic and Randomized encryptions. + assert (plainText != null); + + byte[] iv = new byte[BLOCK_SIZE_IN_BYTES]; + + // Prepare IV + // Should be 1 single block (16 bytes) + if (this.isDeterministic) { + SecurityUtility.getHMACWithSHA256(plainText, this.dataEncryptionKey.getIVKey(), iv); + } else { + SecurityUtility.generateRandomBytes(iv); + } + + int numBlocks = (plainText.length / BLOCK_SIZE_IN_BYTES) + 1; + + // Final blob we return = version + HMAC + iv + cipherText + final int hmacStartIndex = 1; + int authenticationTagLen = hasAuthenticationTag ? KEY_SIZE_IN_BYTES : 0; + int ivStartIndex = hmacStartIndex + authenticationTagLen; + int cipherStartIndex = ivStartIndex + BLOCK_SIZE_IN_BYTES; // this is where hmac starts. + + // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. + int outputBufSize = Bytes.ONE_BYTE_SIZE + authenticationTagLen + iv.length + (numBlocks * BLOCK_SIZE_IN_BYTES); + byte[] outBuffer = new byte[outputBufSize]; + + // Store the version and IV rightaway + outBuffer[0] = this.algorithmVersion; + System.arraycopy(iv, 0, outBuffer, ivStartIndex, iv.length); + + AesCryptoServiceProvider aesAlg = this.cryptoProviderPool.poll(); + + // Try to get a provider from the pool. + // If no provider is available, create a new one. + if (aesAlg == null) { + aesAlg = new AesCryptoServiceProvider(this.dataEncryptionKey.getEncryptionKey(), PADDING_MODE, CIPHER_MODE); + } + + try { + // Always set the IV since it changes from cell to cell. + aesAlg.setIv(iv); + + // Compute CipherText and authentication tag in a single pass + try (AesCryptoServiceProvider.ICryptoTransform encryptor = aesAlg.createEncryptor()) { + // TODO: assert encryptor.CanTransformMultipleBlocks : "AES Encryptor can transform multiple blocks"; + int count = 0; + int cipherIndex = cipherStartIndex; // this is where cipherText starts + if (numBlocks > 1) { + count = (numBlocks - 1) * BLOCK_SIZE_IN_BYTES; + cipherIndex += encryptor.transformBlock(plainText, 0, count, outBuffer, cipherIndex); + } + + byte[] buffTmp = encryptor.transformFinalBlock(plainText, count, plainText.length - count); // done encrypting + System.arraycopy(buffTmp, 0, outBuffer, cipherIndex, buffTmp.length); + cipherIndex += buffTmp.length; + } + + if (hasAuthenticationTag) { + try (HMACSHA256 hmac = new HMACSHA256(this.dataEncryptionKey.getMACKey())) { + // TODO: always true assert(hmac.CanTransformMultipleBlocks, "HMAC can't transform multiple blocks"); + hmac.transformBlock(VERSION, 0, VERSION.length, VERSION, 0); + hmac.transformBlock(iv, 0, iv.length, iv, 0); + + // Compute HMAC on final block + hmac.transformBlock(outBuffer, cipherStartIndex, numBlocks * BLOCK_SIZE_IN_BYTES, outBuffer, cipherStartIndex); + hmac.transformFinalBlock(VERSION_SIZE, 0, VERSION_SIZE.length); + byte[] hash = hmac.getHash(); + assert hash.length >= authenticationTagLen : "Unexpected hash size"; + System.arraycopy(hash, 0, outBuffer, hmacStartIndex, authenticationTagLen); + } + } + } finally { + // Return the provider to the pool. + this.cryptoProviderPool.add(aesAlg); + } + + return outBuffer; + } + + /** + * Decryption steps + * 1. Validate version byte + * 2. Validate Authentication tag + * 3. Decrypt the message + * + * @param cipherText Ciphertext value to be decrypted. + * @return + */ + @Override + public byte[] decryptData(byte[] cipherText) { + return this.decryptData(cipherText, /** hasAuthenticationTag */true); + } + + /** + * Decryption steps + * 1. Validate version byte + * 2. (optional) Validate Authentication tag + * 3. Decrypt the message + * + * @param cipherText + * @param hasAuthenticationTag + * @return + */ + private byte[] decryptData(byte[] cipherText, boolean hasAuthenticationTag) { + assert cipherText != null; + + byte[] iv = new byte[BLOCK_SIZE_IN_BYTES]; + + int minimumCipherTextLength = hasAuthenticationTag ? MINIMUM_CIPHER_TEXT_LENGTH_IN_BYTES_WITH_AUTHENTICATION_TAG : MINIMUM_CIPHER_TEXT_LENGTH_IN_BYTES_NO_AUTHENTICATION_TAG; + if (cipherText.length < minimumCipherTextLength) { + throw EncryptionExceptionFactory.invalidCipherTextSize(cipherText.length, minimumCipherTextLength); + } + + // Validate the version byte + int startIndex = 0; + if (cipherText[startIndex] != this.algorithmVersion) { + // Cipher text was computed with a different algorithm version than this. + throw EncryptionExceptionFactory.invalidAlgorithmVersion(cipherText[startIndex], this.algorithmVersion); + } + + startIndex += 1; + int authenticationTagOffset = 0; + + // Read authentication tag + if (hasAuthenticationTag) { + authenticationTagOffset = startIndex; + startIndex += KEY_SIZE_IN_BYTES; // authentication tag size is KeySizeInBytes + } + + // Read cell IV + System.arraycopy(cipherText, startIndex, iv, 0, iv.length); + startIndex += iv.length; + + // Read encrypted text + int cipherTextOffset = startIndex; + int cipherTextCount = cipherText.length - startIndex; + + if (hasAuthenticationTag) { + // Compute authentication tag + byte[] authenticationTag = this.prepareAuthenticationTag(iv, cipherText, cipherTextOffset, cipherTextCount); + if (!SecurityUtility.compareBytes(authenticationTag, cipherText, authenticationTagOffset, authenticationTag.length)) { + // Potentially tampered data, throw an exception + throw EncryptionExceptionFactory.invalidAuthenticationTag(); + } + } + + // Decrypt the text and return + return this.decryptData(iv, cipherText, cipherTextOffset, cipherTextCount); + } + + /** + * Decrypts plain text data using AES in CBC mode + * + * @param iv + * @param cipherText + * @param offset + * @param count + * @return + */ + private byte[] decryptData(byte[] iv, byte[] cipherText, int offset, int count) { + assert ((iv != null) && (cipherText != null)); + assert (offset > -1 && count > -1); + assert ((count + offset) <= cipherText.length); + + byte[] plainText; + + AesCryptoServiceProvider aesAlg = this.cryptoProviderPool.poll(); + + // Try to get a provider from the pool. + // If no provider is available, create a new one. + if (aesAlg == null) { + aesAlg = new AesCryptoServiceProvider(this.dataEncryptionKey.getEncryptionKey(), PADDING_MODE, CIPHER_MODE); + + } + + try { + // Always set the IV since it changes from cell to cell. + aesAlg.setIv(iv); + + // Create the streams used for decryption. + + try (AesCryptoServiceProvider.ICryptoTransform decryptor = aesAlg.createDecryptor()) { + plainText = decryptor.transformFinalBlock(cipherText, offset, count); + } + } finally { + // Return the provider to the pool. + this.cryptoProviderPool.add(aesAlg); + } + + return plainText; + } + + /** + * Prepares an authentication tag. + * Authentication Tag = HMAC_SHA-2-256(mac_key, versionbyte + cell_iv + cell_ciphertext + versionbyte_length) + * + * @param iv + * @param cipherText + * @param offset + * @param length + * @return + */ + private byte[] prepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset, int length) { + assert (cipherText != null); + + byte[] computedHash; + byte[] authenticationTag = new byte[KEY_SIZE_IN_BYTES]; + + // Raw Tag Length: + // 1 for the version byte + // 1 block for IV (16 bytes) + // cipherText.Length + // 1 byte for version byte length + try (HMACSHA256 hmac = new HMACSHA256(this.dataEncryptionKey.getMACKey())) { + int retVal = 0; + retVal = hmac.transformBlock(VERSION, 0, VERSION.length, VERSION, 0); + assert (retVal == VERSION.length); + retVal = hmac.transformBlock(iv, 0, iv.length, iv, 0); + assert (retVal == iv.length); + retVal = hmac.transformBlock(cipherText, offset, length, cipherText, offset); + assert (retVal == length); + hmac.transformFinalBlock(VERSION_SIZE, 0, VERSION_SIZE.length); + computedHash = hmac.getHash(); + } + + assert (computedHash.length >= authenticationTag.length); + System.arraycopy(computedHash, 0, authenticationTag, 0, authenticationTag.length); + return authenticationTag; + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256AlgorithmProvider.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256AlgorithmProvider.java new file mode 100644 index 000000000000..eafa53cbd54b --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256AlgorithmProvider.java @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.encryption.api.EncryptionType; + +public class AeadAes256CbcHmac256AlgorithmProvider { + public static void generateRandomBytes(byte[] randomBytes) { + SecurityUtility.generateRandomBytes(randomBytes); + } + + public static AeadAes256CbcHmac256Algorithm createAlgorithm(byte[] encryptionKey, EncryptionType encryptionType, byte algorithmVersion) { + AeadAes256CbcHmac256EncryptionKey dataEncryptionKey = new AeadAes256CbcHmac256EncryptionKey(encryptionKey, AeadAes256CbcHmac256Algorithm.ALGORITHM_NAME); + return new AeadAes256CbcHmac256Algorithm(dataEncryptionKey, encryptionType, algorithmVersion); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256EncryptionKey.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256EncryptionKey.java new file mode 100644 index 000000000000..a21c42ea050b --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256EncryptionKey.java @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.guava27.Strings; + + +/** + * Encryption key class containing 4 keys. This class is used by AeadAes256CbcHmac256Algorithm + * 1) root key - Main key that is used to derive the keys used in the encryption algorithm + * 2) encryption key - A derived key that is used to encrypt the plain text and generate cipher text + * 3) mac_key - A derived key that is used to compute HMAC of the cipher text + * 4) iv_key - A derived key that is used to generate a synthetic IV from plain text data. + */ +class AeadAes256CbcHmac256EncryptionKey extends SymmetricKey { + + /** + * Key size in bits + */ + static final int KEY_SIZE = 256; + + /** + * Encryption Key Salt format. This is used to derive the encryption key from the root key. + */ + private static final String ENCRYPTION_KEY_SALT_FORMAT = "Microsoft Azure Cosmos DB encryption key with encryption algorithm:%s and key length:%s"; + + /** + * MAC Key Salt format. This is used to derive the MAC key from the root key. + */ + private static final String MAC_KEY_SALT_FORMAT = "Microsoft Azure Cosmos DB MAC key with encryption algorithm:%s and key length:%s"; + + /** + * IV Key Salt format. This is used to derive the IV key from the root key. This is only used for Deterministic encryption. + */ + private static final String IV_KEY_SALT_FORMAT = "Microsoft Azure Cosmos DB IV key with encryption algorithm:%s and key length:%s"; + + /** + * Encryption Key + */ + private final SymmetricKey encryptionKey; + + /** + * MAC key + */ + private final SymmetricKey macKey; + + /** + * IV Key + */ + private final SymmetricKey ivKey; + + /** + * The name of the algorithm this key will be used with. + */ + private final String algorithmName; + + /** + * Derives all the required keys from the given root key + * + * @param rootKey + * @param algorithmName + */ + public AeadAes256CbcHmac256EncryptionKey(byte[] rootKey, String algorithmName) { + super(rootKey); + this.algorithmName = algorithmName; + + int keySizeInBytes = KEY_SIZE / 8; + + // Key validation + if (rootKey.length != keySizeInBytes) { + throw EncryptionExceptionFactory.invalidKeySize( + this.algorithmName, + rootKey.length, + keySizeInBytes); + } + + // Derive keys from the root key + // + // Derive encryption key + String encryptionKeySalt = Strings.lenientFormat(ENCRYPTION_KEY_SALT_FORMAT, + this.algorithmName, + KEY_SIZE); + byte[] buff1 = new byte[keySizeInBytes]; + SecurityUtility.getHMACWithSHA256(Utils.getUtf16Bytes(encryptionKeySalt), this.getRootKey(), buff1); + this.encryptionKey = new SymmetricKey(buff1); + + // Derive mac key + String macKeySalt = Strings.lenientFormat(MAC_KEY_SALT_FORMAT, this.algorithmName, KEY_SIZE); + byte[] buff2 = new byte[keySizeInBytes]; + SecurityUtility.getHMACWithSHA256(Utils.getUtf16Bytes(macKeySalt), this.getRootKey(), buff2); + this.macKey = new SymmetricKey(buff2); + + // Derive iv key + String ivKeySalt = Strings.lenientFormat(IV_KEY_SALT_FORMAT, this.algorithmName, KEY_SIZE); + byte[] buff3 = new byte[keySizeInBytes]; + SecurityUtility.getHMACWithSHA256(Utils.getUtf16Bytes(ivKeySalt), this.getRootKey(), buff3); + this.ivKey = new SymmetricKey(buff3); + } + + /** + * Gets Encryption key should be used for encryption and decryption + * + * @return encryption key + */ + byte[] getEncryptionKey() { + return this.encryptionKey.getRootKey(); + } + + /** + * Gets MAC key should be used to compute and validate HMAC + * + * @return mac key + */ + byte[] getMACKey() { + return this.macKey.getRootKey(); + } + + /** + * Gets IV key should be used to compute synthetic IV from a given plain text + * + * @return IV key + */ + byte[] getIVKey() { + return this.ivKey.getRootKey(); + } + +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AesCryptoServiceProvider.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AesCryptoServiceProvider.java new file mode 100644 index 000000000000..59479149aa7b --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AesCryptoServiceProvider.java @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.ShortBufferException; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.Closeable; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +class AesCryptoServiceProvider { + private static final String ALGO_NAME = "AES"; + + private final Cipher cipher; + private final SecretKeySpec secretKeySpec; + private IvParameterSpec ivspec; + + enum PaddingMode { + PKCS5("PKCS5Padding"), + PKCS7("PKCS7Padding"); + + String value; + + PaddingMode(String value) { + this.value = value; + } + } + + enum CipherMode { + CBC("CBC"); + + String value; + + CipherMode(String value) { + this.value = value; + } + } + + public AesCryptoServiceProvider(byte[] key, PaddingMode padding, CipherMode mode) { + try { + secretKeySpec = new SecretKeySpec(key, ALGO_NAME); + + cipher = Cipher.getInstance(String.format("%s/%s/%s", ALGO_NAME, mode.value, padding.value)); + } catch (NoSuchAlgorithmException | NoSuchPaddingException e) { + throw new IllegalStateException(e); + } + } + + public static class ICryptoTransform implements Closeable { + private final Cipher cipher; + + public ICryptoTransform(Cipher cipher) { + this.cipher = cipher; + } + + @Override + public void close() { + + } + + public int transformBlock( + byte[] inputBuffer, + int inputOffset, + int inputCount, + byte[] outputBuffer, + int outputOffset) { + + try { + return cipher.update(inputBuffer, inputOffset, inputCount, outputBuffer, outputOffset); + } catch (ShortBufferException e) { + throw new IllegalStateException(e); + } + } + + public byte[] transformFinalBlock(byte[] inputBuffer, int inputOffset, int inputCount) { + try { + return cipher.doFinal(inputBuffer, inputOffset, inputCount); + } catch (IllegalBlockSizeException | BadPaddingException e) { + throw new IllegalStateException(e); + } + } + } + + public ICryptoTransform createDecryptor() { + try { + cipher.init(Cipher.DECRYPT_MODE, secretKeySpec, ivspec); + return new ICryptoTransform(cipher); + } catch (InvalidKeyException | InvalidAlgorithmParameterException e) { + throw new IllegalStateException(e); + } + } + + public ICryptoTransform createEncryptor() { + try { + cipher.init(Cipher.ENCRYPT_MODE, secretKeySpec, ivspec); + return new ICryptoTransform(cipher); + } catch (InvalidKeyException | InvalidAlgorithmParameterException e) { + throw new IllegalStateException(e); + } + } + + public void setIv(byte[] iv) { + ivspec = new IvParameterSpec(iv); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/Bytes.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/Bytes.java new file mode 100644 index 000000000000..dab3efadb8e6 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/Bytes.java @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +class Bytes { + + public static final int ONE_BYTE_SIZE = 1; + + public static String toHex(byte[] input) { + StringBuilder str = new StringBuilder(); + for (byte b : input) { + str.append(toHex(b)); + } + return str.toString(); + } + + public static String toHex(byte b) { + return String.format("%02X", b); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/CachedDekProperties.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/CachedDekProperties.java new file mode 100644 index 000000000000..9a7446210715 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/CachedDekProperties.java @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import java.time.Instant; + +class CachedDekProperties { + private final DataEncryptionKeyProperties ServerProperties ; + private final Instant ServerPropertiesExpiryUtc; + + public CachedDekProperties( + DataEncryptionKeyProperties serverProperties, + Instant serverPropertiesExpiryUtc) { + assert(serverProperties != null); + + this.ServerProperties = serverProperties; + this.ServerPropertiesExpiryUtc = serverPropertiesExpiryUtc; + } + + public DataEncryptionKeyProperties getServerProperties() { + return ServerProperties; + } + public Instant getServerPropertiesExpiryUtc() { + return ServerPropertiesExpiryUtc; + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/CosmosDataEncryptionKeyProvider.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/CosmosDataEncryptionKeyProvider.java new file mode 100644 index 000000000000..d37b34d5593d --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/CosmosDataEncryptionKeyProvider.java @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.CosmosAsyncContainer; +import com.azure.cosmos.CosmosAsyncDatabase; +import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider; +import com.azure.cosmos.implementation.guava25.base.Preconditions; +import com.azure.cosmos.models.CosmosContainerResponse; +import reactor.core.publisher.Mono; +import reactor.util.function.Tuple2; + +import java.time.Duration; +import java.util.List; + +public class CosmosDataEncryptionKeyProvider implements DataEncryptionKeyProvider { + // TODO: proper sample and documentation on container + private static final String ContainerPartitionKeyPath = "/id"; + private final DataEncryptionKeyContainerCore dataEncryptionKeyContainerCore; + private final DekCache DekCache; + private final EncryptionKeyWrapProvider EncryptionKeyWrapProvider; + private CosmosAsyncContainer container; + + public CosmosDataEncryptionKeyProvider(EncryptionKeyWrapProvider encryptionKeyWrapProvider) { + this(encryptionKeyWrapProvider, null); + } + + public CosmosDataEncryptionKeyProvider(EncryptionKeyWrapProvider encryptionKeyWrapProvider, + Duration dekPropertiesTimeToLive) { + this.EncryptionKeyWrapProvider = encryptionKeyWrapProvider; + this.dataEncryptionKeyContainerCore = new DataEncryptionKeyContainerCore(this); + this.DekCache = new DekCache(dekPropertiesTimeToLive); + } + + CosmosAsyncContainer getContainer() { + if (this.container != null) { + return this.container; + } + + throw new IllegalStateException("The CosmosDataEncryptionKeyProvider was not initialized."); + } + + EncryptionKeyWrapProvider getEncryptionKeyWrapProvider() { + return EncryptionKeyWrapProvider; + } + + DataEncryptionKeyContainer getDataEncryptionKeyContainer() { + return dataEncryptionKeyContainerCore; + } + + DekCache getDekCache() { + return DekCache; + } + + // TODO: @moderakh look into if this method needs to be async. + void initialize(CosmosAsyncDatabase database, + String containerId) { + Preconditions.checkNotNull(database, "database"); + Preconditions.checkNotNull(containerId, "containerId"); + + if (this.container != null) { + throw new IllegalStateException("CosmosDataEncryptionKeyProvider has already been initialized."); + } + + CosmosContainerResponse containerResponse = database.createContainerIfNotExists(containerId, CosmosDataEncryptionKeyProvider.ContainerPartitionKeyPath).block(); + List partitionKeyPath = containerResponse.getProperties().getPartitionKeyDefinition().getPaths(); + + if (partitionKeyPath.size() != 1 || !StringUtils.equals(partitionKeyPath.get(0), CosmosDataEncryptionKeyProvider.ContainerPartitionKeyPath)) { + throw new IllegalArgumentException(String.format("Provided container %s did not have the appropriate partition key definition. " + + "The container needs to be created with PartitionKeyPath set to %s.", + containerId, ContainerPartitionKeyPath)); + } + + this.container = database.getContainer(containerId); + } + + @Override + public DataEncryptionKey getDataEncryptionKey(String id, + String encryptionAlgorithm) { + Mono> fetchUnwrapMono = this + .dataEncryptionKeyContainerCore.fetchUnwrappedAsync(id); + + return fetchUnwrapMono + .map(fetchUnwrap -> fetchUnwrap.getT2().getDataEncryptionKey()) + .block(); // TODO: @moderakh I will be looking at if we should do this API async or non async. + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyContainer.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyContainer.java new file mode 100644 index 000000000000..188f8fd47328 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyContainer.java @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.models.CosmosItemResponse; +import com.azure.cosmos.models.CosmosItemRequestOptions; +import com.azure.cosmos.models.CosmosItemResponse; +import reactor.core.publisher.Mono; + + +/** + * TODO: moderakh add read-feed/query apis for data encryption key + * Container for data encryption keys. Provides methods to create, re-wrap, read and enumerate data encryption keys. + * See https://aka.ms/CosmosClientEncryption for more information on client-side encryption support in Azure Cosmos DB. + */ +public interface DataEncryptionKeyContainer { + /** + * Generates a data encryption key, wraps it using the key wrap metadata provided + * with the key wrapping provider in the EncryptionSerializer configured on the client via , + * and saves the wrapped data encryption key as an asynchronous operation in the Azure Cosmos service. + * + * @param id Unique identifier for the data encryption key. + * @param encryptionAlgorithm Encryption algorithm that will be used along with this data encryption key to encrypt/decrypt data. + * @param encryptionKeyWrapMetadata Metadata used by the configured key wrapping provider in order to wrap the key. + * @param requestOptions (Optional) The options for the request. + * @return A Mono response which wraps a {@link DataEncryptionKeyProperties} containing the read resource record. + *

+ * on Failure: {@link com.azure.cosmos.CosmosException} indicating the failure reason. + * + *

  • BadRequest - This means something was wrong with the request supplied. It is likely that an id was not supplied for the new encryption key.
    • + *
  • Conflict - This means an {@link DataEncryptionKeyProperties} with an id matching the id you supplied already existed.
    • + * + */ + Mono> createDataEncryptionKeyAsync( + String id, + String encryptionAlgorithm, + EncryptionKeyWrapMetadata encryptionKeyWrapMetadata, + CosmosItemRequestOptions requestOptions); + + /// + /// Wraps the raw data encryption key (after unwrapping using the old metadata if needed) using the provided + /// metadata with the help of the key wrapping provider in the EncryptionSerializer configured on the client via + /// , and saves the re-wrapped data encryption key as an asynchronous + /// operation in the Azure Cosmos service. + /// + /// Unique identifier of the data encryption key. + /// The metadata using which the data encryption key needs to now be wrapped. + /// (Optional) The options for the request. + /// (Optional) Token representing request cancellation. + /// An awaitable response which wraps a containing details of the data encryption key that was re-wrapped. + /// + /// This exception can encapsulate many different types of errors. + /// To determine the specific error always look at the StatusCode property. + /// Some common codes you may get when re-wrapping a data encryption key are: + /// + /// + /// StatusCode + /// Reason for exception + /// + /// + /// 404 + /// + /// NotFound - This means the resource or parent resource you tried to replace did not exist. + /// + /// + /// + /// 429 + /// + /// TooManyRequests - This means you have exceeded the number of request units per second. + /// Consult the CosmosException.RetryAfter value to see how long you should wait before retrying this operation. + /// + /// + /// + /// + /// + /// + /// + /// + /// + + /** + * + * @param id + * @param newWrapMetadata + * @param requestOptions + * @return + */ + Mono> rewrapDataEncryptionKeyAsync( + String id, + EncryptionKeyWrapMetadata newWrapMetadata, + CosmosItemRequestOptions requestOptions); + + /** + * Reads the properties of a data encryption key from the Azure Cosmos service as an asynchronous operation. + * + * @param id Unique identifier of the data encryption key. + * @param requestOptions (Optional) The options for the request. + * @return An Mono response which wraps a {@link DataEncryptionKeyProperties} containing details of the data encryption key that was read. + *

    + * on Failure: {@link com.azure.cosmos.CosmosException} indicating the failure reason. + * This exception can encapsulate many different types of errors. + * To determine the specific error always look at the StatusCode property. + * Some common codes you may get when reading a data encryption key are: + * + *

      + *
    • + * NotFound - This means the resource or parent resource you tried to read did not exist. + *
      • + * + *
    • + * TooManyRequests - This means you have exceeded the number of request units per second. + * Consult the CosmosException.RetryAfter value to see how long you should wait before retrying this operation. + *
      • + *
    + */ + Mono> readDataEncryptionKeyAsync( + String id, + CosmosItemRequestOptions requestOptions); +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyContainerCore.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyContainerCore.java new file mode 100644 index 000000000000..6013b7b24bbc --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyContainerCore.java @@ -0,0 +1,221 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; +import com.azure.cosmos.implementation.encryption.api.CosmosEncryptionAlgorithm; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey; +import com.azure.cosmos.implementation.guava25.base.Preconditions; +import com.azure.cosmos.models.CosmosItemResponse; +import com.azure.cosmos.models.CosmosItemRequestOptions; +import com.azure.cosmos.models.PartitionKey; +import reactor.core.publisher.Mono; +import reactor.util.function.Tuple2; + +import java.time.Instant; +import java.util.Arrays; + +class DataEncryptionKeyContainerCore implements DataEncryptionKeyContainer { + private final CosmosDataEncryptionKeyProvider DekProvider; + + public DataEncryptionKeyContainerCore(CosmosDataEncryptionKeyProvider dekProvider) { + this.DekProvider = dekProvider; + } + + public Mono> createDataEncryptionKeyAsync(String id, + String encryptionAlgorithm, + EncryptionKeyWrapMetadata encryptionKeyWrapMetadata, + CosmosItemRequestOptions requestOptions) { + + Preconditions.checkArgument(StringUtils.isNotEmpty(id), "id is missing"); + Preconditions.checkArgument(StringUtils.equals(encryptionAlgorithm, + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized), "Unsupported Encryption Algorithm " + encryptionAlgorithm); + Preconditions.checkNotNull(encryptionKeyWrapMetadata, "encryptionKeyWrapMetadata is missing"); + + byte[] rawDek = DataEncryptionKey.generate(encryptionAlgorithm); + + Tuple3 wrapResult = + this.wrapAsync( + id, + rawDek, + encryptionAlgorithm, + encryptionKeyWrapMetadata); + + byte[] wrappedDek = wrapResult.getLeft(); + EncryptionKeyWrapMetadata updatedMetadata = wrapResult.getMiddle(); + InMemoryRawDek inMemoryRawDek = wrapResult.getRight(); + + DataEncryptionKeyProperties dekProperties = new DataEncryptionKeyProperties(id, encryptionAlgorithm, wrappedDek, updatedMetadata, Instant.now()); + + Mono> dekResponseMono = + this.DekProvider.getContainer().createItem(dekProperties, new PartitionKey(dekProperties.id), requestOptions); + + return dekResponseMono.flatMap( + dekResponse -> { + + this.DekProvider.getDekCache().setDekProperties(id, dekResponse.getItem()); + this.DekProvider.getDekCache().setRawDek(id, inMemoryRawDek); + return Mono.just(dekResponse); + } + ); + } + + @Override + public Mono> readDataEncryptionKeyAsync( + String id, + CosmosItemRequestOptions requestOptions) { + Mono> responseMono = this.readInternalAsync( + id, + requestOptions); + + return responseMono.flatMap( + response -> { + + this.DekProvider.getDekCache().setDekProperties(id, response.getItem()); + return Mono.just(response); + } + ); + } + + @Override + public Mono> rewrapDataEncryptionKeyAsync( + String id, + EncryptionKeyWrapMetadata newWrapMetadata, + final CosmosItemRequestOptions requestOptions) { + + Preconditions.checkNotNull(newWrapMetadata, "newWrapMetadata is missing"); + Mono> resultMono = this.fetchUnwrappedAsync( + id); + + return resultMono.flatMap( + result -> { + DataEncryptionKeyProperties dekProperties = result.getT1(); + InMemoryRawDek inMemoryRawDek = result.getT2(); + + Tuple3 wrapResult = + this.wrapAsync( + id, + inMemoryRawDek.getDataEncryptionKey().getRawKey(), + dekProperties.encryptionAlgorithm, + newWrapMetadata); + + byte[] wrappedDek = wrapResult.getLeft(); + EncryptionKeyWrapMetadata updatedMetadata = wrapResult.getMiddle(); + InMemoryRawDek updatedRawDek = wrapResult.getRight(); + + CosmosItemRequestOptions effectiveRequestOptions = requestOptions != null ? requestOptions : new CosmosItemRequestOptions(); + + effectiveRequestOptions.setIfMatchETag(dekProperties.eTag); + + DataEncryptionKeyProperties newDekProperties = new DataEncryptionKeyProperties(dekProperties); + newDekProperties.wrappedDataEncryptionKey = wrappedDek; + newDekProperties.encryptionKeyWrapMetadata = updatedMetadata; + + Mono> responseMono = this.DekProvider.getContainer().replaceItem( + newDekProperties, + newDekProperties.id, + new PartitionKey(newDekProperties.id), + effectiveRequestOptions); + + return responseMono.flatMap( + response -> { + DataEncryptionKeyProperties item = response.getItem(); + + assert (item != null); + this.DekProvider.getDekCache().setDekProperties(id, item); + this.DekProvider.getDekCache().setRawDek(id, updatedRawDek); + return Mono.just(response); + } + ); + }); + } + + Mono> fetchUnwrappedAsync( + String id) { + Mono dekPropertiesMono = this.DekProvider.getDekCache().getOrAddDekPropertiesAsync( + id, + this::readResourceAsync); + + return dekPropertiesMono.flatMap( + dekProperties -> { + Mono inMemoryRawDek = this.DekProvider.getDekCache().getOrAddRawDekAsync( + dekProperties, + dp -> Mono.just(this.unwrapAsync(dp))); + + return Mono.zip(Mono.just(dekProperties), inMemoryRawDek); + } + ); + } + + static class Tuple3 { + private A a; + private B b; + private C c; + + public Tuple3(A a, B b, C c) { + this.a = a; + this.b = b; + this.c = c; + } + + public A getLeft() { + return this.a; + } + public B getMiddle() { + return this.b; + } + public C getRight() { + return this.c; + } + } + + Tuple3 wrapAsync( + String id, + byte[] key, + String encryptionAlgorithm, + EncryptionKeyWrapMetadata metadata) { + EncryptionKeyWrapResult keyWrapResponse; + + keyWrapResponse = this.DekProvider.getEncryptionKeyWrapProvider().wrapKey(key, metadata); + + // Verify + DataEncryptionKeyProperties tempDekProperties = new DataEncryptionKeyProperties(id, encryptionAlgorithm, keyWrapResponse.getWrappedDataEncryptionKey(), keyWrapResponse.getEncryptionKeyWrapMetadata(), Instant.now()); + InMemoryRawDek roundTripResponse = this.unwrapAsync(tempDekProperties); + if (!Arrays.equals(roundTripResponse.getDataEncryptionKey().getRawKey(), key)) { + throw new IllegalStateException("The key wrapping provider configured was unable to unwrap the wrapped key correctly."); + } + + return new Tuple3<>(keyWrapResponse.getWrappedDataEncryptionKey(), keyWrapResponse.getEncryptionKeyWrapMetadata(), roundTripResponse); + } + + InMemoryRawDek unwrapAsync( + DataEncryptionKeyProperties dekProperties) { + EncryptionKeyUnwrapResult unwrapResult; + + unwrapResult = this.DekProvider.getEncryptionKeyWrapProvider().unwrapKey( + dekProperties.wrappedDataEncryptionKey, + dekProperties.encryptionKeyWrapMetadata); + + DataEncryptionKey dek = DataEncryptionKey.create(unwrapResult.getDataEncryptionKey(), dekProperties.encryptionAlgorithm); + return new InMemoryRawDek(dek, unwrapResult.getClientCacheTimeToLive()); + } + + private Mono readResourceAsync( + String id) { + return this.readInternalAsync( + id, + null).map(CosmosItemResponse::getItem); + } + + private Mono> readInternalAsync( + String id, + CosmosItemRequestOptions requestOptions) { + return this.DekProvider.getContainer() + .readItem( + id, + new PartitionKey(id), + requestOptions, + DataEncryptionKeyProperties.class); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyProperties.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyProperties.java new file mode 100644 index 000000000000..f479cc18f1ab --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyProperties.java @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; +import com.azure.cosmos.implementation.guava25.base.Preconditions; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Objects; + +/** + * Details of an encryption key for use with the Azure Cosmos DB service. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({"id", "encryptionAlgorithm", "wrappedDataEncryptionKey", "keyWrapMetadata", "createTime", "_rid", "_self", "_etag", "_ts"}) +class DataEncryptionKeyProperties { + + /** + * Initializes a new instance of {@link DataEncryptionKeyProperties} + * + * @param id Unique identifier for the data encryption key. + * @param encryptionAlgorithm Encryption algorithm that will be used along with this data encryption key to encrypt/decrypt data. + * @param wrappedDataEncryptionKey Wrapped (encrypted) form of the data encryption key. + * @param encryptionKeyWrapMetadata Metadata used by the configured key wrapping provider in order to unwrap the key. + * @param createdTime created time + */ + public DataEncryptionKeyProperties(String id, + String encryptionAlgorithm, + byte[] wrappedDataEncryptionKey, + EncryptionKeyWrapMetadata encryptionKeyWrapMetadata, + Instant createdTime) { + + Preconditions.checkArgument(StringUtils.isNotEmpty(id), "id is null"); + Preconditions.checkNotNull(wrappedDataEncryptionKey, "wrappedDataEncryptionKey is null"); + Preconditions.checkNotNull(encryptionKeyWrapMetadata, "encryptionKeyWrapMetadata is null"); + + this.id = id; + this.encryptionAlgorithm = encryptionAlgorithm; + this.wrappedDataEncryptionKey = wrappedDataEncryptionKey; + this.encryptionKeyWrapMetadata = encryptionKeyWrapMetadata; + this.createdTime = createdTime; + } + + protected DataEncryptionKeyProperties() { + } + + DataEncryptionKeyProperties(DataEncryptionKeyProperties source) { + this.createdTime = source.createdTime; + this.eTag = source.eTag; + this.id = source.id; + this.encryptionAlgorithm = source.encryptionAlgorithm; + this.encryptionKeyWrapMetadata = new EncryptionKeyWrapMetadata(source.encryptionKeyWrapMetadata); + this.lastModified = source.lastModified; + this.resourceId = source.resourceId; + this.selfLink = source.selfLink; + if (source.wrappedDataEncryptionKey != null) { + this.wrappedDataEncryptionKey = new byte[source.wrappedDataEncryptionKey.length]; + + System.arraycopy(source.wrappedDataEncryptionKey, 0, this.wrappedDataEncryptionKey, 0, this.wrappedDataEncryptionKey.length); + } + } + + /** + * The identifier of the resource. + *

    + * Every resource within an Azure Cosmos DB database account needs to have a unique identifier. + * The following characters are restricted and cannot be used in the Id property: + * '/', '\\', '?', '#' + */ + @JsonProperty(value = "id", required = true) + public String id; + + /** + * Encryption algorithm that will be used along with this data encryption key to encrypt/decrypt data. + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty("encryptionAlgorithm") + public String encryptionAlgorithm; + + /** + * Wrapped form of the data encryption key. + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty(value = "wrappedDataEncryptionKey") + public byte[] wrappedDataEncryptionKey; + + /** + * Metadata for the wrapping provider that can be used to unwrap the wrapped data encryption key. + */ + @JsonProperty("keyWrapMetadata") + @JsonInclude(JsonInclude.Include.NON_NULL) + public EncryptionKeyWrapMetadata encryptionKeyWrapMetadata; + + /** + * Gets the creation time of the resource from the Azure Cosmos DB service. + */ + @JsonProperty("createTime") + @JsonDeserialize(using = UnixTimestampDeserializer.class) + @JsonSerialize(using = UnixTimestampSerializer.class) + @JsonInclude(JsonInclude.Include.NON_NULL) + public Instant createdTime; + + /** + * Gets the entity tag associated with the resource from the Azure Cosmos DB service. + *

    + * The entity tag associated with the resource. + * ETags are used for concurrency checking when updating resources. + */ + @JsonProperty("_etag") + @JsonInclude(JsonInclude.Include.NON_NULL) + public String eTag; + + /** + * Gets the last modified time stamp associated with the resource from the Azure Cosmos DB service. + * The last modified time stamp associated with the resource. + */ + @JsonProperty("_ts") + @JsonDeserialize(using = UnixTimestampDeserializer.class) + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonSerialize(using = UnixTimestampSerializer.class) + public Instant lastModified; + + /** + * Gets the self-link associated with the resource from the Azure Cosmos DB service. + *

    + * The self-link associated with the resource. + *

    + * A self-link is a static addressable Uri for each resource within a database account and follows the Azure Cosmos DB resource model. + * E.g. a self-link for a document could be dbs/db_resourceid/colls/coll_resourceid/documents/doc_resourceid + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonProperty("_self") + public String selfLink; + + /** + * Gets the Resource Id associated with the resource in the Azure Cosmos DB service. + *

    + * The Resource Id associated with the resource. + * A Resource Id is the unique, immutable, identifier assigned to each Azure Cosmos DB + *

    + * A Resource Id is the unique, immutable, identifier assigned to each Azure Cosmos DB + * resource whether that is a database, a collection or a document. + * These resource ids are used when building up SelfLinks, a static addressable Uri for each resource within a database account. + */ + @JsonProperty("_rid") + @JsonInclude(JsonInclude.Include.NON_NULL) + String resourceId; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DataEncryptionKeyProperties that = (DataEncryptionKeyProperties) o; + return Objects.equals(id, that.id) && + Objects.equals(encryptionAlgorithm, that.encryptionAlgorithm) && + Arrays.equals(wrappedDataEncryptionKey, that.wrappedDataEncryptionKey) && + Objects.equals(encryptionKeyWrapMetadata, that.encryptionKeyWrapMetadata) && + Objects.equals(createdTime, that.createdTime) && + Objects.equals(eTag, that.eTag) && + Objects.equals(lastModified, that.lastModified) && + Objects.equals(selfLink, that.selfLink) && + Objects.equals(resourceId, that.resourceId); + } + + @Override + public int hashCode() { + int result = Objects.hash(id, encryptionAlgorithm, encryptionKeyWrapMetadata, createdTime, eTag, lastModified, selfLink, resourceId); + result = 31 * result + Arrays.hashCode(wrappedDataEncryptionKey); + return result; + } + + public static boolean equals(byte[] x, byte[] y) { + return (x == null && y == null) + || (x != null && y != null && Arrays.equals(x, y)); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DekCache.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DekCache.java new file mode 100644 index 000000000000..1e3754756f57 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/DekCache.java @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.caches.AsyncCache; +import reactor.core.publisher.Mono; + +import java.time.Duration; +import java.time.Instant; +import java.util.function.Function; + +class DekCache { + private final AsyncCache DekPropertiesCache = new AsyncCache<>(); + private final AsyncCache RawDekCache = new AsyncCache<>(); + private final Duration dekPropertiesTimeToLive; + + public DekCache() { + this(null); + } + + public DekCache(Duration dekPropertiesTimeToLive) { + if (dekPropertiesTimeToLive != null) { + this.dekPropertiesTimeToLive = dekPropertiesTimeToLive; + } else { + this.dekPropertiesTimeToLive = Duration.ofMinutes(30); + } + } + + public Mono getOrAddDekPropertiesAsync( + String dekId, + Function> fetcher) { + Mono cachedDekPropertiesMono = this.DekPropertiesCache.getAsync( + dekId, + null, + () -> this.fetchAsync(dekId, fetcher)); + + return cachedDekPropertiesMono.flatMap(cachedDekProperties -> { + if (cachedDekProperties.getServerPropertiesExpiryUtc().isBefore(Instant.now())) { + return this.DekPropertiesCache.getAsync( + dekId, + null, + () -> this.fetchAsync(dekId, fetcher)); + } else { + return Mono.just(cachedDekProperties); + } + + } + ).map(CachedDekProperties::getServerProperties); + } + + public Mono getOrAddRawDekAsync( + DataEncryptionKeyProperties dekProperties, + Function> unwrapper) { + Mono inMemoryRawDekMono = this.RawDekCache.getAsync( + dekProperties.selfLink, + null, + () -> unwrapper.apply(dekProperties)); + + return inMemoryRawDekMono.flatMap( + inMemoryRawDek -> { + if (inMemoryRawDek.getRawDekExpiry().isBefore(Instant.now())) { + + return this.RawDekCache.getAsync( + dekProperties.selfLink, + null, + () -> unwrapper.apply(dekProperties) + /* forceRefresh: true */); + } else { + return Mono.just(inMemoryRawDek); + } + } + ); + } + + public void setDekProperties(String dekId, DataEncryptionKeyProperties dekProperties) { + CachedDekProperties cachedDekProperties = new CachedDekProperties(dekProperties, Instant.now().plus(this.dekPropertiesTimeToLive)); + this.DekPropertiesCache.set(dekId, cachedDekProperties); + } + + public void setRawDek(String dekId, InMemoryRawDek inMemoryRawDek) { + this.RawDekCache.set(dekId, inMemoryRawDek); + } + + public Mono removeAsync(String dekId) { + Mono cachedDekPropertiesMono = this.DekPropertiesCache.removeAsync(dekId); + + return cachedDekPropertiesMono.flatMap(cachedDekProperties -> this.RawDekCache.removeAsync(dekId)).then(); + } + + private Mono fetchAsync( + String dekId, + Function> fetcher) { + Mono serverPropertiesMono = fetcher.apply(dekId); + return serverPropertiesMono.map(serverProperties -> new CachedDekProperties(serverProperties, Instant.now().plus(this.dekPropertiesTimeToLive))); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionExceptionFactory.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionExceptionFactory.java new file mode 100644 index 000000000000..75d6f5d95a07 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionExceptionFactory.java @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.guava27.Strings; + +class EncryptionExceptionFactory { + + static class InvalidArgumentException extends IllegalArgumentException { + public InvalidArgumentException(String msg, String argName) { + super(Strings.lenientFormat("argName: %s, details: %s", argName, msg)); + } + } + + static RuntimeException invalidKeySize(String algorithmName, int actualKeylength, int expectedLength) { + return new InvalidArgumentException( + Strings.lenientFormat("Invalid key size for %s; actual: %s, expected: %s", + algorithmName, actualKeylength, expectedLength), "dataEncryptionKey"); + } + + static RuntimeException invalidCipherTextSize(int actualSize, int minimumSize) { + return new InvalidArgumentException( + Strings.lenientFormat("Invalid cipher text size; actual: %s, minimum expected: %s.", + actualSize, minimumSize), "cipherText"); + } + + static RuntimeException invalidAlgorithmVersion(byte actual, byte expected) { + return new InvalidArgumentException( + Strings.lenientFormat("Invalid encryption algorithm version; actual: %s, expected: %s.", + Bytes.toHex(actual), Bytes.toHex(expected)), "cipherText"); + } + + static RuntimeException invalidAuthenticationTag() { + return new InvalidArgumentException( + "Invalid authentication tag in cipher text.", + "cipherText"); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyUnwrapResult.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyUnwrapResult.java new file mode 100644 index 000000000000..f942baef1a8a --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyUnwrapResult.java @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.guava25.base.Preconditions; + +import java.time.Duration; + +public class EncryptionKeyUnwrapResult { + + /** + * Initializes a new instance of the result of unwrapping a wrapped data encryption key. + * + * @param dataEncryptionKey Raw form of data encryption key. + * The byte array passed in must not be modified after this call by the {@link EncryptionKeyWrapProvider} + * @param clientCacheTimeToLive Amount of time after which the raw data encryption key must not be used + * without invoking the {@link EncryptionKeyWrapProvider#unwrapKey(byte[], EncryptionKeyWrapMetadata)} + * + */ + public EncryptionKeyUnwrapResult(byte[] dataEncryptionKey, Duration clientCacheTimeToLive) { + Preconditions.checkNotNull(dataEncryptionKey, "dataEncryptionKey is null"); + this.dataEncryptionKey = dataEncryptionKey; + this.clientCacheTimeToLive = clientCacheTimeToLive; + } + + /** + * Gets raw form of the data encryption key. + * @return encrypted key. + */ + public byte[] getDataEncryptionKey() { + return dataEncryptionKey; + } + + /** + * Gets amount of time after which the raw data encryption key must not be used + * without invoking the {@link EncryptionKeyWrapProvider#unwrapKey(byte[], EncryptionKeyWrapMetadata)} + * @return client cache time to live. + */ + public Duration getClientCacheTimeToLive() { + return clientCacheTimeToLive; + } + + private final byte[] dataEncryptionKey; + private final Duration clientCacheTimeToLive; +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyWrapMetadata.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyWrapMetadata.java new file mode 100644 index 000000000000..1bee51f35369 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyWrapMetadata.java @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; +import com.azure.cosmos.implementation.guava25.base.Preconditions; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + + +/** + * Metadata that a key wrapping provider can use to wrap/unwrap data encryption keys. + * {@link EncryptionKeyWrapProvider} + */ +public class EncryptionKeyWrapMetadata { + /** + * For JSON deserialize + */ + EncryptionKeyWrapMetadata() { + } + + /** + * Creates a new instance of key wrap metadata. + * + * @param value Value of the metadata + */ + public EncryptionKeyWrapMetadata(String value) { + this("custom", value); + } + + /** + * Creates a new instance of key wrap metadata based on an existing instance. + * + * @param source Existing instance from which to initialize. + */ + public EncryptionKeyWrapMetadata(EncryptionKeyWrapMetadata source) { + this.type = source.type; + this.algorithm = source.algorithm; + this.value = source.value; + } + + EncryptionKeyWrapMetadata(String type, String value) { + this(type, value, null); + } + + EncryptionKeyWrapMetadata(String type, String value, String algorithm) { + Preconditions.checkNotNull(type, "type is null"); + Preconditions.checkNotNull(value, "value is null"); + this.type = type; + this.value = value; + this.algorithm = algorithm; + } + + @JsonProperty("type") + @JsonInclude(JsonInclude.Include.NON_NULL) + public String type; + + @JsonProperty("algorithm") + @JsonInclude(JsonInclude.Include.NON_NULL) + public String algorithm; + + /** + * Serialized form of metadata. + * Note: This value is saved in the Cosmos DB service. + * implementors of derived implementations should ensure that this does not have (private) key material or credential information. + */ + @JsonProperty("value") + @JsonInclude(JsonInclude.Include.NON_NULL) + public String value; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EncryptionKeyWrapMetadata that = (EncryptionKeyWrapMetadata) o; + return Objects.equals(type, that.type) && + Objects.equals(algorithm, that.algorithm) && + Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(type, algorithm, value); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyWrapProvider.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyWrapProvider.java new file mode 100644 index 000000000000..d280b5b8067f --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyWrapProvider.java @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +/** + * TODO: @moderakh look into if this class needs to be async + * Interface for interacting with a provider that can be used to wrap (encrypt) and unwrap (decrypt) data encryption keys for envelope based encryption. + * Implementations are expected to ensure that master keys are highly available and protected against accidental deletion. + * See https://aka.ms/CosmosClientEncryption for more information on client-side encryption support in Azure Cosmos DB. + */ +public interface EncryptionKeyWrapProvider { + + /** + * Wraps (i.e. encrypts) the provided data encryption key. + * @param key Data encryption key that needs to be wrapped. + * @param metadata Metadata for the wrap provider that should be used to wrap / unwrap the key.< + * @return Wrapped (i.e. encrypted) version of data encryption key passed in possibly with updated metadata. + */ + EncryptionKeyWrapResult wrapKey(byte[] key, EncryptionKeyWrapMetadata metadata); + + /** + * Unwraps (i.e. decrypts) the provided wrapped data encryption key. + * @param wrappedKey Wrapped form of data encryption key that needs to be unwrapped. + * @param metadata Metadata for the wrap provider that should be used to wrap / unwrap the key. + * @return unwrapped (i.e. unencrypted) version of data encryption key passed in and how long the raw data encryption key can be cached on the client. + */ + EncryptionKeyUnwrapResult unwrapKey(byte[] wrappedKey, EncryptionKeyWrapMetadata metadata); +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyWrapResult.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyWrapResult.java new file mode 100644 index 000000000000..acbb46e31c3b --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionKeyWrapResult.java @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.guava25.base.Preconditions; + + +/** + * Result from a {@link EncryptionKeyWrapProvider} on wrapping a data encryption key. + */ +public class EncryptionKeyWrapResult { + + private final byte[] wrappedDataEncryptionKey; + private final EncryptionKeyWrapMetadata encryptionKeyWrapMetadata; + + /** + * Initializes a new instance of the result of wrapping a data encryption key. + * + * @param wrappedDataEncryptionKey Wrapped form of data encryption key. + * The byte array passed in must not be modified after this call by the {@link EncryptionKeyWrapResult} + * @param encryptionKeyWrapMetadata Metadata that can be used by the wrap provider to unwrap the data encryption key. + */ + public EncryptionKeyWrapResult(byte[] wrappedDataEncryptionKey, EncryptionKeyWrapMetadata encryptionKeyWrapMetadata) { + Preconditions.checkNotNull(wrappedDataEncryptionKey, "wrappedDataEncryptionKey is null"); + Preconditions.checkNotNull(encryptionKeyWrapMetadata, "encryptionKeyWrapMetadata is null"); + + this.wrappedDataEncryptionKey = wrappedDataEncryptionKey; + this.encryptionKeyWrapMetadata = encryptionKeyWrapMetadata; + } + + /** + * Gets wrapped form of the data encryption key. + * @return wrapped data encryption key. + */ + public byte[] getWrappedDataEncryptionKey() { + return wrappedDataEncryptionKey; + } + + /** + * Gets metadata that can be used by the wrap provider to unwrap the key. + * @return encryption key wrap metadata. + */ + public EncryptionKeyWrapMetadata getEncryptionKeyWrapMetadata() { + return encryptionKeyWrapMetadata; + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/HMACSHA256.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/HMACSHA256.java new file mode 100644 index 000000000000..a21f3ee1c529 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/HMACSHA256.java @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import javax.crypto.Mac; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.io.Closeable; +import java.io.IOException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +class HMACSHA256 implements Closeable { + private static final String ALGO_NAME = "HMACSHA256"; + private final Mac mac; + private byte[] hashValue; + + public HMACSHA256(byte[] key) { + mac = getMac(ALGO_NAME); + SecretKey secretKey = new SecretKeySpec(key, ALGO_NAME); + try { + mac.init(secretKey); + } catch (InvalidKeyException e) { + throw new IllegalArgumentException(e); + } + } + + private static Mac getMac(String algo) { + try { + return Mac.getInstance(algo); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + @Override + public void close() { + // No op + } + + public byte[] computeHash(byte[] plainText) { + return mac.doFinal(plainText); + } + + /** + * Computes the hash value for the specified region of the input byte array and copies the specified region of the input byte array to the specified region of the output byte array. + * + * @param inputBuffer + * @param inputOffset + * @param inputCount + * @param outputBuffer + * @param outputOffset + * @return + */ + public int transformBlock(byte[] inputBuffer, int inputOffset, int inputCount, byte[] outputBuffer, int outputOffset) { + mac.update(inputBuffer, inputOffset, inputCount); + if ((outputBuffer != null) && ((inputBuffer != outputBuffer) || (inputOffset != outputOffset))) { + // We let BlockCopy do the destination array validation + System.arraycopy(inputBuffer, inputOffset, outputBuffer, outputOffset, inputCount); + } + return inputCount; + } + + /** + * Computes the hash value for the specified region of the specified byte array. + * + * @param inputBuffer + * @param inputOffset + * @param inputCount + * @return + */ + public byte[] transformFinalBlock(byte[] inputBuffer, int inputOffset, int inputCount) { + mac.update(inputBuffer, inputOffset, inputCount); + hashValue = mac.doFinal(); + return hashValue; + } + + public byte[] getHash() { + return hashValue; + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/InMemoryRawDek.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/InMemoryRawDek.java new file mode 100644 index 000000000000..7b8593063f8f --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/InMemoryRawDek.java @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey; + +import java.time.Duration; +import java.time.Instant; + +class InMemoryRawDek { + private final DataEncryptionKey DataEncryptionKey; + private final Instant RawDekExpiry; + + public InMemoryRawDek(DataEncryptionKey dataEncryptionKey, Duration clientCacheTimeToLive) { + this.DataEncryptionKey = dataEncryptionKey; + this.RawDekExpiry = Instant.now().plus(clientCacheTimeToLive); + } + + public DataEncryptionKey getDataEncryptionKey() { + return DataEncryptionKey; + } + public Instant getRawDekExpiry() { + return RawDekExpiry; + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/RNGCryptoServiceProvider.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/RNGCryptoServiceProvider.java new file mode 100644 index 000000000000..b6ba349e3639 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/RNGCryptoServiceProvider.java @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import java.io.Closeable; +import java.io.IOException; +import java.security.SecureRandom; + +class RNGCryptoServiceProvider implements Closeable { + // TODO: is this thread safe? efficient, etc? + private SecureRandom random = new SecureRandom(); + + public void getBytes(byte[] randomBytes) { + random.nextBytes(randomBytes); + } + + @Override + public void close() { + + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/SHA256.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/SHA256.java new file mode 100644 index 000000000000..6fb76b8b679d --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/SHA256.java @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import java.io.Closeable; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +class SHA256 implements Closeable { + private final MessageDigest digest; + + private SHA256() { + digest = getMessageDigest(); + } + + public static SHA256 create() { + return new SHA256(); + } + + public static MessageDigest getMessageDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + @Override + public void close() { + + } + + public byte[] computeHash(byte[] input) { + return digest.digest(input); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/SecurityUtility.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/SecurityUtility.java new file mode 100644 index 000000000000..3bc9e415709d --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/SecurityUtility.java @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +class SecurityUtility { + final static int MAX_SHA_256_HASH_BYTES = 32; + + /** + * Computes a keyed hash of a given text and returns. It fills the buffer "hash" with computed hash value. + * + * @param plainText Plain text bytes whose hash has to be computed. + * @param key key used for the HMAC. + * @param hash Output buffer where the computed hash value is stored. If it is less than 32 bytes, the hash is truncated. + */ + static void getHMACWithSHA256(byte[] plainText, byte[] key, byte[] hash) { + + assert (key != null && plainText != null); + assert (hash.length != 0 && hash.length <= MAX_SHA_256_HASH_BYTES); + + try (HMACSHA256 hmac = new HMACSHA256(key)) { + byte[] computedHash = hmac.computeHash(plainText); + + // Truncate the hash if needed + System.arraycopy(computedHash, 0, hash, 0, hash.length); + } + } + + /** + * Computes SHA256 hash of a given input. + * + * @param input input byte array which needs to be hashed. + * @return Returns SHA256 hash in a string form. + */ + static String getSHA256Hash(byte[] input) { + assert (input != null); + + try (SHA256 sha256 = SHA256.create()) { + byte[] hashValue = sha256.computeHash(input); + return getHexString(hashValue); + } + } + + /** + * Generates cryptographically random bytes. + * + * @param randomBytes Buffer into which cryptographically random bytes are to be generated. + */ + public static void generateRandomBytes(byte[] randomBytes) { + // Generate random bytes cryptographically. + try (RNGCryptoServiceProvider rngCsp = new RNGCryptoServiceProvider()) { + rngCsp.getBytes(randomBytes); + } + } + + /** + * Compares two byte arrays and returns true if all bytes are equal. + * + * @param buffer1 input buffer + * @param buffer2 another buffer to be compared against + * @param buffer2Index + * @param lengthToCompare + * @return returns true if both the arrays have the same byte values else returns false + */ + static boolean compareBytes(byte[] buffer1, byte[] buffer2, int buffer2Index, int lengthToCompare) { + if (buffer1 == null || buffer2 == null) { + return false; + } + + assert buffer1.length >= lengthToCompare : "invalid lengthToCompare"; + assert buffer2Index > -1 && buffer2Index < buffer2.length : "invalid index"; + if ((buffer2.length - buffer2Index) < lengthToCompare) { + return false; + } + + for (int index = 0; index < buffer1.length && index < lengthToCompare; ++index) { + if (buffer1[index] != buffer2[buffer2Index + index]) { + return false; + } + } + + return true; + } + + /** + * Gets hex representation of byte array. + * + * @param input input byte array + * @return + */ + private static String getHexString(byte[] input) { + assert (input != null); + + return Bytes.toHex(input); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/SymmetricKey.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/SymmetricKey.java new file mode 100644 index 000000000000..0c25bac14d36 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/SymmetricKey.java @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +/** + * Base class containing raw key bytes for symmetric key algorithms. Some encryption algorithms can use the key directly while others derive sub keys from this. + * If an algorithm needs to derive more keys, have a derived class from this and use it in the corresponding encryption algorithm. + */ +class SymmetricKey { + + /** + * The underlying key material + */ + protected final byte[] rootKey; + + /** + * Constructor that initializes the root key. + * + * @param rootKey root key + */ + SymmetricKey(byte[] rootKey) { + // Key validation + if (rootKey == null || rootKey.length == 0) { + throw new IllegalArgumentException("rootKey"); + } + + this.rootKey = rootKey; + } + + /** + * Returns a copy of the plain text key + * This is needed for actual encryption/decryption. + * + * @return root key byte array + */ + protected byte[] getRootKey() { + return this.rootKey; + } + + /** + * Computes SHA256 value of the plain text key bytes + * + * @return A string containing SHA256 hash of the root key + */ + protected String getKeyHash() { + return SecurityUtility.getSHA256Hash(this.getRootKey()); + } + + /** + * Gets the length of the root key + * + * @return Returns the length of the root key + */ + int getLength() { + return this.rootKey.length; + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/UnixTimestampDeserializer.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/UnixTimestampDeserializer.java new file mode 100644 index 000000000000..1a264d5754a1 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/UnixTimestampDeserializer.java @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.ObjectCodec; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; + +import java.io.IOException; +import java.time.Instant; + +class UnixTimestampDeserializer extends JsonDeserializer { + @Override + public Instant deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + ObjectCodec codec = jsonParser.getCodec(); + Long seconds = codec.readValue(jsonParser, Long.class); + + if (seconds == null) { + return null; + } + + return Instant.ofEpochSecond(seconds); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/UnixTimestampSerializer.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/UnixTimestampSerializer.java new file mode 100644 index 000000000000..c81153f54a07 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/UnixTimestampSerializer.java @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; +import java.time.Instant; + +class UnixTimestampSerializer extends JsonSerializer { + @Override + public void serialize(Instant instant, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeNumber(instant.getEpochSecond()); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java new file mode 100644 index 000000000000..7d856c2be4b2 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/CosmosAsyncClientTest.java @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos; + +import com.azure.cosmos.implementation.ConnectionPolicy; +import com.azure.cosmos.implementation.guava27.Strings; +import org.testng.ITest; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; + +import java.lang.reflect.Method; + +public abstract class CosmosAsyncClientTest implements ITest { + + private final CosmosClientBuilder clientBuilder; + private String testName; + + public CosmosAsyncClientTest() { + this(new CosmosClientBuilder()); + } + + public CosmosAsyncClientTest(CosmosClientBuilder clientBuilder) { + this.clientBuilder = clientBuilder; + } + + public final CosmosClientBuilder getClientBuilder() { + return this.clientBuilder; + } + + public final ConnectionPolicy getConnectionPolicy() { + return this.clientBuilder.getConnectionPolicy(); + } + + @Override + public final String getTestName() { + return this.testName; + } + + @BeforeMethod(alwaysRun = true) + public final void setTestName(Method method) { + String testClassAndMethodName = Strings.lenientFormat("%s::%s", + method.getDeclaringClass().getSimpleName(), + method.getName()); + + if (this.clientBuilder.getConnectionPolicy() != null && this.clientBuilder.configs() != null) { + String connectionMode = this.clientBuilder.getConnectionPolicy().getConnectionMode() == ConnectionMode.DIRECT + ? "Direct " + this.clientBuilder.configs().getProtocol() + : "Gateway"; + + this.testName = Strings.lenientFormat("%s[%s with %s consistency]", + testClassAndMethodName, + connectionMode, + clientBuilder.getConsistencyLevel()); + } else { + this.testName = testClassAndMethodName; + } + } + + @AfterMethod(alwaysRun = true) + public final void unsetTestName() { + this.testName = null; + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/CosmosDatabaseForTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/CosmosDatabaseForTest.java new file mode 100644 index 000000000000..d5d8a9076519 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/CosmosDatabaseForTest.java @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos; + +import com.azure.cosmos.models.CosmosDatabaseResponse; +import com.azure.cosmos.models.CosmosDatabaseProperties; +import com.azure.cosmos.models.SqlParameter; +import com.azure.cosmos.models.SqlQuerySpec; +import com.azure.cosmos.util.CosmosPagedFlux; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Mono; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CosmosDatabaseForTest { + private static Logger logger = LoggerFactory.getLogger(CosmosDatabaseForTest.class); + public static final String SHARED_DB_ID_PREFIX = "RxJava.SDKTest.SharedDatabase"; + private static final Duration CLEANUP_THRESHOLD_DURATION = Duration.ofHours(2); + private static final String DELIMITER = "_"; + private static DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss"); + + public LocalDateTime createdTime; + public CosmosAsyncDatabase createdDatabase; + + private CosmosDatabaseForTest(CosmosAsyncDatabase db, LocalDateTime createdTime) { + this.createdDatabase = db; + this.createdTime = createdTime; + } + + private boolean isStale() { + return isOlderThan(CLEANUP_THRESHOLD_DURATION); + } + + private boolean isOlderThan(Duration dur) { + return createdTime.isBefore(LocalDateTime.now().minus(dur)); + } + + public static String generateId() { + return SHARED_DB_ID_PREFIX + DELIMITER + TIME_FORMATTER.format(LocalDateTime.now()) + DELIMITER + RandomStringUtils.randomAlphabetic(3); + } + + private static CosmosDatabaseForTest from(CosmosAsyncDatabase db) { + if (db == null || db.getId() == null || db.getLink() == null) { + return null; + } + + String id = db.getId(); + if (id == null) { + return null; + } + + String[] parts = StringUtils.split(id, DELIMITER); + if (parts.length != 3) { + return null; + } + if (!StringUtils.equals(parts[0], SHARED_DB_ID_PREFIX)) { + return null; + } + + try { + LocalDateTime parsedTime = LocalDateTime.parse(parts[1], TIME_FORMATTER); + return new CosmosDatabaseForTest(db, parsedTime); + } catch (Exception e) { + return null; + } + } + + public static CosmosDatabaseForTest create(DatabaseManager client) { + CosmosDatabaseProperties dbDef = new CosmosDatabaseProperties(generateId()); + + client.createDatabase(dbDef).block(); + CosmosAsyncDatabase db = client.getDatabase(dbDef.getId()); + CosmosDatabaseForTest dbForTest = CosmosDatabaseForTest.from(db); + assertThat(dbForTest).isNotNull(); + return dbForTest; + } + + public static void cleanupStaleTestDatabases(DatabaseManager client) { + logger.info("Cleaning stale test databases ..."); + List sqlParameterList = new ArrayList<>(); + sqlParameterList.add(new SqlParameter("@PREFIX", CosmosDatabaseForTest.SHARED_DB_ID_PREFIX)); + List dbs = client.queryDatabases( + new SqlQuerySpec("SELECT * FROM c WHERE STARTSWITH(c.id, @PREFIX)", sqlParameterList)).collectList().block(); + + for (CosmosDatabaseProperties db : dbs) { + assertThat(db.getId()).startsWith(CosmosDatabaseForTest.SHARED_DB_ID_PREFIX); + + CosmosDatabaseForTest dbForTest = CosmosDatabaseForTest.from(client.getDatabase(db.getId())); + + if (db != null && dbForTest.isStale()) { + logger.info("Deleting database {}", db.getId()); + dbForTest.deleteDatabase(db.getId()); + } + } + } + + private void deleteDatabase(String id) { + this.createdDatabase.delete().block(); + } + + public interface DatabaseManager { + CosmosPagedFlux queryDatabases(SqlQuerySpec query); + Mono createDatabase(CosmosDatabaseProperties databaseDefinition); + CosmosAsyncDatabase getDatabase(String id); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/EncryptionCodeSnippet.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/EncryptionCodeSnippet.java new file mode 100644 index 000000000000..e2952326babc --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/EncryptionCodeSnippet.java @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +package com.azure.cosmos; + +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider; +import com.azure.cosmos.implementation.encryption.api.EncryptionOptions; +import com.azure.cosmos.implementation.guava25.collect.ImmutableList; +import com.azure.cosmos.models.CosmosItemRequestOptions; +import com.azure.cosmos.models.CosmosItemResponse; +import com.azure.cosmos.models.ModelBridgeInternal; +import com.azure.cosmos.models.PartitionKey; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.UUID; + +/** + * Code snippets for {@link ChangeFeedProcessor} + */ +public class EncryptionCodeSnippet { + + public void encryptionSample() { + CosmosClientBuilder builder = new CosmosClientBuilder(); + + CosmosClient client = builder.key("key") + .endpoint("endpoint") + .dataEncryptionKeyProvider(naiveDataEncryptionKeyProvider()) + .buildClient(); + + CosmosContainer container = client.getDatabase("myDb").getContainer("myCol"); + + Pojo pojo = new Pojo(); + pojo.id = UUID.randomUUID().toString(); + pojo.mypk = UUID.randomUUID().toString(); + pojo.nonSensitive = UUID.randomUUID().toString(); + pojo.sensitive1 = "this is a secret to be encrypted"; + pojo.sensitive2 = "this is a another secret to be encrypted"; + + CosmosItemRequestOptions options = new CosmosItemRequestOptions(); + EncryptionOptions encryptionOptions = new EncryptionOptions(); + encryptionOptions.setPathsToEncrypt(ImmutableList.of("/sensitive1", "/sensitive2")); + ModelBridgeInternal.setEncryptionOptions(options, encryptionOptions); + + CosmosItemResponse response = container.createItem(pojo, options); + + assert response.getItem().nonSensitive != null; + assert response.getItem().sensitive1 == null; + assert response.getItem().sensitive2 == null; + + + CosmosItemResponse readResponse = container.readItem(pojo.id, new PartitionKey(pojo.mypk), Pojo.class); + + assert response.getItem().nonSensitive != null; + assert response.getItem().sensitive1 != null; + assert response.getItem().sensitive2 != null; + } + + private DataEncryptionKeyProvider naiveDataEncryptionKeyProvider() { + // this is a naive data encryption key provider which always uses the same data encryption key in memory. + // the user should implement DataEncryptionKeyProvider as per use case; + // storing data encryption keys should happen on the app side. + DataEncryptionKey key = createDataEncryptionKey(); + + return new DataEncryptionKeyProvider() { + @Override + public DataEncryptionKey getDataEncryptionKey(String id, String algorithm) { + return key; + } + }; + } + + public static class Pojo { + @JsonProperty + private String id; + @JsonProperty + private String mypk; + @JsonProperty + private String nonSensitive; + @JsonProperty + private String sensitive1; + @JsonProperty + private String sensitive2; + } + + private DataEncryptionKey createDataEncryptionKey() { + byte[] key = DataEncryptionKey.generate("AEAes256CbcHmacSha256Randomized"); + return DataEncryptionKey.create(key, "AEAes256CbcHmacSha256Randomized"); + } +} + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/EncryptionTest2.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/EncryptionTest2.java new file mode 100644 index 000000000000..a77bf5d3abc1 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/EncryptionTest2.java @@ -0,0 +1,257 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos; + +import com.azure.cosmos.implementation.encryption.SimpleInMemoryProvider; +import com.azure.cosmos.implementation.encryption.TestUtils; +import com.azure.cosmos.implementation.encryption.api.CosmosEncryptionAlgorithm; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey; +import com.azure.cosmos.implementation.encryption.api.EncryptionOptions; +import com.azure.cosmos.implementation.guava25.collect.ImmutableList; +import com.azure.cosmos.models.CosmosItemRequestOptions; +import com.azure.cosmos.models.CosmosItemResponse; +import com.azure.cosmos.models.CosmosQueryRequestOptions; +import com.azure.cosmos.models.CosmosQueryRequestOptions; +import com.azure.cosmos.models.FeedResponse; +import com.azure.cosmos.models.ModelBridgeInternal; +import com.azure.cosmos.models.PartitionKey; +import com.azure.cosmos.models.SqlQuerySpec; +import com.azure.cosmos.rx.TestSuiteBase; +import com.azure.cosmos.util.CosmosPagedIterable; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EncryptionTest2 extends TestSuiteBase { + static SimpleInMemoryProvider simpleInMemoryProvider = new SimpleInMemoryProvider(); + + private CosmosClient client; + private CosmosContainer container; + private static final int TIMEOUT = 60_000; + + @Factory(dataProvider = "clientBuilders") + public EncryptionTest2(CosmosClientBuilder clientBuilder) { + super(CosmosBridgeInternal.setDateKeyProvider(clientBuilder, simpleInMemoryProvider)); + } + + @BeforeClass(groups = {"emulator"}, timeOut = SETUP_TIMEOUT) + public void before_CosmosItemTest() { + assertThat(this.client).isNull(); + this.client = getClientBuilder().buildClient(); + CosmosAsyncContainer asyncContainer = getSharedMultiPartitionCosmosContainer(this.client.asyncClient()); + container = client.getDatabase(asyncContainer.getDatabase().getId()).getContainer(asyncContainer.getId()); + } + + @BeforeClass(groups = "emulator") + public void beforeClass() { + TestUtils.initialized(); + } + + @AfterClass(groups = {"emulator"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) + public void afterClass() { + assertThat(this.client).isNotNull(); + this.client.close(); + } + + @Test(groups = {"emulator"}, timeOut = TIMEOUT) + public void createItemEncrypt_readItemDecrypt() throws Exception { + CosmosItemRequestOptions requestOptions = new CosmosItemRequestOptions(); + EncryptionOptions encryptionOptions = new EncryptionOptions(); + encryptionOptions.setPathsToEncrypt(ImmutableList.of("/sensitive")); + + String keyId = UUID.randomUUID().toString(); + + DataEncryptionKey dataEncryptionKey = createDataEncryptionKey(); + simpleInMemoryProvider.addKey(keyId, dataEncryptionKey); + + encryptionOptions.setDataEncryptionKeyId(keyId); + encryptionOptions.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized); + ModelBridgeInternal.setEncryptionOptions(requestOptions, encryptionOptions); + + Pojo properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse = container.createItem(properties, requestOptions); + assertThat(itemResponse.getRequestCharge()).isGreaterThan(0); + + Pojo responseItem = itemResponse.getItem(); + validateWriteResponseIsValid(properties, responseItem); + + Pojo readItem = container.readItem(properties.id, new PartitionKey(properties.mypk), requestOptions, Pojo.class).getItem(); + validateReadResponseIsValid(properties, readItem); + } + + @Test(groups = {"emulator"}, timeOut = TIMEOUT) + public void upsertItem_readItem() throws Exception { + CosmosItemRequestOptions requestOptions = new CosmosItemRequestOptions(); + EncryptionOptions encryptionOptions = new EncryptionOptions(); + encryptionOptions.setPathsToEncrypt(ImmutableList.of("/sensitive")); + + String keyId = UUID.randomUUID().toString(); + DataEncryptionKey dataEncryptionKey = createDataEncryptionKey(); + simpleInMemoryProvider.addKey(keyId, dataEncryptionKey); + + encryptionOptions.setDataEncryptionKeyId(keyId); + ModelBridgeInternal.setEncryptionOptions(requestOptions, encryptionOptions); + encryptionOptions.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized); + + Pojo properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse = container.upsertItem(properties, requestOptions); + assertThat(itemResponse.getRequestCharge()).isGreaterThan(0); + + Pojo responseItem = itemResponse.getItem(); + validateWriteResponseIsValid(properties, responseItem); + + Pojo readItem = container.readItem(properties.id, new PartitionKey(properties.mypk), requestOptions, Pojo.class).getItem(); + validateReadResponseIsValid(properties, readItem); + } + + + private void validateWriteResponseIsValid(Pojo originalItem, Pojo result) { + assertThat(result.sensitive).isEqualTo(originalItem.sensitive); + assertThat(result.id).isEqualTo(originalItem.id); + assertThat(result.mypk).isEqualTo(originalItem.mypk); + assertThat(result.nonSensitive).isEqualTo(originalItem.nonSensitive); + } + + private void validateReadResponseIsValid(Pojo originalItem, Pojo result) { + assertThat(result.id).isEqualTo(originalItem.id); + assertThat(result.mypk).isEqualTo(originalItem.mypk); + assertThat(result.nonSensitive).isEqualTo(originalItem.nonSensitive); + assertThat(result.sensitive).isEqualTo(originalItem.sensitive); + } + + private void validateQueryResponseIsValid(Pojo originalItem, Pojo result) { + assertThat(result.id).isEqualTo(originalItem.id); + assertThat(result.mypk).isEqualTo(originalItem.mypk); + assertThat(result.nonSensitive).isEqualTo(originalItem.nonSensitive); + assertThat(result.sensitive).isNull(); + } + + @Test(groups = {"emulator"}, timeOut = TIMEOUT) + public void readItem() throws Exception { + Pojo properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse = container.createItem(properties); + + CosmosItemResponse readResponse1 = container.readItem(properties.id, + new PartitionKey(properties.mypk), + new CosmosItemRequestOptions(), + Pojo.class); + validateItemResponse(properties, readResponse1); + } + + @Test(groups = {"emulator"}, timeOut = TIMEOUT) + public void readAllItems() throws Exception { + Pojo properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse = container.createItem(properties); + + CosmosQueryRequestOptions CosmosQueryRequestOptions = new CosmosQueryRequestOptions(); + + CosmosPagedIterable feedResponseIterator3 = + container.readAllItems(CosmosQueryRequestOptions, Pojo.class); + assertThat(feedResponseIterator3.iterator().hasNext()).isTrue(); + } + + + @Test(groups = {"emulator"}, timeOut = TIMEOUT) + public void queryItems() throws Exception { + Pojo properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse = container.createItem(properties); + + String query = String.format("SELECT * from c where c.id = '%s'", properties.id); + CosmosQueryRequestOptions CosmosQueryRequestOptions = new CosmosQueryRequestOptions(); + + CosmosPagedIterable feedResponseIterator1 = + container.queryItems(query, CosmosQueryRequestOptions, Pojo.class); + // Very basic validation + assertThat(feedResponseIterator1.iterator().hasNext()).isTrue(); + + SqlQuerySpec querySpec = new SqlQuerySpec(query); + CosmosPagedIterable feedResponseIterator3 = + container.queryItems(querySpec, CosmosQueryRequestOptions, Pojo.class); + assertThat(feedResponseIterator3.iterator().hasNext()).isTrue(); + } + + @Test(groups = {"emulator"}, timeOut = TIMEOUT) + public void queryItemsWithContinuationTokenAndPageSize() throws Exception { + List actualIds = new ArrayList<>(); + Pojo properties = getItem(UUID.randomUUID().toString()); + container.createItem(properties); + actualIds.add(properties.id); + properties = getItem(UUID.randomUUID().toString()); + container.createItem(properties); + actualIds.add(properties.id); + properties = getItem(UUID.randomUUID().toString()); + container.createItem(properties); + actualIds.add(properties.id); + + + String query = String.format("SELECT * from c where c.id in ('%s', '%s', '%s')", actualIds.get(0), actualIds.get(1), actualIds.get(2)); + CosmosQueryRequestOptions CosmosQueryRequestOptions = new CosmosQueryRequestOptions(); + String continuationToken = null; + int pageSize = 1; + + int initialDocumentCount = 3; + int finalDocumentCount = 0; + + CosmosPagedIterable feedResponseIterator1 = + container.queryItems(query, CosmosQueryRequestOptions, Pojo.class); + + do { + Iterable> feedResponseIterable = + feedResponseIterator1.iterableByPage(continuationToken, pageSize); + for (FeedResponse fr : feedResponseIterable) { + int resultSize = fr.getResults().size(); + assertThat(resultSize).isEqualTo(pageSize); + finalDocumentCount += fr.getResults().size(); + continuationToken = fr.getContinuationToken(); + } + } while (continuationToken != null); + + assertThat(finalDocumentCount).isEqualTo(initialDocumentCount); + + } + + + private Pojo getItem(String documentId) { + final String uuid = UUID.randomUUID().toString(); + + Pojo pojo = new Pojo(); + pojo.id = uuid; + pojo.mypk = uuid; + pojo.nonSensitive = UUID.randomUUID().toString(); + pojo.sensitive = UUID.randomUUID().toString(); + + return pojo; + } + + private void validateItemResponse(Pojo containerProperties, + CosmosItemResponse createResponse) { + // Basic validation + assertThat(BridgeInternal.getProperties(createResponse).getId()).isNotNull(); + assertThat(BridgeInternal.getProperties(createResponse).getId()) + .as("check Resource Id") + .isEqualTo(containerProperties.id); + } + + public static class Pojo { + public String id; + @JsonProperty + public String mypk; + @JsonProperty + public String sensitive; + @JsonProperty + public String nonSensitive; + } + + private DataEncryptionKey createDataEncryptionKey() throws Exception { + return TestUtils.createDataEncryptionKey(); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/TestNGLogListener.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/TestNGLogListener.java new file mode 100644 index 000000000000..f6ccfd2d2a2d --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/TestNGLogListener.java @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +package com.azure.cosmos; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.IInvokedMethod; +import org.testng.IInvokedMethodListener; +import org.testng.ITestResult; +import org.testng.SkipException; + +public class TestNGLogListener implements IInvokedMethodListener { + private final Logger logger = LoggerFactory.getLogger(TestNGLogListener.class); + + @Override + public void beforeInvocation(IInvokedMethod iInvokedMethod, ITestResult iTestResult) { + logger.info("beforeInvocation: {}", methodName(iInvokedMethod)); + } + + @Override + public void afterInvocation(IInvokedMethod iInvokedMethod, ITestResult iTestResult) { + logger.info("afterInvocation: {}, total time {}ms, result {}", + methodName(iInvokedMethod), + iTestResult.getEndMillis() - iTestResult.getStartMillis(), + resultDetails(iTestResult) + ); + } + + private String resultDetails(ITestResult iTestResult) { + if (iTestResult.isSuccess()) { + return "success"; + } + + if (iTestResult.getThrowable() instanceof SkipException) { + return "skipped. reason: " + failureDetails(iTestResult); + } + + return "failed. reason: " + failureDetails(iTestResult); + } + + private String failureDetails(ITestResult iTestResult) { + if (iTestResult.isSuccess()) { + return null; + } + + if (iTestResult.getThrowable() == null) { + logger.error("throwable is null"); + return null; + } + + return iTestResult.getThrowable().getClass().getName() + ": " + iTestResult.getThrowable().getMessage(); + } + + private String methodName(IInvokedMethod iInvokedMethod) { + return iInvokedMethod.getTestMethod().getRealClass().getSimpleName() + "#" + iInvokedMethod.getTestMethod().getMethodName(); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/AesCryptoServiceProviderTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/AesCryptoServiceProviderTest.java new file mode 100644 index 000000000000..71fee35dafd9 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/AesCryptoServiceProviderTest.java @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.encryption.api.EncryptionType; +import org.apache.commons.lang3.RandomStringUtils; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; + +public class AesCryptoServiceProviderTest { + private byte[] key; + + @BeforeClass(groups = "unit") + public void beforeClass() throws Exception { + key = TestUtils.generatePBEKeySpec("myPassword"); + } + + @Test(groups = "unit", dataProvider = "encryptionInput") + public void aesEncryptThenDecrypt(byte[] input) { + AeadAes256CbcHmac256EncryptionKey aeadAesKey = new AeadAes256CbcHmac256EncryptionKey(key, "AES"); + AeadAes256CbcHmac256Algorithm encryptionAlgorithm = new AeadAes256CbcHmac256Algorithm(aeadAesKey, EncryptionType.RANDOMIZED, (byte) 0x01); + byte[] encrypted = encryptionAlgorithm.encryptData(input); + + assertThat(encrypted).isNotEqualTo(input); + assertThat(encrypted.length).isGreaterThan(input.length); + + byte[] decrypted = encryptionAlgorithm.decryptData(encrypted); + assertThat(decrypted).isEqualTo(input); + } + + @DataProvider(name = "encryptionInput") + public Object[][] encryptionInput() { + return new Object[][]{ + { new byte[] {} }, + {"secret".getBytes(StandardCharsets.UTF_8) }, + {"محرمانه".getBytes(StandardCharsets.UTF_8) }, + { RandomStringUtils.randomAlphabetic(100_000).getBytes(StandardCharsets.UTF_8) } + }; + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyPropertiesTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyPropertiesTest.java new file mode 100644 index 000000000000..8a40f7775656 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/DataEncryptionKeyPropertiesTest.java @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.apachecommons.lang.RandomStringUtils; +import com.azure.cosmos.implementation.apachecommons.lang.RandomUtils; +import com.azure.cosmos.implementation.encryption.api.CosmosEncryptionAlgorithm; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey; +import com.azure.cosmos.implementation.encryption.api.EncryptionOptions; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class DataEncryptionKeyPropertiesTest { + + @Test(groups = "unit") + public void sameSerializationAsDotNet() throws Exception { + byte[] bytes = TestUtils.getResourceAsByteArray("./encryption/dotnet/DataEncryptionKeyProperties.json"); + + ObjectMapper objectMapper = new ObjectMapper(); + ObjectNode objectNode = objectMapper.readValue(bytes, ObjectNode.class); + + DataEncryptionKeyProperties dataEncryptionKeyProperties = Utils.getSimpleObjectMapper().readValue(bytes, DataEncryptionKeyProperties.class); + objectNode.remove("_attachments"); + String expected = objectMapper.writeValueAsString(objectNode); + String actual = objectMapper.writeValueAsString(dataEncryptionKeyProperties); + assertThat(actual).isEqualTo(expected); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/DecryptDataEncryptedByDotNetTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/DecryptDataEncryptedByDotNetTest.java new file mode 100644 index 000000000000..f815acc58aa1 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/DecryptDataEncryptedByDotNetTest.java @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.CosmosAsyncClient; +import com.azure.cosmos.CosmosAsyncContainer; +import com.azure.cosmos.CosmosAsyncDatabase; +import com.azure.cosmos.CosmosBridgeInternal; +import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.implementation.DatabaseForTest; +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; +import com.azure.cosmos.implementation.encryption.api.CosmosEncryptionAlgorithm; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey; +import com.azure.cosmos.implementation.guava25.collect.Streams; +import com.azure.cosmos.models.PartitionKey; +import com.azure.cosmos.models.ThroughputProperties; +import com.azure.cosmos.rx.TestSuiteBase; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; + +import java.time.Duration; +import java.util.Objects; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; + +public class DecryptDataEncryptedByDotNetTest extends TestSuiteBase { + + private final static EncryptionKeyWrapMetadata METADATA_1 = new EncryptionKeyWrapMetadata("metadata1"); + private final static EncryptionKeyWrapMetadata METADATA_2 = new EncryptionKeyWrapMetadata("metadata2"); + private final static String METADATA_UPDATE_SUFFIX = "updated"; + private final static Duration CACHE_TTL = Duration.ofDays(1); + + private final String databaseForTestId = DatabaseForTest.generateId(); + private final String itemContainerId = UUID.randomUUID().toString(); + private final String keyContainerId = UUID.randomUUID().toString(); + private CosmosAsyncClient client; + private CosmosAsyncDatabase databaseCore; + private CosmosAsyncContainer itemContainer; + private CosmosAsyncContainer keyContainer; + private CosmosDataEncryptionKeyProvider dekProvider; + private TestKeyWrapProvider keyWrapProvider; + + @Factory(dataProvider = "clientBuilders") + public DecryptDataEncryptedByDotNetTest(CosmosClientBuilder clientBuilder) { + super(clientBuilder); + } + + @BeforeClass(groups = "emulator") + public void beforeClass() { + TestUtils.initialized(); + } + + @AfterClass(groups = "emulator") + public void afterClass() { + safeDeleteDatabase(databaseCore); + } + + @AfterMethod(groups = "emulator") + public void afterMethod() { + safeClose(client); + } + + @BeforeMethod(groups = "emulator") + public void beforeMethod() { + keyWrapProvider = new TestKeyWrapProvider(); + dekProvider = new CosmosDataEncryptionKeyProvider(keyWrapProvider); + client = CosmosBridgeInternal.setDateKeyProvider(getClientBuilder(), dekProvider).buildAsyncClient(); + client.createDatabaseIfNotExists(databaseForTestId).block(); + databaseCore = client.getDatabase(databaseForTestId); + databaseCore.createContainerIfNotExists(keyContainerId, "/id", ThroughputProperties.createManualThroughput(400)).block(); + keyContainer = databaseCore.getContainer(keyContainerId); + databaseCore.createContainerIfNotExists(itemContainerId, "/PK", ThroughputProperties.createManualThroughput(400)).block(); + itemContainer = databaseCore.getContainer(itemContainerId); + + truncateCollection(itemContainer); + truncateCollection(keyContainer); + + dekProvider.initialize(databaseCore, keyContainer.getId()); + } + + @Test(groups = "emulator") + public void canReadKeyEncryptionKeyGeneratedByDotNet() throws Exception { + // add key generated by dotnet + ObjectNode dataEncryptionKeyProperties = TestUtils.loadPojo("./encryption/dotnet/DataEncryptionKeyProperties.json", ObjectNode.class); + keyContainer.createItem(dataEncryptionKeyProperties).block(); + + DataEncryptionKey loadedKey = dekProvider.getDataEncryptionKey(dataEncryptionKeyProperties.get("id").asText(), CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized); + + assertThat(loadedKey.getEncryptionAlgorithm()).isEqualTo(CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized); + assertThat(loadedKey.getEncryptionAlgorithm()).isEqualTo(CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized); + + EncryptionKeyWrapMetadata keyWrapMetadata = Utils.getSimpleObjectMapper().convertValue(dataEncryptionKeyProperties.get("keyWrapMetadata"), EncryptionKeyWrapMetadata.class); + byte[] expectedWrappedKey = dataEncryptionKeyProperties.get("wrappedDataEncryptionKey").binaryValue(); + EncryptionKeyUnwrapResult expectedUnWrappedKey = keyWrapProvider.unwrapKey(expectedWrappedKey, keyWrapMetadata); + + assertThat(loadedKey.getRawKey()).isEqualTo(expectedUnWrappedKey.getDataEncryptionKey()); + } + + @Test(groups = "emulator") + public void canDecryptDataEncryptedByDotNet() throws Exception { + // add key generated by dotnet + DataEncryptionKeyProperties dataEncryptionKeyProperties = TestUtils.loadPojo("./encryption/dotnet/DataEncryptionKeyProperties.json", DataEncryptionKeyProperties.class); + keyContainer.createItem(dataEncryptionKeyProperties).block(); + + // add data encrypted by dotnet + ObjectNode objectNode = TestUtils.loadPojo("./encryption/dotnet/EncryptedPOCO.json", ObjectNode.class); + Streams.stream(objectNode.fieldNames()).filter(fieldName -> fieldName.startsWith("_") && !fieldName.equals("_ei")).collect(Collectors.toList()).forEach(filedName -> objectNode.remove(filedName)); + itemContainer.createItem(objectNode).block(); + TestDoc expectedTestDoc = TestUtils.loadPojo("./encryption/dotnet/POCO.json", TestDoc.class); + assertThat(expectedTestDoc.sensitive).isNotNull(); + + TestDoc testDoc = itemContainer.readItem(objectNode.get("id").asText(), new PartitionKey(objectNode.get("PK").asText()), TestDoc.class).block().getItem(); + assertThat(testDoc.sensitive).isNotNull(); + assertThat(testDoc).isEqualTo(expectedTestDoc); + } + + static public class TestDoc { + @JsonProperty("id") + public String id; + @JsonProperty("PK") + public String pk; + @JsonProperty("NonSensitive") + public String nonSensitive; + @JsonProperty("Sensitive") + public String sensitive; + + public TestDoc() { + } + + public TestDoc(TestDoc other) { + this.id = other.id; + this.pk = other.pk; + this.nonSensitive = other.nonSensitive; + this.sensitive = other.sensitive; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestDoc testDoc = (TestDoc) o; + return Objects.equals(id, testDoc.id) && + Objects.equals(pk, testDoc.pk) && + Objects.equals(nonSensitive, testDoc.nonSensitive) && + Objects.equals(sensitive, testDoc.sensitive); + } + + @Override + public int hashCode() { + return Objects.hash(id, pk, nonSensitive, sensitive); + } + } + + private class TestKeyWrapProvider implements EncryptionKeyWrapProvider { + public EncryptionKeyUnwrapResult unwrapKey(byte[] wrappedKey, EncryptionKeyWrapMetadata metadata) { + int moveBy = StringUtils.equals(metadata.value, DecryptDataEncryptedByDotNetTest.METADATA_1.value + DecryptDataEncryptedByDotNetTest.METADATA_UPDATE_SUFFIX) ? 1 : 2; + + for (int i = 0; i < wrappedKey.length; i++) { + wrappedKey[i] = (byte) (wrappedKey[i] - moveBy); + } + + return new EncryptionKeyUnwrapResult(wrappedKey, CACHE_TTL); + } + + public EncryptionKeyWrapResult wrapKey(byte[] key, EncryptionKeyWrapMetadata metadata) { + EncryptionKeyWrapMetadata responseMetadata = new EncryptionKeyWrapMetadata(metadata.value + DecryptDataEncryptedByDotNetTest.METADATA_UPDATE_SUFFIX); + int moveBy = StringUtils.equals(metadata.value, DecryptDataEncryptedByDotNetTest.METADATA_1.value) ? 1 : 2; + + for (int i = 0; i < key.length; i++) { + key[i] = (byte) (key[i] + moveBy); + } + + return new EncryptionKeyWrapResult(key, responseMetadata); + } + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionProcessorTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionProcessorTest.java new file mode 100644 index 000000000000..18bdf0d6e3b5 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionProcessorTest.java @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.encryption.api.CosmosEncryptionAlgorithm; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey; +import com.azure.cosmos.implementation.encryption.api.EncryptionOptions; +import com.azure.cosmos.implementation.encryption.api.EncryptionType; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableList; +import org.apache.commons.lang3.RandomStringUtils; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EncryptionProcessorTest { + public EncryptionProcessorTest() {} + private byte[] key; + + @BeforeClass(groups = "unit") + public void beforeClass() throws Exception { + key = TestUtils.generatePBEKeySpec("myPassword"); + } + + public static class TestPojo { + @JsonProperty + public String id; + @JsonProperty + public String pk; + @JsonProperty + public String nonSensitive; + @JsonProperty + public String sensitive; + } + + private TestPojo getTestDate() { + + TestPojo test = new TestPojo(); + test.id = UUID.randomUUID().toString(); + test.pk = UUID.randomUUID().toString(); + test.nonSensitive = UUID.randomUUID().toString(); + test.sensitive = UUID.randomUUID().toString(); + + return test; + } + + @Test(groups = "unit") + public void aesEncryptThenDecrypt() { + AeadAes256CbcHmac256EncryptionKey aeadAesKey = new AeadAes256CbcHmac256EncryptionKey(key, "AES"); + AeadAes256CbcHmac256Algorithm encryptionAlgorithm = new AeadAes256CbcHmac256Algorithm(aeadAesKey, EncryptionType.RANDOMIZED, (byte) 0x01); + String keyId = UUID.randomUUID().toString(); + + DataEncryptionKey javaDataEncryptionKey = new DataEncryptionKey() { + @Override + public byte[] getRawKey() { + return key; + } + + @Override + public String getEncryptionAlgorithm() { + return CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized; + } + + @Override + public byte[] encryptData(byte[] plainText) { + return encryptionAlgorithm.encryptData(plainText); + } + + @Override + public byte[] decryptData(byte[] cipherText) { + return encryptionAlgorithm.decryptData(cipherText); + } + }; + + SimpleInMemoryProvider keyProvider = new SimpleInMemoryProvider(); + keyProvider.addKey(keyId, javaDataEncryptionKey); + + EncryptionProcessor encryptionProcessor = new EncryptionProcessor(); + com.azure.cosmos.implementation.encryption.api.EncryptionOptions encryptionOptions = new EncryptionOptions(); + encryptionOptions.setPathsToEncrypt(ImmutableList.of("/sensitive")); + encryptionOptions.setDataEncryptionKeyId(keyId); + encryptionOptions.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized); + + TestPojo testDate = getTestDate(); + byte[] inputAsByteArray = toByteArray(testDate); + + ObjectNode objectNode = Utils.getSimpleObjectMapper().convertValue(testDate, ObjectNode.class); + ObjectNode itemObjectWithEncryptedSensitiveData = encryptionProcessor.encryptAsync(objectNode, encryptionOptions, keyProvider); + ObjectNode itemObjectWithDecryptedSensitiveData = encryptionProcessor.decryptAsync(itemObjectWithEncryptedSensitiveData, keyProvider); + + assertThat(serializeToByteArray(Utils.getSimpleObjectMapper(), itemObjectWithDecryptedSensitiveData)).isEqualTo(inputAsByteArray); + } + + @DataProvider(name = "encryptionInput") + public Object[][] encryptionInput() { + return new Object[][]{ + { new byte[] {} }, + {"secret".getBytes(StandardCharsets.UTF_8) }, + {"محرمانه".getBytes(StandardCharsets.UTF_8) }, + { RandomStringUtils.randomAlphabetic(100_000).getBytes(StandardCharsets.UTF_8) } + }; + } + + private static byte[] toByteArray(Object object) { + return serializeToByteArray(Utils.getSimpleObjectMapper(), object); + } + + public static byte[] serializeToByteArray(ObjectMapper mapper, Object object) { + try { + return mapper.writeValueAsBytes(object); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Unable to convert JSON to byte[]", e); + } + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionPropertiesTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionPropertiesTest.java new file mode 100644 index 000000000000..6111575fb675 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionPropertiesTest.java @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.apachecommons.lang.RandomStringUtils; +import com.azure.cosmos.implementation.apachecommons.lang.RandomUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.testng.annotations.Test; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EncryptionPropertiesTest { + + @Test(groups = "unit") + public void fromJsonNode() throws Exception { + ObjectMapper objectMapper = new ObjectMapper(); + ObjectNode objectNode = objectMapper.createObjectNode(); + + objectNode.put("_ef", 1); + objectNode.put("_ea", "myAlgo"); + objectNode.put("_en", "keyId"); + objectNode.put("_ed", "AwQ="); + + EncryptionProperties encryptionProperties = EncryptionProperties.fromObjectNode(objectNode); + + assertThat(encryptionProperties.getEncryptionFormatVersion()).isEqualTo(1); + assertThat(encryptionProperties.getDataEncryptionKeyId()).isEqualTo("keyId"); + assertThat(encryptionProperties.getEncryptionAlgorithm()).isEqualTo("myAlgo"); + assertThat(encryptionProperties.getEncryptedData()).isEqualTo(new byte[]{3, 4}); + } + + @Test(groups = "unit") + public void toJsonNode() throws Exception { + EncryptionProperties encryptionProperties = new EncryptionProperties(1, "myAlgo", "keyId", new byte[]{3, 4}); + ObjectNode objectNode = encryptionProperties.toObjectNode(); + + assertThat(objectNode.get("_ef").isInt()).isTrue(); + assertThat(objectNode.get("_ef").asInt()).isEqualTo(1); + + assertThat(objectNode.get("_en").isTextual()).isTrue(); + assertThat(objectNode.get("_en").asText()).isEqualTo("keyId"); + + + assertThat(objectNode.get("_ea").isTextual()).isTrue(); + assertThat(objectNode.get("_ea").asText()).isEqualTo("myAlgo"); + + + assertThat(objectNode.get("_ed").isBinary()).isTrue(); + assertThat(objectNode.get("_ed").binaryValue()).isEqualTo(new byte[] {3, 4}); + + assertThat(objectNode.fieldNames()).hasSize(4); + } + + @Test(groups = "unit") + public void serialize() throws Exception { + EncryptionProperties encryptionProperties = new EncryptionProperties(1, "myAlgo", "2", new byte[]{3, 4}); + String encryptionPropertiesAsString = EncryptionProperties.getObjectWriter().writeValueAsString(encryptionProperties); + + assertThat(encryptionPropertiesAsString).isEqualTo("{\"_ef\":1,\"_ea\":\"myAlgo\",\"_en\":\"2\",\"_ed\":\"AwQ=\"}"); + } + + @Test(groups = "unit") + public void deserialize() throws Exception { + EncryptionProperties parsedEncryptionProperties = EncryptionProperties.getObjectReader().readValue("{\"_ef\":1,\"_en\":\"2\",\"_ea\":\"myAlgo\",\"_ed\":\"AwQ=\"}"); + + assertThat(parsedEncryptionProperties.getEncryptionFormatVersion()).isEqualTo(1); + assertThat(parsedEncryptionProperties.getDataEncryptionKeyId()).isEqualTo("2"); + assertThat(parsedEncryptionProperties.getEncryptedData()).isEqualTo(new byte[]{3, 4}); + } + + @Test(groups = "unit") + public void e2e() throws Exception { + EncryptionProperties encryptionProperties = new EncryptionProperties( + RandomUtils.nextInt(), + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + RandomStringUtils.randomAlphabetic(10).getBytes(StandardCharsets.UTF_8)); + + byte[] encryptionPropertiesAsBytes = EncryptionProperties.getObjectWriter().writeValueAsBytes(encryptionProperties); + EncryptionProperties parsedEncryptionProperties = EncryptionProperties.getObjectReader().readValue(encryptionPropertiesAsBytes); + + assertThat(parsedEncryptionProperties.getEncryptionFormatVersion()).isEqualTo(encryptionProperties.getEncryptionFormatVersion()); + assertThat(parsedEncryptionProperties.getDataEncryptionKeyId()).isEqualTo(encryptionProperties.getDataEncryptionKeyId()); + assertThat(parsedEncryptionProperties.getEncryptedData()).isEqualTo(encryptionProperties.getEncryptedData()); + } +} + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionTests.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionTests.java new file mode 100644 index 000000000000..bc7590e22ac1 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionTests.java @@ -0,0 +1,261 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.CosmosAsyncClient; +import com.azure.cosmos.CosmosAsyncContainer; +import com.azure.cosmos.CosmosAsyncDatabase; +import com.azure.cosmos.CosmosBridgeInternal; +import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.implementation.DatabaseForTest; +import com.azure.cosmos.implementation.HttpConstants; +import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; +import com.azure.cosmos.implementation.encryption.api.CosmosEncryptionAlgorithm; +import com.azure.cosmos.implementation.encryption.api.EncryptionOptions; +import com.azure.cosmos.implementation.guava25.collect.ImmutableList; +import com.azure.cosmos.models.CosmosItemResponse; +import com.azure.cosmos.models.CosmosItemRequestOptions; +import com.azure.cosmos.models.ModelBridgeInternal; +import com.azure.cosmos.models.PartitionKey; +import com.azure.cosmos.models.ThroughputProperties; +import com.azure.cosmos.rx.TestSuiteBase; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Factory; +import org.testng.annotations.Test; + +import java.time.Duration; +import java.util.Objects; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EncryptionTests extends TestSuiteBase { + + private static EncryptionKeyWrapMetadata metadata1 = new EncryptionKeyWrapMetadata("metadata1"); + private static EncryptionKeyWrapMetadata metadata2 = new EncryptionKeyWrapMetadata("metadata2"); + private final static String metadataUpdateSuffix = "updated"; + private static Duration cacheTTL = Duration.ofDays(1); + + private final String databaseForTestId = DatabaseForTest.generateId(); + private final String itemContainerId = UUID.randomUUID().toString(); + private final String keyContainerId = UUID.randomUUID().toString(); + + private final String dekId = "mydek"; + + private static CosmosAsyncClient client; + + private static CosmosAsyncDatabase databaseCore; + private static DataEncryptionKeyProperties dekProperties; + // private static ContainerCore itemContainerCore; + private static CosmosAsyncContainer itemContainer; + private static CosmosAsyncContainer keyContainer; + private static CosmosDataEncryptionKeyProvider dekProvider; +// private static TestEncryptor encryptor; + + @Factory(dataProvider = "clientBuilders") + public EncryptionTests(CosmosClientBuilder clientBuilder) { + super(clientBuilder); + } + + @BeforeTest(groups = {"emulator"}) + public void beforeTest() { + TestKeyWrapProvider keyWrapProvider = new TestKeyWrapProvider(); + dekProvider = new CosmosDataEncryptionKeyProvider(keyWrapProvider); + client = CosmosBridgeInternal.setDateKeyProvider(getClientBuilder(), dekProvider).buildAsyncClient(); + +// EncryptionTests.encryptor = new TestEncryptor(EncryptionTests.dekProvider); + +// EncryptionTests.client = EncryptionTests.GetClient(EncryptionTests.encryptor); + + client.createDatabaseIfNotExists(databaseForTestId).block(); + databaseCore = client.getDatabase(databaseForTestId); + databaseCore.createContainerIfNotExists(keyContainerId, "/id", ThroughputProperties.createManualThroughput(400)).block(); + keyContainer = databaseCore.getContainer(keyContainerId); + databaseCore.createContainerIfNotExists(itemContainerId, "/PK", ThroughputProperties.createManualThroughput(400)).block(); + itemContainer = databaseCore.getContainer(itemContainerId); + + dekProvider.initialize(databaseCore, EncryptionTests.keyContainer.getId()); + + EncryptionTests.dekProperties = EncryptionTests.createDek(EncryptionTests.dekProvider, dekId); + } + + @BeforeClass(groups = {"emulator"}) + public void beforeClass() { + TestUtils.initialized(); + client = getClientBuilder().buildAsyncClient(); + } + + @AfterMethod(groups = {"emulator"}) + public void afterTest() { + safeClose(client); + } + + @AfterClass(groups = {"emulator"}) + public void afterClass() { + safeDeleteDatabase(databaseCore); + } + + static public class TestDoc { + + @JsonProperty("id") + public String id; + @JsonProperty("PK") + public String pk; + @JsonProperty("NonSensitive") + public String nonSensitive; + @JsonProperty("Sensitive") + public String sensitive; + + public TestDoc() { + } + + public TestDoc(TestDoc other) { + this.id = other.id; + this.pk = other.pk; + this.nonSensitive = other.nonSensitive; + this.sensitive = other.sensitive; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestDoc testDoc = (TestDoc) o; + return Objects.equals(id, testDoc.id) && + Objects.equals(pk, testDoc.pk) && + Objects.equals(nonSensitive, testDoc.nonSensitive) && + Objects.equals(sensitive, testDoc.sensitive); + } + + @Override + public int hashCode() { + return Objects.hash(id, pk, nonSensitive, sensitive); + } + } + + @Test(groups = {"emulator"}) + public void encryptionCreateDek() { + String dekId = "anotherDek"; + DataEncryptionKeyProperties dekProperties = EncryptionTests.createDek(EncryptionTests.dekProvider, dekId); + + assertThat(dekProperties).isNotNull(); + assertThat(dekProperties.createdTime).isNotNull(); + assertThat(dekProperties.lastModified).isNotNull(); + assertThat(dekProperties.selfLink).isNotNull(); + assertThat(dekProperties).isNotNull(); + + assertThat(dekProperties.resourceId).isNotNull(); + + assertThat(dekProperties.lastModified).isEqualTo(dekProperties.lastModified); + + assertThat( + new EncryptionKeyWrapMetadata(EncryptionTests.metadata1.value + EncryptionTests.metadataUpdateSuffix)).isEqualTo( + dekProperties.encryptionKeyWrapMetadata); + + // Use different DEK provider to avoid (unintentional) cache impact + CosmosDataEncryptionKeyProvider dekProvider = new CosmosDataEncryptionKeyProvider(new TestKeyWrapProvider()); + + + dekProvider.initialize(databaseCore, EncryptionTests.keyContainer.getId()); + + DataEncryptionKeyProperties readProperties = dekProvider.getDataEncryptionKeyContainer().readDataEncryptionKeyAsync(dekId, null).block().getItem(); + assertThat(dekProperties).isEqualTo(readProperties); + } + + @Test(groups = {"emulator"}, timeOut = TIMEOUT) + public void createItemEncrypt_readItemDecrypt() throws Exception { + CosmosItemRequestOptions requestOptions = new CosmosItemRequestOptions(); + EncryptionOptions encryptionOptions = new EncryptionOptions(); + encryptionOptions.setPathsToEncrypt(ImmutableList.of("/Sensitive")); + + encryptionOptions.setDataEncryptionKeyId(dekId); + encryptionOptions.setEncryptionAlgorithm(CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized); + ModelBridgeInternal.setEncryptionOptions(requestOptions, encryptionOptions); + + TestDoc properties = getItem(UUID.randomUUID().toString()); + CosmosItemResponse itemResponse = itemContainer.createItem(properties, requestOptions).block(); + assertThat(itemResponse.getRequestCharge()).isGreaterThan(0); + + TestDoc responseItem = itemResponse.getItem(); + validateWriteResponseIsValid(properties, responseItem); + + TestDoc readItem = itemContainer.readItem(properties.id, new PartitionKey(properties.pk), requestOptions, TestDoc.class).block().getItem(); + validateReadResponseIsValid(properties, readItem); + } + + private void validateWriteResponseIsValid(TestDoc originalItem, TestDoc result) { + assertThat(result.sensitive).isEqualTo(originalItem.sensitive); + assertThat(result.id).isEqualTo(originalItem.id); + assertThat(result.pk).isEqualTo(originalItem.pk); + assertThat(result.nonSensitive).isEqualTo(originalItem.nonSensitive); + } + + private void validateReadResponseIsValid(TestDoc originalItem, TestDoc result) { + assertThat(result.id).isEqualTo(originalItem.id); + assertThat(result.pk).isEqualTo(originalItem.pk); + assertThat(result.nonSensitive).isEqualTo(originalItem.nonSensitive); + assertThat(result.sensitive).isEqualTo(originalItem.sensitive); + } + + private void validateQueryResponseIsValid(TestDoc originalItem, TestDoc result) { + assertThat(result.id).isEqualTo(originalItem.id); + assertThat(result.pk).isEqualTo(originalItem.pk); + assertThat(result.nonSensitive).isEqualTo(originalItem.nonSensitive); + assertThat(result.sensitive).isNull(); + } + + private TestDoc getItem(String documentId) { + final String uuid = UUID.randomUUID().toString(); + + TestDoc pojo = new TestDoc(); + pojo.id = uuid; + pojo.pk = uuid; + pojo.nonSensitive = UUID.randomUUID().toString(); + pojo.sensitive = UUID.randomUUID().toString(); + + return pojo; + } + + private static DataEncryptionKeyProperties createDek(CosmosDataEncryptionKeyProvider dekProvider, String dekId) { + CosmosItemResponse dekResponse = dekProvider.getDataEncryptionKeyContainer().createDataEncryptionKeyAsync( + dekId, + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized, + EncryptionTests.metadata1, null).block(); + + assertThat(dekResponse.getRequestCharge()).isGreaterThan(0); + assertThat(dekResponse.getResponseHeaders().get(HttpConstants.HttpHeaders.E_TAG)).isNotNull(); + + DataEncryptionKeyProperties dekProperties = dekResponse.getItem(); + assertThat(dekResponse.getResponseHeaders().get(HttpConstants.HttpHeaders.E_TAG)).isEqualTo(dekProperties.eTag); + assertThat(dekId).isEqualTo(dekProperties.id); + return dekProperties; + } + + private class TestKeyWrapProvider implements EncryptionKeyWrapProvider { + public EncryptionKeyUnwrapResult unwrapKey(byte[] wrappedKey, EncryptionKeyWrapMetadata metadata) { + int moveBy = StringUtils.equals(metadata.value, EncryptionTests.metadata1.value + EncryptionTests.metadataUpdateSuffix) ? 1 : 2; + + for (int i = 0; i < wrappedKey.length; i++) { + wrappedKey[i] = (byte) (wrappedKey[i] - moveBy); + } + + return new EncryptionKeyUnwrapResult(wrappedKey, EncryptionTests.cacheTTL); + } + + public EncryptionKeyWrapResult wrapKey(byte[] key, EncryptionKeyWrapMetadata metadata) { + EncryptionKeyWrapMetadata responseMetadata = new EncryptionKeyWrapMetadata(metadata.value + EncryptionTests.metadataUpdateSuffix); + int moveBy = StringUtils.equals(metadata.value, EncryptionTests.metadata1.value) ? 1 : 2; + + for (int i = 0; i < key.length; i++) { + key[i] = (byte) (key[i] + moveBy); + } + + return new EncryptionKeyWrapResult(key, responseMetadata); + } + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/SecurityUtilityTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/SecurityUtilityTest.java new file mode 100644 index 000000000000..f2e8102b52d2 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/SecurityUtilityTest.java @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.google.common.io.BaseEncoding; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SecurityUtilityTest { + byte[] secretSha256Key; + + @BeforeClass(groups = "unit") + public void beforeClass() throws Exception { + secretSha256Key = generateKey(); + } + + @Test(groups = "unit", dataProvider = "sha256KeyHashInputProvider") + public void getHMACWithSHA256(byte[] input, String expectedHashAsHex) throws Exception { + byte[] expectedHash = hexToByteArray(expectedHashAsHex); + byte[] output = new byte[expectedHashAsHex.length() / 2]; + + SecurityUtility.getHMACWithSHA256(input, secretSha256Key, output); + assertThat(output).isEqualTo(expectedHash); + } + + @Test(groups = "unit", dataProvider = "sha256InputProvider") + public void getSHA256Hash(byte[] input, String expectedHash) throws Exception { + String hash = SecurityUtility.getSHA256Hash(input); + assertThat(hash).isEqualTo(expectedHash); + } + + @Test(groups = "unit") + public void generateRandomBytes() { + ShanonEntropyGauge entropy = new ShanonEntropyGauge(); + int numberOfRandomBytes = 2; + byte[] output = new byte[numberOfRandomBytes]; + + for (int i = 0; i < 1_000_000; i++) { + SecurityUtility.generateRandomBytes(output); + entropy.add(output); + } + + double entropyValue = entropy.calculate(); + + // a smoke test validating shannon entropy + assertThat(entropyValue).isGreaterThan(numberOfRandomBytes * 8 - 1); + assertThat(entropyValue).isLessThan(numberOfRandomBytes * 8); + } + + @DataProvider(name = "sha256KeyHashInputProvider") + public Object[][] sha256KeyHashInputProvider() { + return new Object[][]{ + // byte array plain text, hex of hmac byte array + {"".getBytes(StandardCharsets.UTF_8), "BE360B09127711ED4E"}, + {"test".getBytes(StandardCharsets.UTF_8), "9BC3AF5A"}, + {"تست".getBytes(StandardCharsets.UTF_8), "1A66670142D26FAE72"}, + }; + } + + @DataProvider(name = "sha256InputProvider") + public Object[][] sha256InputProvider() { + return new Object[][]{ + {"".getBytes(StandardCharsets.UTF_8), "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"}, + {"good morning".getBytes(StandardCharsets.UTF_8), "CDF71DEA7D7741A2B6F021F3DD344F75C8333988F547866A8FBF28F064CF7C78"}, + {"صبح بخیر".getBytes(StandardCharsets.UTF_8), "5D504A5D6F1946209EDD3B99FB52AB77A60310675C8FDE03726112796893A650"}, + {"शुभ प्रभात".getBytes(StandardCharsets.UTF_8), "C033E4FD4691F0AD6422FB2B91BB64644C6F56D962DF206D60A0A1FEF23297F8"}, + }; + } + + private static byte[] generateKey() throws Exception { + final SecretKeySpec keyspec = new javax.crypto.spec.SecretKeySpec("کلید".getBytes(StandardCharsets.UTF_8), "HmacSHA256"); + return keyspec.getEncoded(); + } + + private static byte[] hexToByteArray(String hex) { + return BaseEncoding.base16().decode(hex); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/ShanonEntropyGauge.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/ShanonEntropyGauge.java new file mode 100644 index 000000000000..df4640cfc69a --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/ShanonEntropyGauge.java @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import java.math.BigInteger; +import java.util.HashMap; +import java.util.Map; + +public class ShanonEntropyGauge { + private final Map freq; + private int cnt = 0; + + public ShanonEntropyGauge() { + this.freq = new HashMap<>(); + } + + public void add(byte[] input) { + BigInteger number = new BigInteger(1, input); + freq.compute(number, (bigInteger, cnt) -> { + if (cnt == null) { + return 1; + } else { + return cnt + 1; + } + }); + cnt++; + } + + public double calculate() { + // compute Shannon entropy + double entropy = 0.0; + for (Map.Entry entry: freq.entrySet()) { + double p = 1.0 * entry.getValue() / cnt; + entropy -= p * Math.log(p) / Math.log(2); + } + + return entropy; + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/SimpleInMemoryProvider.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/SimpleInMemoryProvider.java new file mode 100644 index 000000000000..db045862a8bf --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/SimpleInMemoryProvider.java @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider; + +import java.util.HashMap; +import java.util.Map; + +public class SimpleInMemoryProvider implements DataEncryptionKeyProvider { + private final Map keyMap = new HashMap<>(); + + public void addKey(String keyId, DataEncryptionKey key) { + keyMap.put(keyId, key); + } + + @Override + public DataEncryptionKey getDataEncryptionKey(String id, String algorithm) { + return keyMap.get(id); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/TestUtils.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/TestUtils.java new file mode 100644 index 000000000000..ef1a1bc10c5f --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/TestUtils.java @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.encryption.api.CosmosEncryptionAlgorithm; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey; +import com.azure.cosmos.implementation.encryption.api.EncryptionType; +import com.google.common.io.BaseEncoding; +import com.google.common.io.ByteStreams; +import org.bouncycastle.jce.provider.BouncyCastleProvider; + +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.IOException; +import java.io.InputStream; +import java.security.NoSuchAlgorithmException; +import java.security.Security; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.KeySpec; +import java.util.Random; + +public class TestUtils { + static { + Security.addProvider(new BouncyCastleProvider()); + } + + public static void initialized() {} + + public static AeadAes256CbcHmac256EncryptionKey instantiateAeadAes256CbcHmac256EncryptionKey(byte[] key) { + return new AeadAes256CbcHmac256EncryptionKey(key, "AES"); + } + + public static AeadAes256CbcHmac256Algorithm instantiateAeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256EncryptionKey aeadAesKey, + EncryptionType encryptionType, + byte version) { + return new AeadAes256CbcHmac256Algorithm(aeadAesKey, EncryptionType.RANDOMIZED, version); + } + + private static byte[] hexToByteArray(String hex) { + return BaseEncoding.base16().decode(hex); + } + + public static byte[] generatePBEKeySpec(String password) throws NoSuchAlgorithmException, InvalidKeySpecException { + Random random = new Random(); + byte[] salt = new byte[16]; + random.nextBytes(salt); + KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, 65536, 256); // AES-256 + SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1"); + byte[] key = secretKeyFactory.generateSecret(spec).getEncoded(); + SecretKeySpec keySpec = new SecretKeySpec(key, "AES"); + return keySpec.getEncoded(); + } + + public static InputStream getResourceAsInputStream(String path) { + return TestUtils.class.getClassLoader().getResourceAsStream(path); + } + + public static byte[] getResourceAsByteArray(String path) { + try { + return ByteStreams.toByteArray(getResourceAsInputStream(path)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static T loadPojo(String path, Class classType) { + try { + return Utils.getSimpleObjectMapper().readValue(getResourceAsByteArray(path), classType); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static DataEncryptionKey createDataEncryptionKey() throws Exception { + byte[] key = TestUtils.generatePBEKeySpec("testPass"); + + AeadAes256CbcHmac256EncryptionKey aeadAesKey = TestUtils.instantiateAeadAes256CbcHmac256EncryptionKey(key); + AeadAes256CbcHmac256Algorithm encryptionAlgorithm = TestUtils.instantiateAeadAes256CbcHmac256Algorithm(aeadAesKey, EncryptionType.RANDOMIZED, (byte) 0x01); + DataEncryptionKey javaDataEncryptionKey = new DataEncryptionKey() { + + @Override + public byte[] getRawKey() { + return key; + } + + @Override + public String getEncryptionAlgorithm() { + return CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized; + } + + @Override + public byte[] encryptData(byte[] plainText) { + return encryptionAlgorithm.encryptData(plainText); + } + + @Override + public byte[] decryptData(byte[] cipherText) { + return encryptionAlgorithm.decryptData(cipherText); + } + }; + return javaDataEncryptionKey; + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/UnixTimestampSerializationTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/UnixTimestampSerializationTest.java new file mode 100644 index 000000000000..39d2258886e7 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/UnixTimestampSerializationTest.java @@ -0,0 +1,49 @@ +package com.azure.cosmos.implementation.encryption; + +import com.azure.cosmos.implementation.guava27.Strings; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.testng.annotations.Test; + +import java.time.Instant; +import java.util.Objects; + +import static org.assertj.core.api.Assertions.assertThat; + +public class UnixTimestampSerializationTest { + public static class TestDoc { + @JsonProperty("time") + @JsonSerialize(using = UnixTimestampSerializer.class) + @JsonDeserialize(using = UnixTimestampDeserializer.class) + public Instant time; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestDoc testDoc = (TestDoc) o; + return Objects.equals(time, testDoc.time); + } + + @Override + public int hashCode() { + return Objects.hash(time); + } + } + + @Test(groups = "unit") + public void serialization() throws Exception { + TestDoc testDoc = new TestDoc(); + int epochSeconds = 1587157090; + testDoc.time = Instant.ofEpochSecond(epochSeconds); + + ObjectMapper objectMapper = new ObjectMapper(); + String json = objectMapper.writeValueAsString(testDoc); + assertThat(json).isEqualTo(Strings.lenientFormat("{\"time\":%s}", epochSeconds)); + + TestDoc deserializedTestDoc = objectMapper.readValue(json, TestDoc.class); + assertThat(deserializedTestDoc).isEqualTo(testDoc); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java new file mode 100644 index 000000000000..f20cba90e7cf --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -0,0 +1,1123 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +package com.azure.cosmos.rx; + +import com.azure.core.credential.AzureKeyCredential; +import com.azure.cosmos.BridgeInternal; +import com.azure.cosmos.ConsistencyLevel; +import com.azure.cosmos.CosmosAsyncClient; +import com.azure.cosmos.CosmosAsyncClientTest; +import com.azure.cosmos.CosmosAsyncContainer; +import com.azure.cosmos.CosmosAsyncDatabase; +import com.azure.cosmos.CosmosAsyncUser; +import com.azure.cosmos.CosmosBridgeInternal; +import com.azure.cosmos.CosmosClient; +import com.azure.cosmos.CosmosClientBuilder; +import com.azure.cosmos.CosmosDatabase; +import com.azure.cosmos.CosmosDatabaseForTest; +import com.azure.cosmos.CosmosException; +import com.azure.cosmos.DirectConnectionConfig; +import com.azure.cosmos.GatewayConnectionConfig; +import com.azure.cosmos.TestNGLogListener; +import com.azure.cosmos.ThrottlingRetryOptions; +import com.azure.cosmos.implementation.Configs; +import com.azure.cosmos.implementation.ConnectionPolicy; +import com.azure.cosmos.implementation.InternalObjectNode; +import com.azure.cosmos.implementation.PathParser; +import com.azure.cosmos.implementation.TestConfigurations; +import com.azure.cosmos.implementation.Utils; +import com.azure.cosmos.implementation.directconnectivity.Protocol; +import com.azure.cosmos.implementation.guava25.base.CaseFormat; +import com.azure.cosmos.implementation.guava25.collect.ImmutableList; +import com.azure.cosmos.models.CompositePath; +import com.azure.cosmos.models.CompositePathSortOrder; +import com.azure.cosmos.models.CosmosContainerProperties; +import com.azure.cosmos.models.CosmosContainerRequestOptions; +import com.azure.cosmos.models.CosmosDatabaseProperties; +import com.azure.cosmos.models.CosmosDatabaseResponse; +import com.azure.cosmos.models.CosmosItemResponse; +import com.azure.cosmos.models.CosmosQueryRequestOptions; +import com.azure.cosmos.models.CosmosStoredProcedureRequestOptions; +import com.azure.cosmos.models.CosmosUserProperties; +import com.azure.cosmos.models.CosmosUserResponse; +import com.azure.cosmos.models.IncludedPath; +import com.azure.cosmos.models.IndexingPolicy; +import com.azure.cosmos.models.ModelBridgeInternal; +import com.azure.cosmos.models.PartitionKey; +import com.azure.cosmos.models.PartitionKeyDefinition; +import com.azure.cosmos.models.SqlQuerySpec; +import com.azure.cosmos.models.ThroughputProperties; +import com.azure.cosmos.util.CosmosPagedFlux; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.mockito.stubbing.Answer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterSuite; +import org.testng.annotations.BeforeSuite; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Listeners; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static com.azure.cosmos.BridgeInternal.extractConfigs; +import static com.azure.cosmos.BridgeInternal.injectConfigs; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; + +@Listeners({TestNGLogListener.class}) +public class TestSuiteBase extends CosmosAsyncClientTest { + + private static final int DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL = 500; + private static final ObjectMapper objectMapper = new ObjectMapper(); + + protected static Logger logger = LoggerFactory.getLogger(TestSuiteBase.class.getSimpleName()); + protected static final int TIMEOUT = 40000; + protected static final int FEED_TIMEOUT = 40000; + protected static final int SETUP_TIMEOUT = 60000; + protected static final int SHUTDOWN_TIMEOUT = 24000; + + protected static final int SUITE_SETUP_TIMEOUT = 120000; + protected static final int SUITE_SHUTDOWN_TIMEOUT = 60000; + + protected static final int WAIT_REPLICA_CATCH_UP_IN_MILLIS = 4000; + + protected final static ConsistencyLevel accountConsistency; + protected static final ImmutableList preferredLocations; + private static final ImmutableList desiredConsistencies; + private static final ImmutableList protocols; + + protected static final AzureKeyCredential credential; + + protected int subscriberValidationTimeout = TIMEOUT; + + private static CosmosAsyncDatabase SHARED_DATABASE; + private static CosmosAsyncContainer SHARED_MULTI_PARTITION_COLLECTION_WITH_ID_AS_PARTITION_KEY; + private static CosmosAsyncContainer SHARED_MULTI_PARTITION_COLLECTION; + private static CosmosAsyncContainer SHARED_MULTI_PARTITION_COLLECTION_WITH_COMPOSITE_AND_SPATIAL_INDEXES; + private static CosmosAsyncContainer SHARED_SINGLE_PARTITION_COLLECTION; + + public TestSuiteBase(CosmosClientBuilder clientBuilder) { + super(clientBuilder); + } + + protected static CosmosAsyncDatabase getSharedCosmosDatabase(CosmosAsyncClient client) { + return CosmosBridgeInternal.getCosmosDatabaseWithNewClient(SHARED_DATABASE, client); + } + + protected static CosmosAsyncContainer getSharedMultiPartitionCosmosContainerWithIdAsPartitionKey(CosmosAsyncClient client) { + return CosmosBridgeInternal.getCosmosContainerWithNewClient(SHARED_MULTI_PARTITION_COLLECTION_WITH_ID_AS_PARTITION_KEY, SHARED_DATABASE, client); + } + + protected static CosmosAsyncContainer getSharedMultiPartitionCosmosContainer(CosmosAsyncClient client) { + return CosmosBridgeInternal.getCosmosContainerWithNewClient(SHARED_MULTI_PARTITION_COLLECTION, SHARED_DATABASE, client); + } + + protected static CosmosAsyncContainer getSharedMultiPartitionCosmosContainerWithCompositeAndSpatialIndexes(CosmosAsyncClient client) { + return CosmosBridgeInternal.getCosmosContainerWithNewClient(SHARED_MULTI_PARTITION_COLLECTION_WITH_COMPOSITE_AND_SPATIAL_INDEXES, SHARED_DATABASE, client); + } + + protected static CosmosAsyncContainer getSharedSinglePartitionCosmosContainer(CosmosAsyncClient client) { + return CosmosBridgeInternal.getCosmosContainerWithNewClient(SHARED_SINGLE_PARTITION_COLLECTION, SHARED_DATABASE, client); + } + + static { + accountConsistency = parseConsistency(TestConfigurations.CONSISTENCY); + desiredConsistencies = immutableListOrNull( + ObjectUtils.defaultIfNull(parseDesiredConsistencies(TestConfigurations.DESIRED_CONSISTENCIES), + allEqualOrLowerConsistencies(accountConsistency))); + preferredLocations = immutableListOrNull(parsePreferredLocation(TestConfigurations.PREFERRED_LOCATIONS)); + protocols = ObjectUtils.defaultIfNull(immutableListOrNull(parseProtocols(TestConfigurations.PROTOCOLS)), + ImmutableList.of(Protocol.HTTPS, Protocol.TCP)); + + // Object mapper configurations + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); + objectMapper.configure(JsonParser.Feature.ALLOW_TRAILING_COMMA, true); + objectMapper.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true); + + credential = new AzureKeyCredential(TestConfigurations.MASTER_KEY); + } + + protected TestSuiteBase() { + logger.debug("Initializing {} ...", this.getClass().getSimpleName()); + } + + private static ImmutableList immutableListOrNull(List list) { + return list != null ? ImmutableList.copyOf(list) : null; + } + + private static class DatabaseManagerImpl implements CosmosDatabaseForTest.DatabaseManager { + public static DatabaseManagerImpl getInstance(CosmosAsyncClient client) { + return new DatabaseManagerImpl(client); + } + + private final CosmosAsyncClient client; + + private DatabaseManagerImpl(CosmosAsyncClient client) { + this.client = client; + } + + @Override + public CosmosPagedFlux queryDatabases(SqlQuerySpec query) { + return client.queryDatabases(query, null); + } + + @Override + public Mono createDatabase(CosmosDatabaseProperties databaseDefinition) { + return client.createDatabase(databaseDefinition); + } + + @Override + public CosmosAsyncDatabase getDatabase(String id) { + return client.getDatabase(id); + } + } + + @BeforeSuite(groups = {"simple", "long", "direct", "multi-master", "emulator", "non-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) + public static void beforeSuite() { + + logger.info("beforeSuite Started"); + + try (CosmosAsyncClient houseKeepingClient = createGatewayHouseKeepingDocumentClient(true).buildAsyncClient()) { + CosmosDatabaseForTest dbForTest = CosmosDatabaseForTest.create(DatabaseManagerImpl.getInstance(houseKeepingClient)); + SHARED_DATABASE = dbForTest.createdDatabase; + CosmosContainerRequestOptions options = new CosmosContainerRequestOptions(); + SHARED_MULTI_PARTITION_COLLECTION = createCollection(SHARED_DATABASE, getCollectionDefinitionWithRangeRangeIndex(), options, 10100); + SHARED_MULTI_PARTITION_COLLECTION_WITH_ID_AS_PARTITION_KEY = createCollection(SHARED_DATABASE, getCollectionDefinitionWithRangeRangeIndexWithIdAsPartitionKey(), options, 10100); + SHARED_MULTI_PARTITION_COLLECTION_WITH_COMPOSITE_AND_SPATIAL_INDEXES = createCollection(SHARED_DATABASE, getCollectionDefinitionMultiPartitionWithCompositeAndSpatialIndexes(), options); + SHARED_SINGLE_PARTITION_COLLECTION = createCollection(SHARED_DATABASE, getCollectionDefinitionWithRangeRangeIndex(), options, 6000); + } + } + + @AfterSuite(groups = {"simple", "long", "direct", "multi-master", "emulator", "non-emulator"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) + public static void afterSuite() { + + logger.info("afterSuite Started"); + + try (CosmosAsyncClient houseKeepingClient = createGatewayHouseKeepingDocumentClient(true).buildAsyncClient()) { + safeDeleteDatabase(SHARED_DATABASE); + CosmosDatabaseForTest.cleanupStaleTestDatabases(DatabaseManagerImpl.getInstance(houseKeepingClient)); + } + } + + protected static void truncateCollection(CosmosAsyncContainer cosmosContainer) { + CosmosContainerProperties cosmosContainerProperties = cosmosContainer.read().block().getProperties(); + String cosmosContainerId = cosmosContainerProperties.getId(); + logger.info("Truncating collection {} ...", cosmosContainerId); + List paths = cosmosContainerProperties.getPartitionKeyDefinition().getPaths(); + CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); + options.setMaxDegreeOfParallelism(-1); + int maxItemCount = 100; + + logger.info("Truncating collection {} documents ...", cosmosContainer.getId()); + + cosmosContainer.queryItems("SELECT * FROM root", options, InternalObjectNode.class) + .byPage(maxItemCount) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(doc -> { + + PartitionKey partitionKey = null; + + Object propertyValue = null; + if (paths != null && !paths.isEmpty()) { + List pkPath = PathParser.getPathParts(paths.get(0)); + propertyValue = ModelBridgeInternal.getObjectByPathFromJsonSerializable(doc, pkPath); + if (propertyValue == null) { + partitionKey = PartitionKey.NONE; + } else { + partitionKey = new PartitionKey(propertyValue); + } + } else { + partitionKey = new PartitionKey(null); + } + + return cosmosContainer.deleteItem(doc.getId(), partitionKey); + }).then().block(); + logger.info("Truncating collection {} triggers ...", cosmosContainerId); + + cosmosContainer.getScripts().queryTriggers("SELECT * FROM root", options) + .byPage(maxItemCount) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(trigger -> { +// if (paths != null && !paths.isEmpty()) { +// Object propertyValue = trigger.getObjectByPath(PathParser.getPathParts(paths.get(0))); +// requestOptions.partitionKey(new PartitionKey(propertyValue)); +// Object propertyValue = getTrigger.getObjectByPath(PathParser.getPathParts(getPaths.get(0))); +// requestOptions.getPartitionKey(new PartitionKey(propertyValue)); +// } + + return cosmosContainer.getScripts().getTrigger(trigger.getId()).delete(); + }).then().block(); + + logger.info("Truncating collection {} storedProcedures ...", cosmosContainerId); + + cosmosContainer.getScripts().queryStoredProcedures("SELECT * FROM root", options) + .byPage(maxItemCount) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(storedProcedure -> { + +// if (getPaths != null && !getPaths.isEmpty()) { +// if (paths != null && !paths.isEmpty()) { +// Object propertyValue = storedProcedure.getObjectByPath(PathParser.getPathParts(paths.get(0))); +// requestOptions.partitionKey(new PartitionKey(propertyValue)); +// requestOptions.getPartitionKey(new PartitionKey(propertyValue)); +// } + + return cosmosContainer.getScripts().getStoredProcedure(storedProcedure.getId()).delete(new CosmosStoredProcedureRequestOptions()); + }).then().block(); + + logger.info("Truncating collection {} udfs ...", cosmosContainerId); + + cosmosContainer.getScripts().queryUserDefinedFunctions("SELECT * FROM root", options) + .byPage(maxItemCount) + .publishOn(Schedulers.parallel()) + .flatMap(page -> Flux.fromIterable(page.getResults())) + .flatMap(udf -> { + +// if (getPaths != null && !getPaths.isEmpty()) { +// if (paths != null && !paths.isEmpty()) { +// Object propertyValue = udf.getObjectByPath(PathParser.getPathParts(paths.get(0))); +// requestOptions.partitionKey(new PartitionKey(propertyValue)); +// requestOptions.getPartitionKey(new PartitionKey(propertyValue)); +// } + + return cosmosContainer.getScripts().getUserDefinedFunction(udf.getId()).delete(); + }).then().block(); + + logger.info("Finished truncating collection {}.", cosmosContainerId); + } + + @SuppressWarnings({"fallthrough"}) + protected static void waitIfNeededForReplicasToCatchUp(CosmosClientBuilder clientBuilder) { + switch (CosmosBridgeInternal.getConsistencyLevel(clientBuilder)) { + case EVENTUAL: + case CONSISTENT_PREFIX: + logger.info(" additional wait in EVENTUAL mode so the replica catch up"); + // give times to replicas to catch up after a write + try { + TimeUnit.MILLISECONDS.sleep(WAIT_REPLICA_CATCH_UP_IN_MILLIS); + } catch (Exception e) { + logger.error("unexpected failure", e); + } + + case SESSION: + case BOUNDED_STALENESS: + case STRONG: + default: + break; + } + } + + public static CosmosAsyncContainer createCollection(CosmosAsyncDatabase database, CosmosContainerProperties cosmosContainerProperties, + CosmosContainerRequestOptions options, int throughput) { + database.createContainer(cosmosContainerProperties, ThroughputProperties.createManualThroughput(throughput), options).block(); + return database.getContainer(cosmosContainerProperties.getId()); + } + + public static CosmosAsyncContainer createCollection(CosmosAsyncDatabase database, CosmosContainerProperties cosmosContainerProperties, + CosmosContainerRequestOptions options) { + database.createContainer(cosmosContainerProperties, options).block(); + return database.getContainer(cosmosContainerProperties.getId()); + } + + private static CosmosContainerProperties getCollectionDefinitionMultiPartitionWithCompositeAndSpatialIndexes() { + final String NUMBER_FIELD = "numberField"; + final String STRING_FIELD = "stringField"; + final String NUMBER_FIELD_2 = "numberField2"; + final String STRING_FIELD_2 = "stringField2"; + final String BOOL_FIELD = "boolField"; + final String NULL_FIELD = "nullField"; + final String OBJECT_FIELD = "objectField"; + final String ARRAY_FIELD = "arrayField"; + final String SHORT_STRING_FIELD = "shortStringField"; + final String MEDIUM_STRING_FIELD = "mediumStringField"; + final String LONG_STRING_FIELD = "longStringField"; + final String PARTITION_KEY = "pk"; + + PartitionKeyDefinition partitionKeyDefinition = new PartitionKeyDefinition(); + ArrayList partitionKeyPaths = new ArrayList(); + partitionKeyPaths.add("/" + PARTITION_KEY); + partitionKeyDefinition.setPaths(partitionKeyPaths); + + CosmosContainerProperties cosmosContainerProperties = new CosmosContainerProperties(UUID.randomUUID().toString(), partitionKeyDefinition); + + IndexingPolicy indexingPolicy = new IndexingPolicy(); + List> compositeIndexes = new ArrayList<>(); + + //Simple + ArrayList compositeIndexSimple = new ArrayList(); + CompositePath compositePath1 = new CompositePath(); + compositePath1.setPath("/" + NUMBER_FIELD); + compositePath1.setOrder(CompositePathSortOrder.ASCENDING); + + CompositePath compositePath2 = new CompositePath(); + compositePath2.setPath("/" + STRING_FIELD); + compositePath2.setOrder(CompositePathSortOrder.DESCENDING); + + compositeIndexSimple.add(compositePath1); + compositeIndexSimple.add(compositePath2); + + //Max Columns + ArrayList compositeIndexMaxColumns = new ArrayList(); + CompositePath compositePath3 = new CompositePath(); + compositePath3.setPath("/" + NUMBER_FIELD); + compositePath3.setOrder(CompositePathSortOrder.DESCENDING); + + CompositePath compositePath4 = new CompositePath(); + compositePath4.setPath("/" + STRING_FIELD); + compositePath4.setOrder(CompositePathSortOrder.ASCENDING); + + CompositePath compositePath5 = new CompositePath(); + compositePath5.setPath("/" + NUMBER_FIELD_2); + compositePath5.setOrder(CompositePathSortOrder.DESCENDING); + + CompositePath compositePath6 = new CompositePath(); + compositePath6.setPath("/" + STRING_FIELD_2); + compositePath6.setOrder(CompositePathSortOrder.ASCENDING); + + compositeIndexMaxColumns.add(compositePath3); + compositeIndexMaxColumns.add(compositePath4); + compositeIndexMaxColumns.add(compositePath5); + compositeIndexMaxColumns.add(compositePath6); + + //Primitive Values + ArrayList compositeIndexPrimitiveValues = new ArrayList(); + CompositePath compositePath7 = new CompositePath(); + compositePath7.setPath("/" + NUMBER_FIELD); + compositePath7.setOrder(CompositePathSortOrder.DESCENDING); + + CompositePath compositePath8 = new CompositePath(); + compositePath8.setPath("/" + STRING_FIELD); + compositePath8.setOrder(CompositePathSortOrder.ASCENDING); + + CompositePath compositePath9 = new CompositePath(); + compositePath9.setPath("/" + BOOL_FIELD); + compositePath9.setOrder(CompositePathSortOrder.DESCENDING); + + CompositePath compositePath10 = new CompositePath(); + compositePath10.setPath("/" + NULL_FIELD); + compositePath10.setOrder(CompositePathSortOrder.ASCENDING); + + compositeIndexPrimitiveValues.add(compositePath7); + compositeIndexPrimitiveValues.add(compositePath8); + compositeIndexPrimitiveValues.add(compositePath9); + compositeIndexPrimitiveValues.add(compositePath10); + + //Long Strings + ArrayList compositeIndexLongStrings = new ArrayList(); + CompositePath compositePath11 = new CompositePath(); + compositePath11.setPath("/" + STRING_FIELD); + + CompositePath compositePath12 = new CompositePath(); + compositePath12.setPath("/" + SHORT_STRING_FIELD); + + CompositePath compositePath13 = new CompositePath(); + compositePath13.setPath("/" + MEDIUM_STRING_FIELD); + + CompositePath compositePath14 = new CompositePath(); + compositePath14.setPath("/" + LONG_STRING_FIELD); + + compositeIndexLongStrings.add(compositePath11); + compositeIndexLongStrings.add(compositePath12); + compositeIndexLongStrings.add(compositePath13); + compositeIndexLongStrings.add(compositePath14); + + compositeIndexes.add(compositeIndexSimple); + compositeIndexes.add(compositeIndexMaxColumns); + compositeIndexes.add(compositeIndexPrimitiveValues); + compositeIndexes.add(compositeIndexLongStrings); + + indexingPolicy.setCompositeIndexes(compositeIndexes); + cosmosContainerProperties.setIndexingPolicy(indexingPolicy); + + return cosmosContainerProperties; + } + + public static CosmosAsyncContainer createCollection(CosmosAsyncClient client, String dbId, CosmosContainerProperties collectionDefinition) { + CosmosAsyncDatabase database = client.getDatabase(dbId); + database.createContainer(collectionDefinition).block(); + return database.getContainer(collectionDefinition.getId()); + } + + public static void deleteCollection(CosmosAsyncClient client, String dbId, String collectionId) { + client.getDatabase(dbId).getContainer(collectionId).delete().block(); + } + + public static InternalObjectNode createDocument(CosmosAsyncContainer cosmosContainer, InternalObjectNode item) { + return BridgeInternal.getProperties(cosmosContainer.createItem(item).block()); + } + + public Flux> bulkInsert(CosmosAsyncContainer cosmosContainer, + List documentDefinitionList, + int concurrencyLevel) { + List>> result = + new ArrayList<>(documentDefinitionList.size()); + for (T docDef : documentDefinitionList) { + result.add(cosmosContainer.createItem(docDef)); + } + + return Flux.merge(Flux.fromIterable(result), concurrencyLevel); + } + public List bulkInsertBlocking(CosmosAsyncContainer cosmosContainer, + List documentDefinitionList) { + return bulkInsert(cosmosContainer, documentDefinitionList, DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL) + .publishOn(Schedulers.parallel()) + .map(itemResponse -> itemResponse.getItem()) + .collectList() + .block(); + } + + public void voidBulkInsertBlocking(CosmosAsyncContainer cosmosContainer, + List documentDefinitionList) { + bulkInsert(cosmosContainer, documentDefinitionList, DEFAULT_BULK_INSERT_CONCURRENCY_LEVEL) + .publishOn(Schedulers.parallel()) + .map(itemResponse -> BridgeInternal.getProperties(itemResponse)) + .then() + .block(); + } + + public static CosmosAsyncUser createUser(CosmosAsyncClient client, String databaseId, CosmosUserProperties userSettings) { + CosmosAsyncDatabase database = client.getDatabase(databaseId); + CosmosUserResponse userResponse = database.createUser(userSettings).block(); + return database.getUser(userResponse.getProperties().getId()); + } + + public static CosmosAsyncUser safeCreateUser(CosmosAsyncClient client, String databaseId, CosmosUserProperties user) { + deleteUserIfExists(client, databaseId, user.getId()); + return createUser(client, databaseId, user); + } + + private static CosmosAsyncContainer safeCreateCollection(CosmosAsyncClient client, String databaseId, CosmosContainerProperties collection, CosmosContainerRequestOptions options) { + deleteCollectionIfExists(client, databaseId, collection.getId()); + return createCollection(client.getDatabase(databaseId), collection, options); + } + + static protected CosmosContainerProperties getCollectionDefinition() { + PartitionKeyDefinition partitionKeyDef = new PartitionKeyDefinition(); + ArrayList paths = new ArrayList(); + paths.add("/mypk"); + partitionKeyDef.setPaths(paths); + + CosmosContainerProperties collectionDefinition = new CosmosContainerProperties(UUID.randomUUID().toString(), partitionKeyDef); + + return collectionDefinition; + } + + static protected CosmosContainerProperties getCollectionDefinitionWithRangeRangeIndexWithIdAsPartitionKey() { + return getCollectionDefinitionWithRangeRangeIndex(Collections.singletonList("/id")); + } + + static protected CosmosContainerProperties getCollectionDefinitionWithRangeRangeIndex() { + return getCollectionDefinitionWithRangeRangeIndex(Collections.singletonList("/mypk")); + } + + static protected CosmosContainerProperties getCollectionDefinitionWithRangeRangeIndex(List partitionKeyPath) { + PartitionKeyDefinition partitionKeyDef = new PartitionKeyDefinition(); + + partitionKeyDef.setPaths(partitionKeyPath); + IndexingPolicy indexingPolicy = new IndexingPolicy(); + List includedPaths = new ArrayList<>(); + IncludedPath includedPath = new IncludedPath("/*"); + includedPaths.add(includedPath); + indexingPolicy.setIncludedPaths(includedPaths); + + CosmosContainerProperties cosmosContainerProperties = new CosmosContainerProperties(UUID.randomUUID().toString(), partitionKeyDef); + cosmosContainerProperties.setIndexingPolicy(indexingPolicy); + + return cosmosContainerProperties; + } + + public static void deleteCollectionIfExists(CosmosAsyncClient client, String databaseId, String collectionId) { + CosmosAsyncDatabase database = client.getDatabase(databaseId); + database.read().block(); + List res = database.queryContainers(String.format("SELECT * FROM root r where r.id = '%s'", collectionId), null) + .collectList() + .block(); + + if (!res.isEmpty()) { + deleteCollection(database, collectionId); + } + } + + public static void deleteCollection(CosmosAsyncDatabase cosmosDatabase, String collectionId) { + cosmosDatabase.getContainer(collectionId).delete().block(); + } + + public static void deleteCollection(CosmosAsyncContainer cosmosContainer) { + cosmosContainer.delete().block(); + } + + public static void deleteDocumentIfExists(CosmosAsyncClient client, String databaseId, String collectionId, String docId) { + CosmosQueryRequestOptions options = new CosmosQueryRequestOptions(); + options.setPartitionKey(new PartitionKey(docId)); + CosmosAsyncContainer cosmosContainer = client.getDatabase(databaseId).getContainer(collectionId); + + List res = cosmosContainer + .queryItems(String.format("SELECT * FROM root r where r.id = '%s'", docId), options, InternalObjectNode.class) + .byPage() + .flatMap(page -> Flux.fromIterable(page.getResults())) + .collectList().block(); + + if (!res.isEmpty()) { + deleteDocument(cosmosContainer, docId); + } + } + + public static void safeDeleteDocument(CosmosAsyncContainer cosmosContainer, String documentId, Object partitionKey) { + if (cosmosContainer != null && documentId != null) { + try { + cosmosContainer.deleteItem(documentId, new PartitionKey(partitionKey)).block(); + } catch (Exception e) { + CosmosException dce = Utils.as(e, CosmosException.class); + if (dce == null || dce.getStatusCode() != 404) { + throw e; + } + } + } + } + + public static void deleteDocument(CosmosAsyncContainer cosmosContainer, String documentId) { + cosmosContainer.deleteItem(documentId, PartitionKey.NONE).block(); + } + + public static void deleteUserIfExists(CosmosAsyncClient client, String databaseId, String userId) { + CosmosAsyncDatabase database = client.getDatabase(databaseId); + client.getDatabase(databaseId).read().block(); + List res = database + .queryUsers(String.format("SELECT * FROM root r where r.id = '%s'", userId), null) + .collectList().block(); + if (!res.isEmpty()) { + deleteUser(database, userId); + } + } + + public static void deleteUser(CosmosAsyncDatabase database, String userId) { + database.getUser(userId).delete().block(); + } + + static private CosmosAsyncDatabase safeCreateDatabase(CosmosAsyncClient client, CosmosDatabaseProperties databaseSettings) { + safeDeleteDatabase(client.getDatabase(databaseSettings.getId())); + client.createDatabase(databaseSettings).block(); + return client.getDatabase(databaseSettings.getId()); + } + + static protected CosmosAsyncDatabase createDatabase(CosmosAsyncClient client, String databaseId) { + CosmosDatabaseProperties databaseSettings = new CosmosDatabaseProperties(databaseId); + client.createDatabase(databaseSettings).block(); + return client.getDatabase(databaseSettings.getId()); + } + + static protected CosmosDatabase createSyncDatabase(CosmosClient client, String databaseId) { + CosmosDatabaseProperties databaseSettings = new CosmosDatabaseProperties(databaseId); + try { + client.createDatabase(databaseSettings); + return client.getDatabase(databaseSettings.getId()); + } catch (CosmosException e) { + e.printStackTrace(); + } + return null; + } + + static protected CosmosAsyncDatabase createDatabaseIfNotExists(CosmosAsyncClient client, String databaseId) { + List res = client.queryDatabases(String.format("SELECT * FROM r where r.id = '%s'", databaseId), null) + .collectList() + .block(); + if (res.size() != 0) { + CosmosAsyncDatabase database = client.getDatabase(databaseId); + database.read().block(); + return database; + } else { + CosmosDatabaseProperties databaseSettings = new CosmosDatabaseProperties(databaseId); + client.createDatabase(databaseSettings).block(); + return client.getDatabase(databaseSettings.getId()); + } + } + + static protected void safeDeleteDatabase(CosmosAsyncDatabase database) { + if (database != null) { + try { + database.delete().block(); + } catch (Exception e) { + } + } + } + + static protected void safeDeleteSyncDatabase(CosmosDatabase database) { + if (database != null) { + try { + logger.info("attempting to delete database ...."); + database.delete(); + logger.info("database deletion completed"); + } catch (Exception e) { + logger.error("failed to delete sync database", e); + } + } + } + + static protected void safeDeleteAllCollections(CosmosAsyncDatabase database) { + if (database != null) { + List collections = database.readAllContainers() + .collectList() + .block(); + + for(CosmosContainerProperties collection: collections) { + database.getContainer(collection.getId()).delete().block(); + } + } + } + + static protected void safeDeleteCollection(CosmosAsyncContainer collection) { + if (collection != null) { + try { + collection.delete().block(); + } catch (Exception e) { + } + } + } + + static protected void safeDeleteCollection(CosmosAsyncDatabase database, String collectionId) { + if (database != null && collectionId != null) { + try { + database.getContainer(collectionId).delete().block(); + } catch (Exception e) { + } + } + } + + static protected void safeCloseAsync(CosmosAsyncClient client) { + if (client != null) { + new Thread(() -> { + try { + client.close(); + } catch (Exception e) { + logger.error("failed to close client", e); + } + }).start(); + } + } + + static protected void safeClose(CosmosAsyncClient client) { + if (client != null) { + try { + client.close(); + } catch (Exception e) { + logger.error("failed to close client", e); + } + } + } + + static protected void safeCloseSyncClient(CosmosClient client) { + if (client != null) { + try { + logger.info("closing client ..."); + client.close(); + logger.info("closing client completed"); + } catch (Exception e) { + logger.error("failed to close client", e); + } + } + } + +// @SuppressWarnings("rawtypes") +// public void validateSuccess(Mono single, CosmosResponseValidator validator) { +// validateSuccess(single, validator, subscriberValidationTimeout); +// } +// +// @SuppressWarnings("rawtypes") +// public void validateSuccess(Mono single, CosmosResponseValidator validator, long timeout) { +// validateSuccess(single.flux(), validator, timeout); +// } +// +// @SuppressWarnings("rawtypes") +// public static void validateSuccess(Flux flowable, +// CosmosResponseValidator validator, long timeout) { +// +// TestSubscriber testSubscriber = new TestSubscriber<>(); +// +// flowable.subscribe(testSubscriber); +// testSubscriber.awaitTerminalEvent(timeout, TimeUnit.MILLISECONDS); +// testSubscriber.assertNoErrors(); +// testSubscriber.assertComplete(); +// testSubscriber.assertValueCount(1); +// validator.validate(testSubscriber.values().get(0)); +// } +// +// @SuppressWarnings("rawtypes") +// public void validateFailure(Mono mono, FailureValidator validator) +// throws InterruptedException { +// validateFailure(mono.flux(), validator, subscriberValidationTimeout); +// } +// +// @SuppressWarnings("rawtypes") +// public static void validateFailure(Flux flowable, +// FailureValidator validator, long timeout) throws InterruptedException { +// +// TestSubscriber testSubscriber = new TestSubscriber<>(); +// +// flowable.subscribe(testSubscriber); +// testSubscriber.awaitTerminalEvent(timeout, TimeUnit.MILLISECONDS); +// testSubscriber.assertNotComplete(); +// testSubscriber.assertTerminated(); +// assertThat(testSubscriber.errors()).hasSize(1); +// validator.validate((Throwable) testSubscriber.getEvents().get(1).get(0)); +// } +// +// @SuppressWarnings("rawtypes") +// public void validateItemSuccess( +// Mono responseMono, CosmosItemResponseValidator validator) { +// +// TestSubscriber testSubscriber = new TestSubscriber<>(); +// responseMono.subscribe(testSubscriber); +// testSubscriber.awaitTerminalEvent(subscriberValidationTimeout, TimeUnit.MILLISECONDS); +// testSubscriber.assertNoErrors(); +// testSubscriber.assertComplete(); +// testSubscriber.assertValueCount(1); +// validator.validate(testSubscriber.values().get(0)); +// } +// +// @SuppressWarnings("rawtypes") +// public void validateItemFailure( +// Mono responseMono, FailureValidator validator) { +// TestSubscriber testSubscriber = new TestSubscriber<>(); +// responseMono.subscribe(testSubscriber); +// testSubscriber.awaitTerminalEvent(subscriberValidationTimeout, TimeUnit.MILLISECONDS); +// testSubscriber.assertNotComplete(); +// testSubscriber.assertTerminated(); +// assertThat(testSubscriber.errors()).hasSize(1); +// validator.validate((Throwable) testSubscriber.getEvents().get(1).get(0)); +// } +// +// public void validateQuerySuccess(Flux> flowable, +// FeedResponseListValidator validator) { +// validateQuerySuccess(flowable, validator, subscriberValidationTimeout); +// } +// +// public static void validateQuerySuccess(Flux> flowable, +// FeedResponseListValidator validator, long timeout) { +// +// TestSubscriber> testSubscriber = new TestSubscriber<>(); +// +// flowable.subscribe(testSubscriber); +// testSubscriber.awaitTerminalEvent(timeout, TimeUnit.MILLISECONDS); +// testSubscriber.assertNoErrors(); +// testSubscriber.assertComplete(); +// validator.validate(testSubscriber.values()); +// } +// +// public void validateQueryFailure(Flux> flowable, FailureValidator validator) { +// validateQueryFailure(flowable, validator, subscriberValidationTimeout); +// } +// +// public static void validateQueryFailure(Flux> flowable, +// FailureValidator validator, long timeout) { +// +// TestSubscriber> testSubscriber = new TestSubscriber<>(); +// +// flowable.subscribe(testSubscriber); +// testSubscriber.awaitTerminalEvent(timeout, TimeUnit.MILLISECONDS); +// testSubscriber.assertNotComplete(); +// testSubscriber.assertTerminated(); +// assertThat(testSubscriber.getEvents().get(1)).hasSize(1); +// validator.validate((Throwable) testSubscriber.getEvents().get(1).get(0)); +// } + + @DataProvider + public static Object[][] clientBuilders() { + return new Object[][]{{createGatewayRxDocumentClient(ConsistencyLevel.SESSION, false, null, true)}}; + } + + @DataProvider + public static Object[][] clientBuildersWithSessionConsistency() { + return new Object[][]{ + {createDirectRxDocumentClient(ConsistencyLevel.SESSION, Protocol.HTTPS, false, null, true)}, + {createDirectRxDocumentClient(ConsistencyLevel.SESSION, Protocol.TCP, false, null, true)}, + {createGatewayRxDocumentClient(ConsistencyLevel.SESSION, false, null, true)} + }; + } + + static ConsistencyLevel parseConsistency(String consistency) { + if (consistency != null) { + consistency = CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, consistency).trim(); + return ConsistencyLevel.valueOf(consistency); + } + + logger.error("INVALID configured test consistency [{}].", consistency); + throw new IllegalStateException("INVALID configured test consistency " + consistency); + } + + static List parsePreferredLocation(String preferredLocations) { + if (StringUtils.isEmpty(preferredLocations)) { + return null; + } + + try { + return objectMapper.readValue(preferredLocations, new TypeReference>() { + }); + } catch (Exception e) { + logger.error("INVALID configured test preferredLocations [{}].", preferredLocations); + throw new IllegalStateException("INVALID configured test preferredLocations " + preferredLocations); + } + } + + static List parseProtocols(String protocols) { + if (StringUtils.isEmpty(protocols)) { + return null; + } + List protocolList = new ArrayList<>(); + try { + List protocolStrings = objectMapper.readValue(protocols, new TypeReference>() { + }); + for(String protocol : protocolStrings) { + protocolList.add(Protocol.valueOf(CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, protocol))); + } + return protocolList; + } catch (Exception e) { + logger.error("INVALID configured test protocols [{}].", protocols); + throw new IllegalStateException("INVALID configured test protocols " + protocols); + } + } + + @DataProvider + public static Object[][] simpleClientBuildersWithDirect() { + return simpleClientBuildersWithDirect(true, toArray(protocols)); + } + + @DataProvider + public static Object[][] simpleClientBuildersWithDirectHttps() { + return simpleClientBuildersWithDirect(true, Protocol.HTTPS); + } + + @DataProvider + public static Object[][] simpleClientBuildersWithDirectTcp() { + return simpleClientBuildersWithDirect(true, Protocol.TCP); + } + + @DataProvider + public static Object[][] simpleClientBuildersWithDirectTcpWithContentResponseOnWriteDisabled() { + return simpleClientBuildersWithDirect(false, Protocol.TCP); + } + + private static Object[][] simpleClientBuildersWithDirect(boolean contentResponseOnWriteEnabled, Protocol... protocols) { + logger.info("Max test consistency to use is [{}]", accountConsistency); + List testConsistencies = ImmutableList.of(ConsistencyLevel.EVENTUAL); + + boolean isMultiMasterEnabled = preferredLocations != null && accountConsistency == ConsistencyLevel.SESSION; + + List cosmosConfigurations = new ArrayList<>(); + + for (Protocol protocol : protocols) { + testConsistencies.forEach(consistencyLevel -> cosmosConfigurations.add(createDirectRxDocumentClient( + consistencyLevel, + protocol, + isMultiMasterEnabled, + preferredLocations, + contentResponseOnWriteEnabled))); + } + + cosmosConfigurations.forEach(c -> { + ConnectionPolicy connectionPolicy = CosmosBridgeInternal.getConnectionPolicy(c); + ConsistencyLevel consistencyLevel = CosmosBridgeInternal.getConsistencyLevel(c); + logger.info("Will Use ConnectionMode [{}], Consistency [{}], Protocol [{}]", + connectionPolicy.getConnectionMode(), + consistencyLevel, + extractConfigs(c).getProtocol() + ); + }); + + cosmosConfigurations.add(createGatewayRxDocumentClient(ConsistencyLevel.SESSION, false, null, contentResponseOnWriteEnabled)); + + return cosmosConfigurations.stream().map(b -> new Object[]{b}).collect(Collectors.toList()).toArray(new Object[0][]); + } + + @DataProvider + public static Object[][] clientBuildersWithDirect() { + return clientBuildersWithDirectAllConsistencies(true, toArray(protocols)); + } + + @DataProvider + public static Object[][] clientBuildersWithDirectHttps() { + return clientBuildersWithDirectAllConsistencies(true, Protocol.HTTPS); + } + + @DataProvider + public static Object[][] clientBuildersWithDirectTcp() { + return clientBuildersWithDirectAllConsistencies(true, Protocol.TCP); + } + + @DataProvider + public static Object[][] clientBuildersWithDirectTcpWithContentResponseOnWriteDisabled() { + return clientBuildersWithDirectAllConsistencies(false, Protocol.TCP); + } + + @DataProvider + public static Object[][] clientBuildersWithDirectSession() { + return clientBuildersWithDirectSession(true, toArray(protocols)); + } + + static Protocol[] toArray(List protocols) { + return protocols.toArray(new Protocol[protocols.size()]); + } + + private static Object[][] clientBuildersWithDirectSession(boolean contentResponseOnWriteEnabled, Protocol... protocols) { + return clientBuildersWithDirect(new ArrayList() {{ + add(ConsistencyLevel.SESSION); + }}, contentResponseOnWriteEnabled, protocols); + } + + private static Object[][] clientBuildersWithDirectAllConsistencies(boolean contentResponseOnWriteEnabled, Protocol... protocols) { + logger.info("Max test consistency to use is [{}]", accountConsistency); + return clientBuildersWithDirect(desiredConsistencies, contentResponseOnWriteEnabled, protocols); + } + + static List parseDesiredConsistencies(String consistencies) { + if (StringUtils.isEmpty(consistencies)) { + return null; + } + List consistencyLevels = new ArrayList<>(); + try { + List consistencyStrings = objectMapper.readValue(consistencies, new TypeReference>() {}); + for(String consistency : consistencyStrings) { + consistencyLevels.add(ConsistencyLevel.valueOf(CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, consistency))); + } + return consistencyLevels; + } catch (Exception e) { + logger.error("INVALID consistency test desiredConsistencies [{}].", consistencies); + throw new IllegalStateException("INVALID configured test desiredConsistencies " + consistencies); + } + } + + @SuppressWarnings("fallthrough") + static List allEqualOrLowerConsistencies(ConsistencyLevel accountConsistency) { + List testConsistencies = new ArrayList<>(); + switch (accountConsistency) { + + case STRONG: + testConsistencies.add(ConsistencyLevel.STRONG); + case BOUNDED_STALENESS: + testConsistencies.add(ConsistencyLevel.BOUNDED_STALENESS); + case SESSION: + testConsistencies.add(ConsistencyLevel.SESSION); + case CONSISTENT_PREFIX: + testConsistencies.add(ConsistencyLevel.CONSISTENT_PREFIX); + case EVENTUAL: + testConsistencies.add(ConsistencyLevel.EVENTUAL); + break; + default: + throw new IllegalStateException("INVALID configured test consistency " + accountConsistency); + } + return testConsistencies; + } + + private static Object[][] clientBuildersWithDirect(List testConsistencies, boolean contentResponseOnWriteEnabled, Protocol... protocols) { + boolean isMultiMasterEnabled = preferredLocations != null && accountConsistency == ConsistencyLevel.SESSION; + + List cosmosConfigurations = new ArrayList<>(); + + for (Protocol protocol : protocols) { + testConsistencies.forEach(consistencyLevel -> cosmosConfigurations.add(createDirectRxDocumentClient(consistencyLevel, + protocol, + isMultiMasterEnabled, + preferredLocations, + contentResponseOnWriteEnabled))); + } + + cosmosConfigurations.forEach(c -> { + ConnectionPolicy connectionPolicy = CosmosBridgeInternal.getConnectionPolicy(c); + ConsistencyLevel consistencyLevel = CosmosBridgeInternal.getConsistencyLevel(c); + logger.info("Will Use ConnectionMode [{}], Consistency [{}], Protocol [{}]", + connectionPolicy.getConnectionMode(), + consistencyLevel, + extractConfigs(c).getProtocol() + ); + }); + + cosmosConfigurations.add(createGatewayRxDocumentClient(ConsistencyLevel.SESSION, isMultiMasterEnabled, preferredLocations, contentResponseOnWriteEnabled)); + + return cosmosConfigurations.stream().map(c -> new Object[]{c}).collect(Collectors.toList()).toArray(new Object[0][]); + } + + static protected CosmosClientBuilder createGatewayHouseKeepingDocumentClient(boolean contentResponseOnWriteEnabled) { + ThrottlingRetryOptions options = new ThrottlingRetryOptions(); + options.setMaxRetryWaitTime(Duration.ofSeconds(SUITE_SETUP_TIMEOUT)); + GatewayConnectionConfig gatewayConnectionConfig = new GatewayConnectionConfig(); + return new CosmosClientBuilder().endpoint(TestConfigurations.HOST) + .credential(credential) + .gatewayMode(gatewayConnectionConfig) + .throttlingRetryOptions(options) + .contentResponseOnWriteEnabled(contentResponseOnWriteEnabled) + .consistencyLevel(ConsistencyLevel.SESSION); + } + + static protected CosmosClientBuilder createGatewayRxDocumentClient(ConsistencyLevel consistencyLevel, boolean multiMasterEnabled, + List preferredRegions, boolean contentResponseOnWriteEnabled) { + GatewayConnectionConfig gatewayConnectionConfig = new GatewayConnectionConfig(); + return new CosmosClientBuilder().endpoint(TestConfigurations.HOST) + .credential(credential) + .gatewayMode(gatewayConnectionConfig) + .multipleWriteRegionsEnabled(multiMasterEnabled) + .preferredRegions(preferredRegions) + .contentResponseOnWriteEnabled(contentResponseOnWriteEnabled) + .consistencyLevel(consistencyLevel); + } + + static protected CosmosClientBuilder createGatewayRxDocumentClient() { + return createGatewayRxDocumentClient(ConsistencyLevel.SESSION, false, null, true); + } + + static protected CosmosClientBuilder createDirectRxDocumentClient(ConsistencyLevel consistencyLevel, + Protocol protocol, + boolean multiMasterEnabled, + List preferredRegions, + boolean contentResponseOnWriteEnabled) { + CosmosClientBuilder builder = new CosmosClientBuilder().endpoint(TestConfigurations.HOST) + .credential(credential) + .directMode(DirectConnectionConfig.getDefaultConfig()) + .contentResponseOnWriteEnabled(contentResponseOnWriteEnabled) + .consistencyLevel(consistencyLevel); + if (preferredRegions != null) { + builder.preferredRegions(preferredRegions); + } + + if (multiMasterEnabled && consistencyLevel == ConsistencyLevel.SESSION) { + builder.multipleWriteRegionsEnabled(true); + } + + Configs configs = spy(new Configs()); + doAnswer((Answer)invocation -> protocol).when(configs).getProtocol(); + + return injectConfigs(builder, configs); + } + + protected int expectedNumberOfPages(int totalExpectedResult, int maxPageSize) { + return Math.max((totalExpectedResult + maxPageSize - 1 ) / maxPageSize, 1); + } + + @DataProvider(name = "queryMetricsArgProvider") + public Object[][] queryMetricsArgProvider() { + return new Object[][]{ + {true}, + {false}, + }; + } + + public static CosmosClientBuilder copyCosmosClientBuilder(CosmosClientBuilder builder) { + return CosmosBridgeInternal.cloneCosmosClientBuilder(builder); + } +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/direct-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/direct-testng.xml new file mode 100644 index 000000000000..ab1bebd6701e --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/direct-testng.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/e2e-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/e2e-testng.xml new file mode 100644 index 000000000000..5a882b76bc62 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/e2e-testng.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/emulator-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/emulator-testng.xml new file mode 100644 index 000000000000..042b8527fef4 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/emulator-testng.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption/dotnet/DataEncryptionKeyProperties.json b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption/dotnet/DataEncryptionKeyProperties.json new file mode 100644 index 000000000000..1b3b95bb17f4 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption/dotnet/DataEncryptionKeyProperties.json @@ -0,0 +1,15 @@ +{ + "id": "mydek", + "encryptionAlgorithm": "AEAes256CbcHmacSha256Randomized", + "wrappedDataEncryptionKey": "6JA4qWfHgxVYZlrFlM6ol5kdYzQC8pzHZ2cJF17dZf4=", + "keyWrapMetadata": { + "type": "custom", + "value": "metadata1updated" + }, + "createTime": 1587096167, + "_rid": "YJIgAPkqHVYBAAAAAAAAAA==", + "_self": "dbs/YJIgAA==/colls/YJIgAPkqHVY=/docs/YJIgAPkqHVYBAAAAAAAAAA==/", + "_etag": "\"00000000-0000-0000-146d-0e494cde01d6\"", + "_attachments": "attachments/", + "_ts": 1587096167 +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption/dotnet/EncryptedPOCO.json b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption/dotnet/EncryptedPOCO.json new file mode 100644 index 000000000000..06d64adbd150 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption/dotnet/EncryptedPOCO.json @@ -0,0 +1,16 @@ +{ + "id": "46df58fe-0781-4c8c-b844-1cd49c9f41a6", + "PK": "652a5b30-bdfa-4ae7-b62c-4a528736cd4a", + "NonSensitive": "502a226d-1dd2-4107-8bb0-570f0aa1bdcf", + "_ei": { + "_ef": 2, + "_en": "mydek", + "_ea": "AEAes256CbcHmacSha256Randomized", + "_ed": "ASM9gn6FvVWdU5r2SCkqeugzGz04/ZkOEQNLBOcmYsbCclkRXPAasSNfrta5WZ+9BB6hNOlODTTbg4qNYILRkg8Fj8xHaQiYbKD1G9Sepvpy4HqkSTERJOMnuP/BSPT8KBX3ihEdE51QtZw4iwaQ1BPHP9R8Mr4yTnrak7NNAjYfunLUMpdojm6fob/NUFb+S3EMqUjZS/tVLLOhm510tCdK520W4RFEauPEZTVKXgR3lj4+BkYrbn2wpY6Y+s6CAc/e+rf31SGWRWzlWinA7AxAkPxRGN1nfNQxs+rxjK/yZLh3kVlFbc11RuFP8UZOeyQ+nSZ1b6GesddsUCgyYjIe/JE/hQvNgN5FQJs7xxV3Csd4qWXvBnN57hgdsQ/ynOrNLZjnumJhSOufhqOCHLxpYSZktR76fgl75QDiyK39" + }, + "_rid": "YJIgAL2lPHkBAAAAAAAAAA==", + "_self": "dbs/YJIgAA==/colls/YJIgAL2lPHk=/docs/YJIgAL2lPHkBAAAAAAAAAA==/", + "_etag": "\"00000000-0000-0000-146f-cc7fe12b01d6\"", + "_attachments": "attachments/", + "_ts": 1587097345 +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption/dotnet/POCO.json b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption/dotnet/POCO.json new file mode 100644 index 000000000000..d2bd3c771363 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption/dotnet/POCO.json @@ -0,0 +1,6 @@ +{ + "id": "46df58fe-0781-4c8c-b844-1cd49c9f41a6", + "PK": "652a5b30-bdfa-4ae7-b62c-4a528736cd4a", + "NonSensitive": "502a226d-1dd2-4107-8bb0-570f0aa1bdcf", + "Sensitive": "d47ea50f-b671-4227-890b-4f38251d4f0c" +} diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/examples-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/examples-testng.xml new file mode 100644 index 000000000000..f9ba695377b7 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/examples-testng.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/fast-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/fast-testng.xml new file mode 100644 index 000000000000..10c2c47b59b7 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/fast-testng.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/log4j2.properties b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/log4j2.properties new file mode 100644 index 000000000000..b68efa3afd76 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/log4j2.properties @@ -0,0 +1,24 @@ +# Set root logger level to INFO and its default appender to be 'STDOUT'. +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT + +# Uncomment here and lines 21 - 25 to enable logging to a file as well. +# rootLogger.appenderRef.logFile.ref = FILE + +property.logDirectory = $${sys:azure.cosmos.logger.directory} +property.hostName = $${sys:azure.cosmos.hostname} + +logger.netty.name = io.netty +logger.netty.level = off + +# STDOUT is a ConsoleAppender and uses PatternLayout. +appender.console.name = STDOUT +appender.console.type = Console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d %5X{pid} [%t] %-5p %c - %m%n + +# appender.logfile.name = FILE +# appender.logfile.type = File +# appender.logfile.filename = ${logDirectory}/azure-cosmos-benchmark.log +# appender.logfile.layout.type = PatternLayout +# appender.logfile.layout.pattern = [%d][%p][${hostName}][thread:%t][logger:%c] %m%n diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/long-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/long-testng.xml new file mode 100644 index 000000000000..3bb293f53fb8 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/long-testng.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/multi-master-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/multi-master-testng.xml new file mode 100644 index 000000000000..ffc341c5b07a --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/multi-master-testng.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/non-emulator-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/non-emulator-testng.xml new file mode 100644 index 000000000000..b1ea939fbbba --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/non-emulator-testng.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/BridgeInternal.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/BridgeInternal.java index 5e3a47527f8e..53cb2ec1b098 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/BridgeInternal.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/BridgeInternal.java @@ -9,6 +9,8 @@ import com.azure.cosmos.implementation.InternalObjectNode; import com.azure.cosmos.implementation.DatabaseAccount; import com.azure.cosmos.implementation.Document; +import com.azure.cosmos.implementation.HttpConstants; +import com.azure.cosmos.implementation.ItemSerializer; import com.azure.cosmos.implementation.FeedResponseDiagnostics; import com.azure.cosmos.implementation.JsonSerializable; import com.azure.cosmos.implementation.MetadataDiagnosticsContext; @@ -26,6 +28,8 @@ import com.azure.cosmos.implementation.directconnectivity.StoreResponse; import com.azure.cosmos.implementation.directconnectivity.StoreResult; import com.azure.cosmos.implementation.directconnectivity.Uri; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider; +import com.azure.cosmos.implementation.encryption.api.EncryptionOptions; import com.azure.cosmos.implementation.query.metrics.ClientSideMetrics; import com.azure.cosmos.implementation.routing.PartitionKeyInternal; import com.azure.cosmos.models.CosmosItemResponse; @@ -65,6 +69,12 @@ public static Document documentFromObject(Object document, ObjectMapper mapper) return Document.fromObject(document, mapper); } + @Warning(value = INTERNAL_USE_ONLY_WARNING) + public static ByteBuffer serializeJsonToByteBuffer(Object document, ObjectMapper mapper, DataEncryptionKeyProvider dataEncryptionKeyProvider, EncryptionOptions encryptionOptions) { + ItemSerializer.CosmosSerializer cosmosSerializer = new ItemSerializer.CosmosSerializer(dataEncryptionKeyProvider, encryptionOptions); + return cosmosSerializer.serializeTo(document); + } + @Warning(value = INTERNAL_USE_ONLY_WARNING) public static ByteBuffer serializeJsonToByteBuffer(Object document, ObjectMapper mapper) { return InternalObjectNode.serializeJsonToByteBuffer(document, mapper); diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncClient.java index bbe989db8e80..3552c39a283a 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncClient.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncClient.java @@ -11,6 +11,7 @@ import com.azure.cosmos.implementation.Database; import com.azure.cosmos.implementation.HttpConstants; import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdMetrics; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider; import com.azure.cosmos.models.CosmosDatabaseResponse; import com.azure.cosmos.models.CosmosDatabaseProperties; import com.azure.cosmos.models.CosmosDatabaseRequestOptions; @@ -51,6 +52,7 @@ public final class CosmosAsyncClient implements Closeable { private final AzureKeyCredential credential; private final boolean sessionCapturingOverride; private final boolean enableTransportClientSharing; + private final DataEncryptionKeyProvider dataEncryptionKeyProvider; private final boolean contentResponseOnWriteEnabled; CosmosAsyncClient(CosmosClientBuilder builder) { @@ -63,6 +65,7 @@ public final class CosmosAsyncClient implements Closeable { this.cosmosAuthorizationTokenResolver = builder.getAuthorizationTokenResolver(); this.credential = builder.getCredential(); this.sessionCapturingOverride = builder.isSessionCapturingOverrideEnabled(); + this.dataEncryptionKeyProvider = builder.getDataEncryptionKeyProvider(); this.enableTransportClientSharing = builder.isConnectionSharingAcrossClientsEnabled(); this.contentResponseOnWriteEnabled = builder.isContentResponseOnWriteEnabled(); this.asyncDocumentClient = new AsyncDocumentClient.Builder() @@ -75,6 +78,7 @@ public final class CosmosAsyncClient implements Closeable { .withTokenResolver(this.cosmosAuthorizationTokenResolver) .withCredential(this.credential) .withTransportClientSharing(this.enableTransportClientSharing) + .withDataEncryptionKeyProvider(this.dataEncryptionKeyProvider) .withContentResponseOnWriteEnabled(this.contentResponseOnWriteEnabled) .build(); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java index 3c1c0bca8c88..c48a8f82aab6 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java @@ -9,6 +9,7 @@ import com.azure.cosmos.implementation.Offer; import com.azure.cosmos.implementation.Paths; import com.azure.cosmos.implementation.RequestOptions; +import com.azure.cosmos.implementation.ItemDeserializer; import com.azure.cosmos.implementation.Utils; import com.azure.cosmos.implementation.query.QueryInfo; import com.azure.cosmos.models.CosmosContainerResponse; @@ -229,7 +230,7 @@ public Mono> createItem(T item, CosmosItemRequestOptio item, requestOptions, true) - .map(response -> ModelBridgeInternal.createCosmosAsyncItemResponse(response, itemType)) + .map(response -> ModelBridgeInternal.createCosmosAsyncItemResponse(response, itemType, getItemDeserializer())) .single(); } @@ -270,7 +271,7 @@ public Mono> upsertItem(T item, CosmosItemRequestOptio .upsertDocument(this.getLink(), item, ModelBridgeInternal.toRequestOptions(options), true) - .map(response -> ModelBridgeInternal.createCosmosAsyncItemResponse(response, itemType)) + .map(response -> ModelBridgeInternal.createCosmosAsyncItemResponse(response, itemType, getItemDeserializer())) .single(); } @@ -453,11 +454,14 @@ public Mono> readItem( if (options == null) { options = new CosmosItemRequestOptions(); } + ModelBridgeInternal.setPartitionKey(options, partitionKey); RequestOptions requestOptions = ModelBridgeInternal.toRequestOptions(options); + return this.getDatabase().getDocClientWrapper() .readDocument(getItemLink(itemId), requestOptions) - .map(response -> ModelBridgeInternal.createCosmosAsyncItemResponse(response, itemType)) + // TODO: add a deserializer and pass down? + .map(response -> ModelBridgeInternal.createCosmosAsyncItemResponse(response, itemType, this.getItemDeserializer())) .single(); } @@ -503,7 +507,7 @@ public Mono> replaceItem( return this.getDatabase() .getDocClientWrapper() .replaceDocument(getItemLink(itemId), doc, ModelBridgeInternal.toRequestOptions(options)) - .map(response -> ModelBridgeInternal.createCosmosAsyncItemResponse(response, itemType)) + .map(response -> ModelBridgeInternal.createCosmosAsyncItemResponse(response, itemType, getItemDeserializer())) .single(); } @@ -709,4 +713,8 @@ String getParentLink() { String getLink() { return this.link; } + + ItemDeserializer getItemDeserializer() { + return getDatabase().getDocClientWrapper().getItemDeserializer(); + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosBridgeInternal.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosBridgeInternal.java index 476939832026..e7c26c76e6f2 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosBridgeInternal.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosBridgeInternal.java @@ -6,6 +6,7 @@ import com.azure.cosmos.implementation.AsyncDocumentClient; import com.azure.cosmos.implementation.ConnectionPolicy; import com.azure.cosmos.implementation.Warning; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider; import static com.azure.cosmos.implementation.Warning.INTERNAL_USE_ONLY_WARNING; @@ -94,4 +95,11 @@ public static CosmosClientBuilder cloneCosmosClientBuilder(CosmosClientBuilder b return copy; } + + @Warning(value = INTERNAL_USE_ONLY_WARNING) + public static CosmosClientBuilder setDateKeyProvider(CosmosClientBuilder cosmosClientBuilder, + DataEncryptionKeyProvider dataEncryptionKeyProvider) { + cosmosClientBuilder.dataEncryptionKeyProvider(dataEncryptionKeyProvider); + return cosmosClientBuilder; + } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java index 13892748126c..3dfac4761233 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosClientBuilder.java @@ -8,6 +8,7 @@ import com.azure.cosmos.implementation.ConnectionPolicy; import com.azure.cosmos.implementation.CosmosAuthorizationTokenResolver; import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; +import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider; import com.azure.cosmos.models.CosmosPermissionProperties; import java.util.Collections; @@ -22,7 +23,7 @@ * Though consistencyLevel is not mandatory, but we strongly suggest to pay attention to this API when building client. * By default, account consistency level is used if none is provided. *

    - * By default, direct connection mode is used if none specified. + * By default, direct connection mode is used if none specified. * 

      *     Building Cosmos Async Client minimal APIs (without any customized configurations)
  * {@code
@@ -88,6 +89,7 @@ public class CosmosClientBuilder {
     private CosmosAuthorizationTokenResolver cosmosAuthorizationTokenResolver;
     private AzureKeyCredential credential;
     private boolean sessionCapturingOverrideEnabled;
+    private DataEncryptionKeyProvider dataEncryptionKeyProvider;
     private boolean connectionSharingAcrossClientsEnabled;
     private boolean contentResponseOnWriteEnabled;
     private String userAgentSuffix;
@@ -490,6 +492,15 @@ public CosmosClientBuilder preferredRegions(List preferredRegions) {
         return this;
     }
 
+    CosmosClientBuilder dataEncryptionKeyProvider(DataEncryptionKeyProvider dataEncryptionKeyProvider) {
+        this.dataEncryptionKeyProvider = dataEncryptionKeyProvider;
+        return this;
+    }
+
+    DataEncryptionKeyProvider getDataEncryptionKeyProvider() {
+        return this.dataEncryptionKeyProvider;
+    }
+
     /**
      * Sets the flag to enable endpoint discovery for geo-replicated database accounts.
      * 
    
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java
index f1077bf084c0..b984d19cc0bd 100644
--- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/AsyncDocumentClient.java
@@ -5,6 +5,7 @@
 import com.azure.core.credential.AzureKeyCredential;
 import com.azure.cosmos.ConsistencyLevel;
 import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
+import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider;
 import com.azure.cosmos.models.CosmosQueryRequestOptions;
 import com.azure.cosmos.models.FeedResponse;
 import com.azure.cosmos.models.PartitionKey;
@@ -74,6 +75,7 @@ class Builder {
         AzureKeyCredential credential;
         boolean sessionCapturingOverride;
         boolean transportClientSharing;
+        private DataEncryptionKeyProvider dataEncryptionKeyProvider;
         boolean contentResponseOnWriteEnabled;
 
         public Builder withServiceEndpoint(String serviceEndpoint) {
@@ -148,6 +150,12 @@ public Builder withTransportClientSharing(boolean transportClientSharing) {
             return this;
         }
 
+        public Builder withDataEncryptionKeyProvider(DataEncryptionKeyProvider dataEncryptionKeyProvider) {
+            this.dataEncryptionKeyProvider = dataEncryptionKeyProvider;
+            return this;
+        }
+
+
         public Builder withCredential(AzureKeyCredential credential) {
             if (credential != null && StringUtils.isEmpty(credential.getKey())) {
                 throw new IllegalArgumentException("Cannot buildAsyncClient client with empty key credential");
@@ -200,6 +208,11 @@ public AsyncDocumentClient build() {
                 sessionCapturingOverride,
                 transportClientSharing,
                 contentResponseOnWriteEnabled);
+
+            if (dataEncryptionKeyProvider != null) {
+                client.registerDataEncryptionKeyProvider(dataEncryptionKeyProvider);
+            }
+
             client.init();
             return client;
         }
@@ -1372,4 +1385,7 @@  Mono> readMany(
      */
     void close();
 
+    ItemDeserializer getItemDeserializer();
+
+    ItemDeserializer getItemDeserializerWithoutDecryption();
 }
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Constants.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Constants.java
index fea6a976cb21..7d67d54f3213 100644
--- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Constants.java
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Constants.java
@@ -191,6 +191,20 @@ public static final class Properties {
         public static final String SSL_COMPLETION_HANDLER_NAME = "ssl-completion-handler";
         public static final String HTTP_PROXY_HANDLER_NAME = "http-proxy-handler";
         public static final String LOGGING_HANDLER_NAME = "logging-handler";
+
+        // encryption
+        public static final String WrappedDataEncryptionKey = "wrappedDataEncryptionKey";
+        public static final String  EncryptionAlgorithmId = "encryptionAlgorithmId";
+        public static final String  KeyWrapMetadata = "keyWrapMetadata";
+        public static final String  KeyWrapMetadataType = "type";
+        public static final String  KeyWrapMetadataValue = "value";
+        public static final String  EncryptedInfo = "_ei";
+//        public static final String  DataEncryptionKeyRid = "_ek";
+        public static final String  EncryptionFormatVersion = "_ef";
+        public static final String  EncryptedData = "_ed";
+        public static final String  EncryptionAlgorithm = "_ea";
+        public static final String  DataEncryptionKeyId = "_en";
+
     }
 
     public static final class UrlEncodingInfo {
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ItemDeserializer.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ItemDeserializer.java
new file mode 100644
index 000000000000..fd032bd6e396
--- /dev/null
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ItemDeserializer.java
@@ -0,0 +1,68 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation;
+
+import com.azure.cosmos.implementation.encryption.EncryptionProcessor;
+import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+
+public interface ItemDeserializer {
+     T parseFrom(Class classType, byte[] bytes);
+     T convert(Class classType, ObjectNode objectNode);
+
+
+    class JsonDeserializer implements ItemDeserializer {
+        public  T parseFrom(Class classType, byte[] bytes) {
+            if (bytes == null) {
+                return null;
+            }
+
+            // TODO: does this handdle jackson ObjectNode?
+            return Utils.parse(bytes, classType);
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public  T convert(Class classType, ObjectNode objectNode) {
+            if (classType == ObjectNode.class) {
+                return (T) objectNode;
+            }
+
+            return Utils.getSimpleObjectMapper().convertValue(objectNode, classType);
+        }
+    }
+
+    class EncryptionDeserializer implements ItemDeserializer {
+        private final DataEncryptionKeyProvider dataEncryptionKeyProvider;
+        private final JsonDeserializer jsonDeserializer;
+
+        public EncryptionDeserializer(DataEncryptionKeyProvider dataEncryptionKeyProvider, JsonDeserializer jsonDeserializer) {
+            this.dataEncryptionKeyProvider = dataEncryptionKeyProvider;
+            this.jsonDeserializer = jsonDeserializer;
+        }
+
+        @Override
+        public  T parseFrom(Class classType, byte[] bytes) {
+            if (bytes == null) {
+                return null;
+            }
+
+            ObjectNode objectNode = jsonDeserializer.parseFrom(ObjectNode.class, bytes);
+            return convert(classType, objectNode);
+        }
+
+        @Override
+        public  T convert(Class classType, ObjectNode objectNode) {
+            if (objectNode == null) {
+                return null;
+            }
+
+            EncryptionProcessor encryptionProcessor = new EncryptionProcessor();
+            ObjectNode objectNodeWithSensitiveDataDecrypted = encryptionProcessor.decryptAsync(objectNode, dataEncryptionKeyProvider);
+
+            return Utils.getSimpleObjectMapper().convertValue(objectNodeWithSensitiveDataDecrypted, classType);
+        }
+    }
+}
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ItemSerializer.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ItemSerializer.java
new file mode 100644
index 000000000000..848ba7944444
--- /dev/null
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ItemSerializer.java
@@ -0,0 +1,66 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation;
+
+import com.azure.cosmos.implementation.encryption.EncryptionProcessor;
+import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider;
+import com.azure.cosmos.implementation.encryption.api.EncryptionOptions;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import java.nio.ByteBuffer;
+
+public interface ItemSerializer {
+     ByteBuffer serializeTo(T item);
+
+    class JsonSerializer implements ItemSerializer {
+        @Override
+        public  ByteBuffer serializeTo(T item) {
+            return InternalObjectNode.serializeJsonToByteBuffer(item, Utils.getSimpleObjectMapper());
+        }
+    }
+
+    class CosmosSerializer implements ItemSerializer {
+        private final JsonSerializer jsonSerializer = new JsonSerializer();
+        private final DataEncryptionKeyProvider dataEncryptionKeyProvider;
+        private final EncryptionOptions encryptionOptions;
+
+        public CosmosSerializer(DataEncryptionKeyProvider dataEncryptionKeyProvider, EncryptionOptions encryptionOptions) {
+            this.dataEncryptionKeyProvider = dataEncryptionKeyProvider;
+            this.encryptionOptions = encryptionOptions;
+        }
+
+        @Override
+        public  ByteBuffer serializeTo(T item) {
+            if (dataEncryptionKeyProvider == null
+                || encryptionOptions == null
+                || encryptionOptions.getPathsToEncrypt() == null
+                || encryptionOptions.getPathsToEncrypt().isEmpty()
+                || item instanceof Document
+                || item instanceof InternalObjectNode) {
+                return jsonSerializer.serializeTo(item);
+            } else {
+                return new EncryptionSerializer(dataEncryptionKeyProvider, jsonSerializer, encryptionOptions).serializeTo(item);
+            }
+        }
+
+    }
+
+    class EncryptionSerializer implements ItemSerializer {
+        private final EncryptionOptions encryptionOptions;
+        private final DataEncryptionKeyProvider dataEncryptionKeyProvider;
+
+        public EncryptionSerializer(DataEncryptionKeyProvider dataEncryptionKeyProvider, JsonSerializer jsonSerializer, EncryptionOptions encryptionOptions) {
+            this.encryptionOptions = encryptionOptions;
+            this.dataEncryptionKeyProvider = dataEncryptionKeyProvider;
+        }
+
+        @Override
+        public  ByteBuffer serializeTo(T item) {
+            ObjectNode objectNode = Utils.getSimpleObjectMapper().convertValue(item, ObjectNode.class);
+            EncryptionProcessor encryptionProcessor = new EncryptionProcessor();
+            ObjectNode result = encryptionProcessor.encryptAsync(objectNode, encryptionOptions, dataEncryptionKeyProvider);
+            return Utils.serializeJsonToByteBuffer(Utils.getSimpleObjectMapper(), result);
+        }
+    }
+}
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RMResources.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RMResources.java
index 922bbbee04d3..5a638a932062 100644
--- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RMResources.java
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RMResources.java
@@ -49,4 +49,5 @@ public class RMResources {
     public static final String InvalidUrl = "InvalidUrl";
     public static final String InvalidResourceUrlQuery = "The value %s specified  for query %s is invalid.";
     public static final String PartitionKeyRangeIdAbsentInContext = "PartitionKeyRangeId is absent in the context.";
+    public static final String EncryptionKeyProviderNotConfigured = "Encryption Key Provider is not configured";
 }
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RequestOptions.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RequestOptions.java
index f73e6624a131..98ceb5bc81f5 100644
--- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RequestOptions.java
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RequestOptions.java
@@ -3,6 +3,8 @@
 
 package com.azure.cosmos.implementation;
 
+import com.azure.cosmos.implementation.encryption.api.EncryptionOptions;
+
 import com.azure.cosmos.ConsistencyLevel;
 import com.azure.cosmos.models.IndexingDirective;
 import com.azure.cosmos.models.PartitionKey;
@@ -32,6 +34,7 @@ public class RequestOptions {
     private boolean scriptLoggingEnabled;
     private boolean quotaInfoEnabled;
     private Map properties;
+    private EncryptionOptions encryptionOptions;
     private ThroughputProperties throughputProperties;
 
     /**
@@ -339,4 +342,12 @@ public void setProperties(Map properties) {
         this.properties = properties;
     }
 
+
+    public void setEncryptionOptions(EncryptionOptions encryptionOptions) {
+        this.encryptionOptions = encryptionOptions;
+    }
+
+    public EncryptionOptions getEncryptionOptions() {
+        return this.encryptionOptions;
+    }
 }
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java
index 120f0efd21bc..1856ff826b6f 100644
--- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/RxDocumentClientImpl.java
@@ -2,11 +2,15 @@
 // Licensed under the MIT License.
 package com.azure.cosmos.implementation;
 
+import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
+import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider;
+
 import com.azure.core.credential.AzureKeyCredential;
 import com.azure.cosmos.BridgeInternal;
 import com.azure.cosmos.ConnectionMode;
 import com.azure.cosmos.ConsistencyLevel;
 import com.azure.cosmos.DirectConnectionConfig;
+import com.azure.cosmos.implementation.encryption.api.EncryptionOptions;
 import com.azure.cosmos.implementation.query.PipelinedDocumentQueryExecutionContext;
 import com.azure.cosmos.implementation.query.QueryInfo;
 import com.azure.cosmos.models.CosmosQueryRequestOptions;
@@ -16,7 +20,6 @@
 import com.azure.cosmos.models.PartitionKeyDefinition;
 import com.azure.cosmos.models.SqlQuerySpec;
 import com.azure.cosmos.models.SqlParameter;
-import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
 import com.azure.cosmos.implementation.apachecommons.lang.tuple.Pair;
 import com.azure.cosmos.implementation.caches.RxClientCollectionCache;
 import com.azure.cosmos.implementation.caches.RxCollectionCache;
@@ -122,6 +125,7 @@ public class RxDocumentClientImpl implements AsyncDocumentClient, IAuthorization
     private StoreClientFactory storeClientFactory;
 
     private GatewayServiceConfigurationReader gatewayConfigurationReader;
+    public DataEncryptionKeyProvider dataEncryptionKeyProvider;
 
     public RxDocumentClientImpl(URI serviceEndpoint,
                                 String masterKeyOrResourceToken,
@@ -254,6 +258,10 @@ private RxDocumentClientImpl(URI serviceEndpoint,
         this.resetSessionTokenRetryPolicy = retryPolicy;
     }
 
+    void registerDataEncryptionKeyProvider(DataEncryptionKeyProvider dataEncryptionKeyProvider) {
+        this.dataEncryptionKeyProvider = dataEncryptionKeyProvider;
+    }
+
     private void initializeGatewayConfigurationReader() {
         this.gatewayConfigurationReader = new GatewayServiceConfigurationReader(this.globalEndpointManager);
         DatabaseAccount databaseAccount = this.globalEndpointManager.getLatestDatabaseAccount();
@@ -1110,8 +1118,9 @@ private Mono getCreateDocumentRequest(DocumentClientRe
         }
 
         Instant serializationStartTimeUTC = Instant.now();
-        ByteBuffer content = BridgeInternal.serializeJsonToByteBuffer(document, mapper);
+        ByteBuffer content = BridgeInternal.serializeJsonToByteBuffer(document, mapper, dataEncryptionKeyProvider, options == null ? null : options.getEncryptionOptions());
         Instant serializationEndTimeUTC = Instant.now();
+
         SerializationDiagnosticsContext.SerializationDiagnostics serializationDiagnostics = new SerializationDiagnosticsContext.SerializationDiagnostics(
             serializationStartTimeUTC,
             serializationEndTimeUTC,
@@ -3185,4 +3194,18 @@ public void close() {
         }
         logger.info("Shutting down completed.");
     }
+
+    @Override
+    public ItemDeserializer getItemDeserializer() {
+        if (dataEncryptionKeyProvider == null) {
+            return new ItemDeserializer.JsonDeserializer();
+        } else {
+            return new ItemDeserializer.EncryptionDeserializer(dataEncryptionKeyProvider, new ItemDeserializer.JsonDeserializer());
+        }
+    }
+
+    @Override
+    public ItemDeserializer getItemDeserializerWithoutDecryption() {
+        return new ItemDeserializer.JsonDeserializer();
+    }
 }
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java
index d91327ff08ed..ed20084d4dfe 100644
--- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java
@@ -4,6 +4,7 @@
 
 import com.azure.cosmos.ConsistencyLevel;
 import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
+import com.azure.cosmos.implementation.encryption.api.EncryptionOptions;
 import com.azure.cosmos.implementation.uuid.EthernetAddress;
 import com.azure.cosmos.implementation.uuid.Generators;
 import com.azure.cosmos.implementation.uuid.impl.TimeBasedGenerator;
@@ -91,6 +92,10 @@ public static byte[] getUTF8Bytes(String str) {
 
     }
 
+    public static byte[] getUtf16Bytes(String str) {
+        return str.getBytes(StandardCharsets.UTF_16LE);
+    }
+
     public static String encodeBase64String(byte[] binaryData) {
         String encodedString = Base64Encoder.encodeToString(binaryData);
 
@@ -597,6 +602,27 @@ public static  T parse(byte[] item, Class itemClassType) {
         }
     }
 
+    public static  T parse(byte[] item, Class itemClassType, ItemDeserializer itemDeserializer) {
+        if (Utils.isEmpty(item)) {
+            return null;
+        }
+
+        if (itemDeserializer == null) {
+            return Utils.parse(item, itemClassType);
+        }
+
+        return itemDeserializer.parseFrom(itemClassType, item);
+    }
+
+    public static  T parse(byte[] item, Class itemClassType, EncryptionOptions encryptionOptions) {
+
+        try {
+            return getSimpleObjectMapper().readValue(item, itemClassType);
+        } catch (IOException e) {
+            throw new IllegalStateException("Failed to get POJO.", e);
+        }
+    }
+
     public static ByteBuffer serializeJsonToByteBuffer(ObjectMapper objectMapper, Object object) {
         try {
             ByteBufferOutputStream byteBufferOutputStream = new ByteBufferOutputStream(ONE_KB);
@@ -657,12 +683,16 @@ static String escapeNonAscii(String partitionKeyJson) {
         }
     }
 
-    static byte[] toByteArray(ByteBuf buf) {
+    public static byte[] toByteArray(ByteBuf buf) {
         byte[] bytes = new byte[buf.readableBytes()];
         buf.readBytes(bytes);
         return bytes;
     }
 
+    public static ByteBuffer toByteBuffer(byte[] bytes) {
+        return ByteBuffer.wrap(bytes);
+    }
+
     public static String toJson(ObjectMapper mapper, ObjectNode object) {
         try {
             return mapper.writeValueAsString(object);
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionProcessor.java
new file mode 100644
index 000000000000..04e225a377f3
--- /dev/null
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionProcessor.java
@@ -0,0 +1,149 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation.encryption;
+
+import com.azure.cosmos.implementation.Constants;
+import com.azure.cosmos.implementation.InternalServerErrorException;
+import com.azure.cosmos.implementation.RMResources;
+import com.azure.cosmos.implementation.Utils;
+import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
+import com.azure.cosmos.implementation.encryption.api.CosmosEncryptionAlgorithm;
+import com.azure.cosmos.implementation.encryption.api.DataEncryptionKey;
+import com.azure.cosmos.implementation.encryption.api.DataEncryptionKeyProvider;
+import com.azure.cosmos.implementation.encryption.api.EncryptionOptions;
+import com.azure.cosmos.implementation.guava27.Strings;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeType;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+
+public class EncryptionProcessor {
+
+    public ObjectNode encryptAsync(
+        ObjectNode itemJObj,
+        EncryptionOptions encryptionOptions,
+        DataEncryptionKeyProvider dataEncryptionKeyProvider) {
+        assert (itemJObj != null);
+        assert (encryptionOptions != null);
+        assert (encryptionOptions.getPathsToEncrypt() != null);
+        assert (!encryptionOptions.getPathsToEncrypt().isEmpty());
+
+        for (String path : encryptionOptions.getPathsToEncrypt()) {
+            if (StringUtils.isEmpty(path) || path.charAt(0) != '/' || path.lastIndexOf('/') != 0) {
+                throw new IllegalArgumentException("Invalid encryption path: " + path);
+            }
+        }
+
+        if (encryptionOptions.getDataEncryptionKeyId() == null) {
+            throw new IllegalArgumentException("Invalid encryption options: encryptionOptions.getDataEncryptionKeyId." + encryptionOptions.getDataEncryptionKeyId());
+        }
+
+        if (encryptionOptions.getEncryptionAlgorithm() == null) {
+            throw new IllegalArgumentException("Invalid encryption options: encryptionOptions.getEncryptionAlgorithm." + encryptionOptions.getEncryptionAlgorithm());
+        }
+
+        if (dataEncryptionKeyProvider == null) {
+            throw new IllegalArgumentException(RMResources.EncryptionKeyProviderNotConfigured);
+        }
+
+        ObjectNode toEncryptJObj = Utils.getSimpleObjectMapper().createObjectNode();
+
+        for (String pathToEncrypt : encryptionOptions.getPathsToEncrypt()) {
+            String propertyName = pathToEncrypt.substring(1);
+            JsonNode propertyValueHolder = itemJObj.get(propertyName);
+
+            // Even null in the JSON is a JToken with Type Null, this null check is just a sanity check
+            if (propertyValueHolder != null) {
+                toEncryptJObj.set(propertyName, itemJObj.get(propertyName));
+                itemJObj.remove(propertyName);
+            }
+        }
+
+        SensitiveDataTransformer serializer = new SensitiveDataTransformer();
+        byte[] plainText = serializer.toByteArray(toEncryptJObj);
+
+        DataEncryptionKey dataEncryptionKey = dataEncryptionKeyProvider.getDataEncryptionKey(encryptionOptions.getDataEncryptionKeyId(), encryptionOptions.getEncryptionAlgorithm());
+
+        EncryptionProperties encryptionProperties = new EncryptionProperties(
+            /* encryptionFormatVersion: */ 2,
+            encryptionOptions.getEncryptionAlgorithm(),
+            encryptionOptions.getDataEncryptionKeyId(),
+            dataEncryptionKey.encryptData(plainText));
+
+        itemJObj.set(Constants.Properties.EncryptedInfo, encryptionProperties.toObjectNode());
+
+        return itemJObj;
+    }
+
+
+    public ObjectNode decryptAsync(
+        ObjectNode itemJObj,
+        DataEncryptionKeyProvider keyProvider) {
+        assert (itemJObj != null);
+        assert (keyProvider != null);
+
+        JsonNode encryptionPropertiesJProp = itemJObj.get(Constants.Properties.EncryptedInfo);
+        ObjectNode encryptionPropertiesJObj = null;
+        if (encryptionPropertiesJProp != null && !encryptionPropertiesJProp.isNull() && encryptionPropertiesJProp.getNodeType() == JsonNodeType.OBJECT) {
+            encryptionPropertiesJObj = (ObjectNode) encryptionPropertiesJProp;
+        }
+
+        if (encryptionPropertiesJProp == null) {
+            return itemJObj;
+        }
+
+        EncryptionProperties encryptionProperties = null;
+        try {
+            encryptionProperties = EncryptionProperties.fromObjectNode(encryptionPropertiesJObj);
+        } catch (IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+        if (encryptionProperties.getEncryptionFormatVersion() != 2) {
+            throw new InternalServerErrorException(Strings.lenientFormat(
+                "Unknown encryption format version: %s. Please upgrade your SDK to the latest version.", encryptionProperties.getEncryptionFormatVersion()));
+        }
+
+        // get key
+        DataEncryptionKey inMemoryRawDek = keyProvider.getDataEncryptionKey(encryptionProperties.getDataEncryptionKeyId(), CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized);
+
+        byte[] plainText = inMemoryRawDek.decryptData(encryptionProperties.getEncryptedData());
+
+        SensitiveDataTransformer parser = new SensitiveDataTransformer();
+        ObjectNode plainTextJObj = parser.toObjectNode(plainText);
+
+        Iterator> it = plainTextJObj.fields();
+        while (it.hasNext()) {
+            Map.Entry entry = it.next();
+            itemJObj.set(entry.getKey(), entry.getValue());
+        }
+
+        itemJObj.remove(Constants.Properties.EncryptedInfo);
+        return itemJObj;
+    }
+
+    static class SensitiveDataTransformer {
+        public  ObjectNode toObjectNode(byte[] plainText) {
+            if (Utils.isEmpty(plainText)) {
+                return null;
+            }
+            try {
+                return (ObjectNode) Utils.getSimpleObjectMapper().readTree(plainText);
+            } catch (IOException e) {
+                throw new IllegalStateException("Failed to parse to ObjectNode.", e);
+            }
+        }
+
+        public byte[] toByteArray(ObjectNode objectNode) {
+            try {
+                return Utils.getSimpleObjectMapper().writeValueAsBytes(objectNode);
+            } catch (JsonProcessingException e) {
+                throw new IllegalStateException("Unable to convert JSON to byte[]", e);
+            }
+        }
+    }
+}
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionProperties.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionProperties.java
new file mode 100644
index 000000000000..d151e21a720a
--- /dev/null
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionProperties.java
@@ -0,0 +1,181 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation.encryption;
+
+import com.azure.cosmos.implementation.Constants;
+import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
+import com.azure.cosmos.implementation.guava25.base.Preconditions;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.ObjectCodec;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+
+import java.io.IOException;
+
+@JsonSerialize(using = EncryptionProperties.JsonSerializer.class)
+@JsonDeserialize(using = EncryptionProperties.JsonDeserializer.class)
+class EncryptionProperties {
+    private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    public static ObjectReader getObjectReader() {
+        return OBJECT_READER;
+    }
+
+    public static ObjectWriter getObjectWriter() {
+        return OBJECT_WRITER;
+    }
+
+    private final static ObjectReader OBJECT_READER = OBJECT_MAPPER.readerFor(EncryptionProperties.class);
+    private final static ObjectWriter OBJECT_WRITER = OBJECT_MAPPER.writerFor(EncryptionProperties.class);
+
+    public ObjectNode toObjectNode() {
+        ObjectNode objectNode = OBJECT_MAPPER.createObjectNode();
+        objectNode.put(Constants.Properties.EncryptionFormatVersion, this.encryptionFormatVersion);
+        objectNode.put(Constants.Properties.EncryptionAlgorithm, this.encryptionAlgorithm);
+        objectNode.put(Constants.Properties.DataEncryptionKeyId, this.dataEncryptionKeyId);
+        objectNode.put(Constants.Properties.EncryptedData, this.encryptedData);
+        return objectNode;
+    }
+
+    public static EncryptionProperties fromObjectNode(ObjectNode objectNode) throws IOException {
+        return EncryptionProperties.OBJECT_READER.readValue(objectNode);
+    }
+
+    public EncryptionProperties() {
+    }
+
+    public String getEncryptionAlgorithm() {
+        return this.encryptionAlgorithm;
+    }
+
+    public int getEncryptionFormatVersion() {
+        return encryptionFormatVersion;
+    }
+
+    public String getDataEncryptionKeyId() {
+        return dataEncryptionKeyId;
+    }
+
+    public byte[] getEncryptedData() {
+        return encryptedData;
+    }
+
+
+    private String encryptionAlgorithm;
+
+    private int encryptionFormatVersion;
+    private String dataEncryptionKeyId;
+    private byte[] encryptedData;
+
+    public EncryptionProperties(
+        int encryptionFormatVersion,
+        String encryptionAlgorithm,
+        String dataEncryptionKeyId,
+        byte[] encryptedData) {
+
+        if (StringUtils.isEmpty(encryptionAlgorithm)) {
+            throw new IllegalArgumentException("encryptionAlgorithm is missing");
+        }
+
+        if (StringUtils.isEmpty(dataEncryptionKeyId)) {
+            throw new IllegalArgumentException("dataEncryptionKeyId is missing");
+        }
+
+        this.encryptionFormatVersion = encryptionFormatVersion;
+        this.encryptionAlgorithm = encryptionAlgorithm;
+        this.dataEncryptionKeyId = dataEncryptionKeyId;
+        this.encryptedData = encryptedData;
+    }
+
+    static final class JsonSerializer extends StdSerializer {
+        private static final long serialVersionUID = 1L;
+
+        JsonSerializer() {
+            super(EncryptionProperties.class);
+        }
+
+        @Override
+        public void serialize(final EncryptionProperties value, final JsonGenerator generator, final SerializerProvider provider) throws IOException {
+            generator.writeStartObject();
+            generator.writeNumberField(Constants.Properties.EncryptionFormatVersion, value.encryptionFormatVersion);
+            generator.writeStringField(Constants.Properties.EncryptionAlgorithm, value.encryptionAlgorithm);
+            generator.writeStringField(Constants.Properties.DataEncryptionKeyId, value.dataEncryptionKeyId);
+            generator.writeBinaryField(Constants.Properties.EncryptedData, value.encryptedData);
+            generator.writeEndObject();
+        }
+    }
+
+    static final class JsonDeserializer extends StdDeserializer {
+        private static final long serialVersionUID = 4L;
+
+        public JsonDeserializer() {
+            super(EncryptionProperties.class);
+        }
+
+        @Override
+        public EncryptionProperties deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JsonProcessingException {
+
+            ObjectCodec objectCodec = jsonParser.getCodec();
+            JsonNode root;
+            try {
+                root = objectCodec.readTree(jsonParser);
+            } catch (IOException e) {
+                throw new IllegalArgumentException(e);
+            }
+
+            validateOrThrow(jsonParser, root.isObject(), "can't deserialize");
+
+            EncryptionProperties encryptionProperties = new EncryptionProperties();
+
+            JsonNode node = root.get(Constants.Properties.EncryptionFormatVersion);
+            Preconditions.checkNotNull(node, Constants.Properties.EncryptionFormatVersion +  "can't deserialize");
+            validateOrThrow(jsonParser, node.isInt(), Constants.Properties.EncryptionFormatVersion, "can't deserialize");
+            encryptionProperties.encryptionFormatVersion = node.asInt();
+
+            node = root.get(Constants.Properties.EncryptionAlgorithm);
+            Preconditions.checkNotNull(node, Constants.Properties.EncryptionAlgorithm +  "can't deserialize");
+            validateOrThrow(jsonParser, node.isTextual(), Constants.Properties.EncryptionAlgorithm, "can't deserialize");
+            encryptionProperties.encryptionAlgorithm = node.asText();
+
+            node = root.get(Constants.Properties.DataEncryptionKeyId);
+            Preconditions.checkNotNull(node, Constants.Properties.DataEncryptionKeyId +  "can't deserialize");
+            validateOrThrow(jsonParser, node.isTextual(), Constants.Properties.DataEncryptionKeyId, "can't deserialize");
+            encryptionProperties.dataEncryptionKeyId = node.asText();
+
+            node = root.get(Constants.Properties.EncryptedData);
+            Preconditions.checkNotNull(node, Constants.Properties.EncryptedData +  "can't deserialize");
+            validateOrThrow(jsonParser, node.isBinary() || node.isTextual(), Constants.Properties.EncryptedData, "can't deserialize");
+            encryptionProperties.encryptedData = node.binaryValue();
+
+            return encryptionProperties;
+        }
+
+        private void validateOrThrow(JsonParser jsonParser, boolean expectedToBeTrue, String msg) throws JsonProcessingException {
+            validateOrThrow(jsonParser, expectedToBeTrue, null, msg);
+        }
+
+        private void validateOrThrow(JsonParser jsonParser, boolean expectedToBeTrue, String fieldName, String msg) throws JsonProcessingException {
+            if (!expectedToBeTrue) {
+                if (fieldName == null) {
+                    throw new JsonParseException(jsonParser, msg);
+                } else {
+                    throw new JsonParseException(jsonParser, fieldName + " : " + msg);
+                }
+            }
+        }
+    }
+}
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/CosmosEncryptionAlgorithm.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/CosmosEncryptionAlgorithm.java
new file mode 100644
index 000000000000..ebec258d5edb
--- /dev/null
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/CosmosEncryptionAlgorithm.java
@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation.encryption.api;
+
+// TODO: enum string type?
+
+/**
+ * Algorithms for use with client-side encryption support in Azure Cosmos DB.
+ */
+public class CosmosEncryptionAlgorithm {
+
+    /**
+     * Authenticated Encryption algorithm based on https://tools.ietf.org/html/draft-mcgrew-aead-aes-cbc-hmac-sha2-05
+     */
+    public static final String AEAes256CbcHmacSha256Randomized = "AEAes256CbcHmacSha256Randomized";
+
+}
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/DataEncryptionKey.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/DataEncryptionKey.java
new file mode 100644
index 000000000000..63e5957924d9
--- /dev/null
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/DataEncryptionKey.java
@@ -0,0 +1,96 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation.encryption.api;
+
+import com.azure.cosmos.implementation.apachecommons.lang.StringUtils;
+
+import java.lang.reflect.InvocationTargetException;
+
+/**
+ * Abstraction for a data encryption key for use in client-side encryption.
+ * See https://aka.ms/CosmosClientEncryption for more information on client-side encryption support in Azure Cosmos DB.
+ */
+public interface DataEncryptionKey {
+
+    /**
+     * Gets Raw key bytes of the data encryption key.
+     * @return
+     */
+    byte[] getRawKey();
+
+
+    /**
+     * Gets Encryption algorithm to be used with this data encryption key.
+     * @return encryption algorithm.
+     */
+    String getEncryptionAlgorithm();
+
+    /**
+     * Encrypts the plainText with a data encryption key.
+     * @param plainText >Plain text value to be encrypted.
+     * @return encrypted data.
+     */
+    byte[] encryptData(byte[] plainText);
+
+    /**
+     * Decrypts the cipherText with a data encryption key.
+     * @param cipherText Ciphertext value to be decrypted.
+     * @return Plain text.
+     */
+    byte[] decryptData(byte[] cipherText);
+
+    /**
+     * Generates raw data encryption key bytes suitable for use with the provided encryption algorithm.
+     * @param encryptionAlgorithm Encryption algorithm the returned key is intended to be used with.
+     * @return New instance of data encryption key.
+     */
+    static byte[] generate(String encryptionAlgorithm) {
+        if (!StringUtils.equals(encryptionAlgorithm, CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized)) {
+            throw new IllegalArgumentException(String.format("Encryption algorithm not supported: {%s}", encryptionAlgorithm));
+        }
+
+        byte[] rawKey = new byte[32];
+
+        try {
+            DataEncryptionKey.class.getClassLoader()
+                .loadClass("com.azure.cosmos.implementation.encryption.AeadAes256CbcHmac256AlgorithmProvider")
+                .getDeclaredMethod("generateRandomBytes", byte[].class)
+                .invoke(null, rawKey);
+
+        } catch (IllegalAccessException|InvocationTargetException|NoSuchMethodException|ClassNotFoundException e) {
+            throw new IllegalStateException("azure-cosmos-encryption is not in the classpath");
+        }
+
+        return rawKey;
+    }
+
+    /**
+     * Creates a new instance of data encryption key given the raw key bytes
+     * suitable for use with the provided encryption algorithm.
+     * @param rawKey Raw key bytes.
+     * @param encryptionAlgorithm Encryption algorithm the returned key is intended to be used with.
+     * @return New instance of data encryption key.
+     */
+    static DataEncryptionKey create(
+        byte[] rawKey,
+        String encryptionAlgorithm) {
+        if (rawKey == null) {
+            throw new NullPointerException("rawKey");
+        }
+
+        if (!StringUtils.equals(encryptionAlgorithm, CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized)) {
+            throw new IllegalArgumentException(String.format("Encryption algorithm not supported: {%s}", encryptionAlgorithm));
+        }
+
+        try {
+            return (DataEncryptionKey) DataEncryptionKey.class.getClassLoader()
+                .loadClass("com.azure.cosmos.implementation.encryption.AeadAes256CbcHmac256AlgorithmProvider")
+                .getDeclaredMethod("createAlgorithm", byte[].class, EncryptionType.class, byte.class)
+                .invoke(null, rawKey, EncryptionType.RANDOMIZED, /** algorithmVersion **/(byte) 1);
+
+        } catch (IllegalAccessException|InvocationTargetException|NoSuchMethodException|ClassNotFoundException e) {
+            throw new IllegalStateException("azure-cosmos-encryption is not in the classpath");
+        }
+    }
+}
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/DataEncryptionKeyProvider.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/DataEncryptionKeyProvider.java
new file mode 100644
index 000000000000..15a8973d0a2b
--- /dev/null
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/DataEncryptionKeyProvider.java
@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation.encryption.api;
+
+/**
+ * Abstraction for a provider to get data encryption keys for use in client-side encryption.
+ * See https://aka.ms/CosmosClientEncryption for more information on client-side encryption support in Azure Cosmos DB.
+ */
+public interface DataEncryptionKeyProvider {
+
+    /**
+     * Retrieves the data encryption key for the given id.
+     * @param id Identifier of the data encryption key.
+     * @param encryptionAlgorithm Encryption algorithm that the retrieved key will be used with.
+     * @return Data encryption key bytes.
+     * TODO: @moderakh look into if this method needs to be async.
+     */
+    DataEncryptionKey getDataEncryptionKey(
+        String id,
+        String encryptionAlgorithm);
+}
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/EncryptionOptions.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/EncryptionOptions.java
new file mode 100644
index 000000000000..8b0ef7e1292d
--- /dev/null
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/EncryptionOptions.java
@@ -0,0 +1,73 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation.encryption.api;
+
+import java.util.List;
+
+public class EncryptionOptions {
+
+
+    /**
+     *  Gets Algorithm to be used for encrypting the data in the request payload.
+     * @return
+     */
+    public String getEncryptionAlgorithm() {
+        return encryptionAlgorithm;
+    }
+
+
+    /**
+     * Sets Algorithm to be used for encrypting the data in the request payload.
+     * @param encryptionAlgorithm
+     * @return
+     */
+    public EncryptionOptions setEncryptionAlgorithm(String encryptionAlgorithm) {
+        this.encryptionAlgorithm = encryptionAlgorithm;
+        return this;
+    }
+
+
+    /**
+     * Gets Identifier of the data encryption key to be used for encrypting the data in the request payload.
+     * The data encryption key must be suitable for use with the EncryptionAlgorithm provided.
+     * @return data encryption key id.
+     */
+    public String getDataEncryptionKeyId() {
+        return dataEncryptionKeyId;
+    }
+
+    public EncryptionOptions setDataEncryptionKeyId(String dataEncryptionKeyId) {
+        this.dataEncryptionKeyId = dataEncryptionKeyId;
+        return this;
+    }
+
+    public List getPathsToEncrypt() {
+        return pathsToEncrypt;
+    }
+
+    public EncryptionOptions setPathsToEncrypt(List pathsToEncrypt) {
+        this.pathsToEncrypt = pathsToEncrypt;
+        return this;
+    }
+
+    /**
+     * Identifier of the data encryption key to be used for encrypting the data in the request payload.
+     * The data encryption key must be suitable for use with the {@link com.azure.cosmos.implementation.encryption.api.DataEncryptionKey} provided.
+     * 
    
+     * The {@link DataEncryptionKeyProvider} configured on the client is used to retrieve the actual data encryption key.
+     */
+    private String dataEncryptionKeyId;
+
+    /**
+     * For the request payload, list of JSON paths to encrypt.
+     * Only top level paths are supported.
+     * Example of a path specification: /sensitive
+     */
+    private List pathsToEncrypt;
+
+    /**
+     * Algorithm to be used for encrypting the data in the request payload.
+     */
+    private String encryptionAlgorithm;
+}
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/EncryptionType.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/EncryptionType.java
new file mode 100644
index 000000000000..0cedd828ef67
--- /dev/null
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/api/EncryptionType.java
@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation.encryption.api;
+
+
+/**
+ * Encryption types that may be supported.
+ */
+public enum EncryptionType {
+    RANDOMIZED;
+}
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosItemRequestOptions.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosItemRequestOptions.java
index 4f56ebcdb94a..d38952791701 100644
--- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosItemRequestOptions.java
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosItemRequestOptions.java
@@ -4,6 +4,7 @@
 
 import com.azure.cosmos.ConsistencyLevel;
 import com.azure.cosmos.implementation.RequestOptions;
+import com.azure.cosmos.implementation.encryption.api.EncryptionOptions;
 
 import java.util.List;
 
@@ -17,6 +18,7 @@ public final class CosmosItemRequestOptions {
     private List postTriggerInclude;
     private String sessionToken;
     private PartitionKey partitionKey;
+    private EncryptionOptions encryptionOptions;
     private String ifMatchETag;
     private String ifNoneMatchETag;
 
@@ -209,6 +211,12 @@ RequestOptions toRequestOptions() {
         requestOptions.setPostTriggerInclude(postTriggerInclude);
         requestOptions.setSessionToken(sessionToken);
         requestOptions.setPartitionKey(partitionKey);
+        requestOptions.setEncryptionOptions(encryptionOptions);
         return requestOptions;
     }
+
+    CosmosItemRequestOptions setEncryptionOptions(EncryptionOptions options) {
+        this.encryptionOptions = options;
+        return this;
+    }
 }
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosItemResponse.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosItemResponse.java
index 1149aee36953..74519217a18f 100644
--- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosItemResponse.java
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/CosmosItemResponse.java
@@ -6,6 +6,7 @@
 import com.azure.cosmos.CosmosDiagnostics;
 import com.azure.cosmos.implementation.InternalObjectNode;
 import com.azure.cosmos.implementation.Document;
+import com.azure.cosmos.implementation.ItemDeserializer;
 import com.azure.cosmos.implementation.ResourceResponse;
 import com.azure.cosmos.implementation.SerializationDiagnosticsContext;
 import com.azure.cosmos.implementation.Utils;
@@ -22,14 +23,16 @@
 public class CosmosItemResponse {
     private final Class itemClassType;
     private final byte[] responseBodyAsByteArray;
+    private final ItemDeserializer itemDeserializer;
     private T item;
     private final ResourceResponse resourceResponse;
     private InternalObjectNode props;
 
-    CosmosItemResponse(ResourceResponse response, Class classType) {
+    CosmosItemResponse(ResourceResponse response, Class classType, ItemDeserializer itemDeserializer) {
         this.itemClassType = classType;
         this.responseBodyAsByteArray = response.getBodyAsByteArray();
         this.resourceResponse = response;
+        this.itemDeserializer = itemDeserializer;
     }
 
     /**
@@ -61,7 +64,7 @@ public T getItem() {
             synchronized (this) {
                 if (item == null && !Utils.isEmpty(responseBodyAsByteArray)) {
                     Instant serializationStartTime = Instant.now();
-                    item = Utils.parse(responseBodyAsByteArray, itemClassType);
+                    item = Utils.parse(responseBodyAsByteArray, itemClassType, itemDeserializer);
                     Instant serializationEndTime = Instant.now();
                     SerializationDiagnosticsContext.SerializationDiagnostics diagnostics = new SerializationDiagnosticsContext.SerializationDiagnostics(
                         serializationStartTime,
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java
index 18b644710772..b89f6c72a2a5 100644
--- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java
@@ -33,6 +33,8 @@
 import com.azure.cosmos.implementation.Utils;
 import com.azure.cosmos.implementation.Warning;
 import com.azure.cosmos.implementation.directconnectivity.Address;
+import com.azure.cosmos.implementation.ItemDeserializer;
+import com.azure.cosmos.implementation.encryption.api.EncryptionOptions;
 import com.azure.cosmos.implementation.query.PartitionedQueryExecutionInfoInternal;
 import com.azure.cosmos.implementation.query.QueryInfo;
 import com.azure.cosmos.implementation.query.QueryItem;
@@ -79,13 +81,13 @@ public static CosmosDatabaseResponse createCosmosDatabaseResponse(ResourceRespon
     }
 
     @Warning(value = INTERNAL_USE_ONLY_WARNING)
-    public static  CosmosItemResponse createCosmosAsyncItemResponse(ResourceResponse response, Class classType) {
-        return new CosmosItemResponse<>(response, classType);
+    public static  CosmosItemResponse createCosmosAsyncItemResponse(ResourceResponse response, Class classType, ItemDeserializer itemDeserializer) {
+        return new CosmosItemResponse<>(response, classType, itemDeserializer);
     }
 
     @Warning(value = INTERNAL_USE_ONLY_WARNING)
     public static CosmosItemResponse createCosmosAsyncItemResponseWithObjectType(ResourceResponse response) {
-        return new CosmosItemResponse<>(response, Object.class);
+        return new CosmosItemResponse<>(response, Object.class, null);
     }
 
     @Warning(value = INTERNAL_USE_ONLY_WARNING)
@@ -610,6 +612,12 @@ public static  JsonSerializable getJsonSerializable(T t) {
         }
     }
 
+    public static CosmosItemRequestOptions setEncryptionOptions(CosmosItemRequestOptions options,
+                                                                EncryptionOptions encryptionOptions) {
+        options.setEncryptionOptions(encryptionOptions);
+        return options;
+    }
+
     @Warning(value = INTERNAL_USE_ONLY_WARNING)
     public static  Resource getResource(T t) {
         if (t == null) {
diff --git a/sdk/cosmos/azure-cosmos/src/main/java/module-info.java b/sdk/cosmos/azure-cosmos/src/main/java/module-info.java
index 553cca6bf71f..7b2c9380962a 100644
--- a/sdk/cosmos/azure-cosmos/src/main/java/module-info.java
+++ b/sdk/cosmos/azure-cosmos/src/main/java/module-info.java
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+
 module com.azure.cosmos {
 
     requires transitive com.azure.core;
@@ -46,6 +47,7 @@
     opens com.azure.cosmos.implementation.query.orderbyquery to com.fasterxml.jackson.databind;
     opens com.azure.cosmos.implementation.routing to com.fasterxml.jackson.databind;
     opens com.azure.cosmos.models to com.fasterxml.jackson.databind;
+    opens com.azure.cosmos.implementation.encryption to com.fasterxml.jackson.databind;
     opens com.azure.cosmos.util to com.fasterxml.jackson.databind;
 
     uses com.azure.cosmos.implementation.guava25.base.PatternCompiler;
diff --git a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosDatabaseTest.java b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosDatabaseTest.java
index 05a48f263026..893d1c029101 100644
--- a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosDatabaseTest.java
+++ b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosDatabaseTest.java
@@ -93,6 +93,7 @@ public void createDatabase_alreadyExists() throws Exception {
     @Test(groups = {"emulator"}, timeOut = TIMEOUT)
     public void createDatabase_withId() throws Exception {
         CosmosDatabaseProperties databaseDefinition = new CosmosDatabaseProperties(CosmosDatabaseForTest.generateId());
+        databases.add(databaseDefinition.getId());
 
         CosmosDatabaseResponse createResponse = client.createDatabase(databaseDefinition.getId());
         validateDatabaseResponse(databaseDefinition, createResponse);
@@ -101,6 +102,7 @@ public void createDatabase_withId() throws Exception {
     @Test(groups = {"emulator"}, timeOut = TIMEOUT)
     public void createDatabase_withPropertiesThroughputAndOptions() throws Exception {
         CosmosDatabaseProperties databaseDefinition = new CosmosDatabaseProperties(CosmosDatabaseForTest.generateId());
+        databases.add(databaseDefinition.getId());
         CosmosDatabaseProperties databaseProperties = new CosmosDatabaseProperties(databaseDefinition.getId());
         CosmosDatabaseRequestOptions requestOptions = new CosmosDatabaseRequestOptions();
         int throughput = 400;
@@ -115,6 +117,8 @@ public void createDatabase_withPropertiesThroughputAndOptions() throws Exception
     @Test(groups = {"emulator"}, timeOut = TIMEOUT)
     public void createDatabase_withPropertiesAndThroughput() throws Exception {
         CosmosDatabaseProperties databaseDefinition = new CosmosDatabaseProperties(CosmosDatabaseForTest.generateId());
+        databases.add(databaseDefinition.getId());
+
         CosmosDatabaseProperties databaseProperties = new CosmosDatabaseProperties(databaseDefinition.getId());
         int throughput = 1000;
         try {
@@ -132,6 +136,7 @@ public void createDatabase_withPropertiesAndThroughput() throws Exception {
     @Test(groups = {"emulator"}, timeOut = TIMEOUT)
     public void createDatabase_withIdAndThroughput() throws Exception {
         CosmosDatabaseProperties databaseDefinition = new CosmosDatabaseProperties(CosmosDatabaseForTest.generateId());
+        databases.add(databaseDefinition.getId());
         int throughput = 1000;
         try {
             CosmosDatabaseResponse createResponse = client.createDatabase(databaseDefinition.getId(), ThroughputProperties.createManualThroughput(throughput));
@@ -191,6 +196,7 @@ public void queryAllDatabases() throws Exception {
     @Test(groups = {"emulator"}, timeOut = TIMEOUT)
     public void deleteDatabase() throws Exception {
         CosmosDatabaseProperties databaseDefinition = new CosmosDatabaseProperties(CosmosDatabaseForTest.generateId());
+        databases.add(databaseDefinition.getId());
         CosmosDatabaseProperties databaseProperties = new CosmosDatabaseProperties(databaseDefinition.getId());
         CosmosDatabaseResponse createResponse = client.createDatabase(databaseProperties);
 
@@ -200,6 +206,7 @@ public void deleteDatabase() throws Exception {
     @Test(groups = {"emulator"}, timeOut = TIMEOUT)
     public void deleteDatabase_withOptions() throws Exception {
         CosmosDatabaseProperties databaseDefinition = new CosmosDatabaseProperties(CosmosDatabaseForTest.generateId());
+        databases.add(databaseDefinition.getId());
         CosmosDatabaseProperties databaseProperties = new CosmosDatabaseProperties(databaseDefinition.getId());
         CosmosDatabaseResponse createResponse = client.createDatabase(databaseProperties);
         CosmosDatabaseRequestOptions options = new CosmosDatabaseRequestOptions();
diff --git a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/rx/CollectionCrudTest.java b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/rx/CollectionCrudTest.java
index 1b05b05355b7..de0aeb006891 100644
--- a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/rx/CollectionCrudTest.java
+++ b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/rx/CollectionCrudTest.java
@@ -361,10 +361,6 @@ public void sessionTokenConsistencyCollectionDeleteCreateSameName() {
 
     @Test(groups = {"emulator"}, timeOut = TIMEOUT)
     public void replaceProvisionedThroughput(){
-        final String databaseName = CosmosDatabaseForTest.generateId();
-        client.createDatabase(databaseName).block();
-        CosmosAsyncDatabase database = client.getDatabase(databaseName);
-
         CosmosContainerProperties containerProperties = new CosmosContainerProperties("testCol", "/myPk");
         database.createContainer(
             containerProperties,
diff --git a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/rx/ContainerCreateDeleteWithSameNameTest.java b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/rx/ContainerCreateDeleteWithSameNameTest.java
index 24380fcdb4f3..6d1691d2e671 100644
--- a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/rx/ContainerCreateDeleteWithSameNameTest.java
+++ b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/rx/ContainerCreateDeleteWithSameNameTest.java
@@ -46,7 +46,7 @@ public class ContainerCreateDeleteWithSameNameTest extends TestSuiteBase {
     private final static int TIMEOUT = 300000;
     // Delete collections in emulator is not instant,
     // so to avoid get 500 back, we are adding delay for creating the collection with same name, since in this case we want to test 410/1000
-    private final static int COLLECTION_RECREATION_TIME_DELAY = 2000;
+    private final static int COLLECTION_RECREATION_TIME_DELAY = 5000;
     private CosmosAsyncClient client;
     private CosmosAsyncDatabase createdDatabase;
 
@@ -146,7 +146,6 @@ public  void upsertItem() throws Exception {
 
     @Test(groups = {"emulator"}, timeOut = TIMEOUT)
     public  void changeFeed() throws Exception {
-
         ObjectMapper objectMapper = Utils.getSimpleObjectMapper();
         BiConsumer func = (feedContainer, leaseContainer) -> {
             String hostName = RandomStringUtils.randomAlphabetic(6);
@@ -187,12 +186,8 @@ public  void changeFeed() throws Exception {
                 changeFeedProcessor.start().subscribeOn(Schedulers.elastic())
                     .timeout(Duration.ofMillis(2 * CHANGE_FEED_PROCESSOR_TIMEOUT))
                     .subscribe();
-            } catch (Exception ex) {
-                throw ex;
-            }
 
-            // Wait for the feed processor to receive and process the documents.
-            try {
+                // Wait for the feed processor to receive and process the documents.
                 Thread.sleep(2 * CHANGE_FEED_PROCESSOR_TIMEOUT);
                 assertThat(changeFeedProcessor.isStarted()).as("Change Feed Processor instance is running").isTrue();
 
@@ -203,14 +198,16 @@ public  void changeFeed() throws Exception {
                 }
 
                 assertThat(remainingWork >= 0).as("Failed to receive all the feed documents").isTrue();
-
+            } catch (InterruptedException e) {
+                throw new RuntimeException("Interrupted exception", e);
+            } finally {
                 changeFeedProcessor.stop().subscribeOn(Schedulers.elastic()).timeout(Duration.ofMillis(CHANGE_FEED_PROCESSOR_TIMEOUT)).subscribe();
 
                 // Wait for the feed processor to shutdown.
-                Thread.sleep(CHANGE_FEED_PROCESSOR_TIMEOUT);
-
-            } catch (InterruptedException e) {
-                throw new RuntimeException("Interrupted exception", e);
+                try {
+                    Thread.sleep(CHANGE_FEED_PROCESSOR_TIMEOUT);
+                } catch (InterruptedException e) {
+                }
             }
         };
 
diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml
index 4b65101fddcd..82e01edc6920 100644
--- a/sdk/cosmos/ci.yml
+++ b/sdk/cosmos/ci.yml
@@ -46,6 +46,10 @@ extends:
       - name: azure-spring-data-cosmos
         groupId: com.azure
         safeName: azurespringdatacosmos
+      - name: azure-cosmos-encryption
+        groupId: com.azure
+        safeName: azurecosmosencryption
     AdditionalModules:
       - name: azure-cosmos-benchmark
         groupId: com.azure
+
diff --git a/sdk/cosmos/pom.xml b/sdk/cosmos/pom.xml
index 025f5523dfa2..71650c9f1bd1 100644
--- a/sdk/cosmos/pom.xml
+++ b/sdk/cosmos/pom.xml
@@ -11,6 +11,7 @@
   
     azure-cosmos
     azure-cosmos-benchmark
+    azure-cosmos-encryption
     azure-spring-data-cosmos