diff --git a/eng/pipelines/templates/stages/cosmos-sdk-client.yml b/eng/pipelines/templates/stages/cosmos-sdk-client.yml index d31b37c199e8..81382ae03867 100644 --- a/eng/pipelines/templates/stages/cosmos-sdk-client.yml +++ b/eng/pipelines/templates/stages/cosmos-sdk-client.yml @@ -107,6 +107,14 @@ stages: ProfileFlag: '-P integration-test-emulator' DisplayName: 'Spring Emulator only Integration Tests' AdditionalArgs: '-DargLine="-DACCOUNT_HOST=https://localhost:8081/ -DACCOUNT_KEY=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw== -DSECONDARY_ACCOUNT_KEY=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="' + Encryption_Integration_Tests_Java8: + OSVmImage: 'windows-2019' + JavaTestVersion: '1.8' + ProfileFlag: '-Pencryption-integration' + DisplayName: 'Encryption Emulator only Integration Tests' + PROTOCOLS: '["Tcp"]' + DESIRED_CONSISTENCIES: '["Session"]' + AdditionalArgs: '-DargLine="-DACCOUNT_HOST=https://localhost:8081/"' TestStepMavenInputs: goals: 'verify' diff --git a/sdk/cosmos/azure-cosmos-encryption/README.md b/sdk/cosmos/azure-cosmos-encryption/README.md index 341592e2250d..05f45bb79799 100644 --- a/sdk/cosmos/azure-cosmos-encryption/README.md +++ b/sdk/cosmos/azure-cosmos-encryption/README.md @@ -1,5 +1,5 @@ -# Azure CosmosDB client library for Java -TODO +# Encryption plugin library for Azure Cosmos DB Java SDK for Java +The encryption plugin is under development and not ready to be consumed yet. ## Getting started ### Include the package diff --git a/sdk/cosmos/azure-cosmos-encryption/pom.xml b/sdk/cosmos/azure-cosmos-encryption/pom.xml index 0467daae1538..3447105d62bb 100644 --- a/sdk/cosmos/azure-cosmos-encryption/pom.xml +++ b/sdk/cosmos/azure-cosmos-encryption/pom.xml @@ -14,8 +14,8 @@ Licensed under the MIT License. com.azure azure-cosmos-encryption 1.0.0-beta.1 - Microsoft Azure SDK for SQL API of Azure Cosmos DB Service - This Package contains Microsoft Azure Cosmos SDK (with Reactive Extension Reactor support) for Azure Cosmos DB SQL API + Encryption Plugin for Azure Cosmos DB SDK + This Package contains Encryption Plugin for Microsoft Azure Cosmos SDK jar https://github.com/Azure/azure-sdk-for-java @@ -202,7 +202,6 @@ Licensed under the MIT License. com.fasterxml.jackson.core:jackson-databind:[2.10.1] com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.10.1] com.fasterxml.jackson.module:jackson-module-afterburner:[2.10.1] - com.google.code.findbugs:jsr305:[3.0.2] io.dropwizard.metrics:metrics-core:[4.1.0] io.micrometer:micrometer-core:[1.2.0] io.netty:netty-codec-http:[4.1.49.Final] @@ -246,167 +245,11 @@ Licensed under the MIT License. - - - fast - - simple,cosmosv3 - - - - - org.apache.maven.plugins - maven-failsafe-plugin - 2.22.0 - - - src/test/resources/fast-testng.xml - - - - - - - - - long - - long - - - - - org.apache.maven.plugins - maven-failsafe-plugin - 2.22.0 - - - src/test/resources/long-testng.xml - - - - - - - - - direct - - direct - - - - - org.apache.maven.plugins - maven-failsafe-plugin - 2.22.0 - - - src/test/resources/direct-testng.xml - - - - - - - - - multi-master - - multi-master - - - - - org.apache.maven.plugins - maven-failsafe-plugin - 2.22.0 - - - src/test/resources/multi-master-testng.xml - - - - - - - - - examples - - - samples,examples - - - - - org.apache.maven.plugins - maven-failsafe-plugin - 2.22.0 - - - src/test/resources/examples-testng.xml - - - - - - integration-test - verify - - - - - - - - - - emulator - - emulator - - - - - org.apache.maven.plugins - maven-failsafe-plugin - 2.22.0 - - - src/test/resources/emulator-testng.xml - - - - - - - non-emulator - - non-emulator - - - - - org.apache.maven.plugins - maven-failsafe-plugin - 2.22.0 - - - src/test/resources/non-emulator-testng.xml - - - - - - - - - e2e + encryption-integration - e2e + encryption @@ -416,7 +259,7 @@ Licensed under the MIT License. 2.22.0 - src/test/resources/e2e-testng.xml + src/test/resources/encryption-testng.xml diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256Algorithm.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256Algorithm.java index 5ee1be227aa1..1d5a4317eb7f 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256Algorithm.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/AeadAes256CbcHmac256Algorithm.java @@ -18,7 +18,7 @@ * SQL client implementation with StyleCop related changes - also, we restrict to randomized encryption to start with. */ class AeadAes256CbcHmac256Algorithm implements DataEncryptionKey { - + // TODO: moderakh is there any other library which we can use? public final static String ALGORITHM_NAME = "AEAD_AES_256_CBC_HMAC_SHA256"; /** diff --git a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/CosmosDataEncryptionKeyProvider.java b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/CosmosDataEncryptionKeyProvider.java index d37b34d5593d..8fa62015fd2c 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/CosmosDataEncryptionKeyProvider.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/main/java/com/azure/cosmos/implementation/encryption/CosmosDataEncryptionKeyProvider.java @@ -19,6 +19,9 @@ public class CosmosDataEncryptionKeyProvider implements DataEncryptionKeyProvider { // TODO: proper sample and documentation on container private static final String ContainerPartitionKeyPath = "/id"; + // TODO: moderakh + // Is it a requirement on container definition? + //Then the code-docs and samples needs to explicit cover it. private final DataEncryptionKeyContainerCore dataEncryptionKeyContainerCore; private final DekCache DekCache; private final EncryptionKeyWrapProvider EncryptionKeyWrapProvider; diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/EncryptionTest2.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/EncryptionTest2.java index a77bf5d3abc1..b7a117f724cb 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/EncryptionTest2.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/EncryptionTest2.java @@ -43,7 +43,7 @@ public EncryptionTest2(CosmosClientBuilder clientBuilder) { super(CosmosBridgeInternal.setDateKeyProvider(clientBuilder, simpleInMemoryProvider)); } - @BeforeClass(groups = {"emulator"}, timeOut = SETUP_TIMEOUT) + @BeforeClass(groups = {"encryption"}, timeOut = SETUP_TIMEOUT) public void before_CosmosItemTest() { assertThat(this.client).isNull(); this.client = getClientBuilder().buildClient(); @@ -51,18 +51,18 @@ public void before_CosmosItemTest() { container = client.getDatabase(asyncContainer.getDatabase().getId()).getContainer(asyncContainer.getId()); } - @BeforeClass(groups = "emulator") + @BeforeClass(groups = "encryption") public void beforeClass() { TestUtils.initialized(); } - @AfterClass(groups = {"emulator"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) + @AfterClass(groups = {"encryption"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true) public void afterClass() { assertThat(this.client).isNotNull(); this.client.close(); } - @Test(groups = {"emulator"}, timeOut = TIMEOUT) + @Test(groups = {"encryption"}, timeOut = TIMEOUT) public void createItemEncrypt_readItemDecrypt() throws Exception { CosmosItemRequestOptions requestOptions = new CosmosItemRequestOptions(); EncryptionOptions encryptionOptions = new EncryptionOptions(); @@ -88,7 +88,7 @@ public void createItemEncrypt_readItemDecrypt() throws Exception { validateReadResponseIsValid(properties, readItem); } - @Test(groups = {"emulator"}, timeOut = TIMEOUT) + @Test(groups = {"encryption"}, timeOut = TIMEOUT) public void upsertItem_readItem() throws Exception { CosmosItemRequestOptions requestOptions = new CosmosItemRequestOptions(); EncryptionOptions encryptionOptions = new EncryptionOptions(); @@ -135,7 +135,7 @@ private void validateQueryResponseIsValid(Pojo originalItem, Pojo result) { assertThat(result.sensitive).isNull(); } - @Test(groups = {"emulator"}, timeOut = TIMEOUT) + @Test(groups = {"encryption"}, timeOut = TIMEOUT) public void readItem() throws Exception { Pojo properties = getItem(UUID.randomUUID().toString()); CosmosItemResponse itemResponse = container.createItem(properties); @@ -147,7 +147,7 @@ public void readItem() throws Exception { validateItemResponse(properties, readResponse1); } - @Test(groups = {"emulator"}, timeOut = TIMEOUT) + @Test(groups = {"encryption"}, timeOut = TIMEOUT) public void readAllItems() throws Exception { Pojo properties = getItem(UUID.randomUUID().toString()); CosmosItemResponse itemResponse = container.createItem(properties); @@ -160,7 +160,7 @@ public void readAllItems() throws Exception { } - @Test(groups = {"emulator"}, timeOut = TIMEOUT) + @Test(groups = {"encryption"}, timeOut = TIMEOUT) public void queryItems() throws Exception { Pojo properties = getItem(UUID.randomUUID().toString()); CosmosItemResponse itemResponse = container.createItem(properties); @@ -179,7 +179,7 @@ public void queryItems() throws Exception { assertThat(feedResponseIterator3.iterator().hasNext()).isTrue(); } - @Test(groups = {"emulator"}, timeOut = TIMEOUT) + @Test(groups = {"encryption"}, timeOut = TIMEOUT) public void queryItemsWithContinuationTokenAndPageSize() throws Exception { List actualIds = new ArrayList<>(); Pojo properties = getItem(UUID.randomUUID().toString()); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/DecryptDataEncryptedByDotNetTest.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/DecryptDataEncryptedByDotNetTest.java index f815acc58aa1..372120586028 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/DecryptDataEncryptedByDotNetTest.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/DecryptDataEncryptedByDotNetTest.java @@ -55,22 +55,22 @@ public DecryptDataEncryptedByDotNetTest(CosmosClientBuilder clientBuilder) { super(clientBuilder); } - @BeforeClass(groups = "emulator") + @BeforeClass(groups = "encryption") public void beforeClass() { TestUtils.initialized(); } - @AfterClass(groups = "emulator") + @AfterClass(groups = "encryption") public void afterClass() { safeDeleteDatabase(databaseCore); } - @AfterMethod(groups = "emulator") + @AfterMethod(groups = "encryption") public void afterMethod() { safeClose(client); } - @BeforeMethod(groups = "emulator") + @BeforeMethod(groups = "encryption") public void beforeMethod() { keyWrapProvider = new TestKeyWrapProvider(); dekProvider = new CosmosDataEncryptionKeyProvider(keyWrapProvider); @@ -88,7 +88,7 @@ public void beforeMethod() { dekProvider.initialize(databaseCore, keyContainer.getId()); } - @Test(groups = "emulator") + @Test(groups = "encryption") public void canReadKeyEncryptionKeyGeneratedByDotNet() throws Exception { // add key generated by dotnet ObjectNode dataEncryptionKeyProperties = TestUtils.loadPojo("./encryption/dotnet/DataEncryptionKeyProperties.json", ObjectNode.class); @@ -106,7 +106,7 @@ public void canReadKeyEncryptionKeyGeneratedByDotNet() throws Exception { assertThat(loadedKey.getRawKey()).isEqualTo(expectedUnWrappedKey.getDataEncryptionKey()); } - @Test(groups = "emulator") + @Test(groups = "encryption") public void canDecryptDataEncryptedByDotNet() throws Exception { // add key generated by dotnet DataEncryptionKeyProperties dataEncryptionKeyProperties = TestUtils.loadPojo("./encryption/dotnet/DataEncryptionKeyProperties.json", DataEncryptionKeyProperties.class); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionTests.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionTests.java index bc7590e22ac1..7eb5ab7c239f 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionTests.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/implementation/encryption/EncryptionTests.java @@ -62,7 +62,7 @@ public EncryptionTests(CosmosClientBuilder clientBuilder) { super(clientBuilder); } - @BeforeTest(groups = {"emulator"}) + @BeforeTest(groups = {"encryption"}) public void beforeTest() { TestKeyWrapProvider keyWrapProvider = new TestKeyWrapProvider(); dekProvider = new CosmosDataEncryptionKeyProvider(keyWrapProvider); @@ -84,18 +84,18 @@ public void beforeTest() { EncryptionTests.dekProperties = EncryptionTests.createDek(EncryptionTests.dekProvider, dekId); } - @BeforeClass(groups = {"emulator"}) + @BeforeClass(groups = {"encryption"}) public void beforeClass() { TestUtils.initialized(); client = getClientBuilder().buildAsyncClient(); } - @AfterMethod(groups = {"emulator"}) + @AfterMethod(groups = {"encryption"}) public void afterTest() { safeClose(client); } - @AfterClass(groups = {"emulator"}) + @AfterClass(groups = {"encryption"}) public void afterClass() { safeDeleteDatabase(databaseCore); } @@ -138,7 +138,7 @@ public int hashCode() { } } - @Test(groups = {"emulator"}) + @Test(groups = {"encryption"}) public void encryptionCreateDek() { String dekId = "anotherDek"; DataEncryptionKeyProperties dekProperties = EncryptionTests.createDek(EncryptionTests.dekProvider, dekId); @@ -167,7 +167,7 @@ public void encryptionCreateDek() { assertThat(dekProperties).isEqualTo(readProperties); } - @Test(groups = {"emulator"}, timeOut = TIMEOUT) + @Test(groups = {"encryption"}, timeOut = TIMEOUT) public void createItemEncrypt_readItemDecrypt() throws Exception { CosmosItemRequestOptions requestOptions = new CosmosItemRequestOptions(); EncryptionOptions encryptionOptions = new EncryptionOptions(); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index f20cba90e7cf..df2a4fb567a4 100644 --- a/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -187,7 +187,7 @@ public CosmosAsyncDatabase getDatabase(String id) { } } - @BeforeSuite(groups = {"simple", "long", "direct", "multi-master", "emulator", "non-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) + @BeforeSuite(groups = {"simple", "long", "direct", "multi-master", "encryption", "non-emulator"}, timeOut = SUITE_SETUP_TIMEOUT) public static void beforeSuite() { logger.info("beforeSuite Started"); @@ -203,7 +203,7 @@ public static void beforeSuite() { } } - @AfterSuite(groups = {"simple", "long", "direct", "multi-master", "emulator", "non-emulator"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) + @AfterSuite(groups = {"simple", "long", "direct", "multi-master", "encryption", "non-emulator"}, timeOut = SUITE_SHUTDOWN_TIMEOUT) public static void afterSuite() { logger.info("afterSuite Started"); diff --git a/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption-testng.xml b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption-testng.xml new file mode 100644 index 000000000000..bd1c2ac785a2 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-encryption/src/test/resources/encryption-testng.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java index ed20084d4dfe..6e9b2a2a7866 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/Utils.java @@ -629,6 +629,7 @@ public static ByteBuffer serializeJsonToByteBuffer(ObjectMapper objectMapper, Ob objectMapper.writeValue(byteBufferOutputStream, object); return byteBufferOutputStream.asByteBuffer(); } catch (IOException e) { + // TODO moderakh: on serialization/deserialization failure we should throw CosmosException here and elsewhere throw new IllegalArgumentException("Failed to serialize the object into json", e); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionProcessor.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionProcessor.java index 04e225a377f3..e264f2486213 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionProcessor.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/encryption/EncryptionProcessor.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.Map; +// TODO: can this be moved to encryption package? public class EncryptionProcessor { public ObjectNode encryptAsync( @@ -55,6 +56,7 @@ public ObjectNode encryptAsync( for (String pathToEncrypt : encryptionOptions.getPathsToEncrypt()) { String propertyName = pathToEncrypt.substring(1); + // TODO: moderakh should support JPath JsonNode propertyValueHolder = itemJObj.get(propertyName); // Even null in the JSON is a JToken with Type Null, this null check is just a sanity check diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java index b89f6c72a2a5..0f2d57605310 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/models/ModelBridgeInternal.java @@ -612,6 +612,7 @@ public static JsonSerializable getJsonSerializable(T t) { } } + @Warning(value = INTERNAL_USE_ONLY_WARNING) public static CosmosItemRequestOptions setEncryptionOptions(CosmosItemRequestOptions options, EncryptionOptions encryptionOptions) { options.setEncryptionOptions(encryptionOptions);