diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh index 6d213f9b5fbe..407418a2ef11 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh @@ -36,6 +36,7 @@ execute_robot_test scm kinit.robot execute_robot_test scm basic execute_robot_test scm security +execute_robot_test scm repair/bucket-encryption.robot execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot diff --git a/hadoop-ozone/dist/src/main/smoketest/repair/bucket-encryption.robot b/hadoop-ozone/dist/src/main/smoketest/repair/bucket-encryption.robot new file mode 100644 index 000000000000..36772d2c68a6 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/repair/bucket-encryption.robot @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Resource ../lib/os.robot +Resource ../ozone-lib/shell.robot + +*** Variables *** +${ENCRYPTION_KEY} key1 + + +*** Keywords *** +Verify Bucket Encryption Key + [arguments] ${bucket} ${expected} + ${output} = Execute ozone sh bucket info ${bucket} + ${actual} = Execute echo '${output}' | jq -r '.encryptionKeyName' + Should Be Equal ${expected} ${actual} + + +*** Test Cases *** +Set Bucket Encryption Key + ${random} = Generate Random String 10 [NUMBERS] + ${bucket} = Set Variable /vol${random}/encrypted + + Ozone Shell Batch volume create /vol${random} + ... bucket create ${bucket} + Verify Bucket Encryption Key ${bucket} null + + Execute ozone sh bucket set-encryption-key -k ${ENCRYPTION_KEY} ${bucket} + Verify Bucket Encryption Key ${bucket} ${ENCRYPTION_KEY} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 750b90fa4aed..a6a509bc4c1d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -69,6 +69,7 @@ import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ozone.test.GenericTestUtils; +import org.apache.ozone.test.tag.Unhealthy; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.ozone.om.TrashPolicyOzone; import org.apache.hadoop.hdds.JsonTestUtils; @@ -109,6 +110,7 @@ import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; @@ -128,6 +130,7 @@ * This class tests Ozone sh shell command. * Inspired by TestS3Shell */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) @Timeout(300) @TestMethodOrder(OrderAnnotation.class) public class TestOzoneShellHA { @@ -165,11 +168,11 @@ public class TestOzoneShellHA { * @throws Exception */ @BeforeAll - public static void init() throws Exception { + public void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.setBoolean(OZONE_HBASE_ENHANCEMENTS_ALLOWED, true); conf.setBoolean(OZONE_FS_HSYNC_ENABLED, true); - startKMS(); + // startKMS(); startCluster(conf); } @@ -209,7 +212,7 @@ protected static void startCluster(OzoneConfiguration conf) throws Exception { * shutdown MiniOzoneCluster. */ @AfterAll - public static void shutdown() { + public void shutdown() { IOUtils.closeQuietly(client); if (cluster != null) { cluster.shutdown(); @@ -956,7 +959,7 @@ public void testOzoneAdminCmdListAllContainer() execute(ozoneAdminShell, args); } - String[] args1 = new String[] {"container", "list", "-c", "10", "--scm", + String[] args1 = new String[] {"container", "list", "-c", "1", "--scm", "localhost:" + cluster.getStorageContainerManager().getClientRpcPort()}; execute(ozoneAdminShell, args1); //results will be capped at the maximum allowed count @@ -965,8 +968,9 @@ public void testOzoneAdminCmdListAllContainer() String[] args2 = new String[] {"container", "list", "-a", "--scm", "localhost:" + cluster.getStorageContainerManager().getClientRpcPort()}; execute(ozoneAdminShell, args2); - //Lists all containers - assertNotEquals(1, getNumOfContainers()); + //Lists all containers, at least the two created for this method + assertThat(getNumOfContainers()) + .isGreaterThanOrEqualTo(2); } private int getNumOfContainers() @@ -1807,6 +1811,7 @@ public void testSetECReplicationConfigOnBucket() throws Exception { } @Test + @Unhealthy("HDDS-11879") public void testSetEncryptionKey() throws Exception { final String volumeName = "volume111"; getVolume(volumeName); @@ -2190,8 +2195,7 @@ public void testClientBucketLayoutValidation() { ParameterException exception = assertThrows(ParameterException.class, () -> execute(ozoneShell, arg2)); assertThat(exception.getMessage()) - .contains("expected one of [FILE_SYSTEM_OPTIMIZED, OBJECT_STORE, " + - "LEGACY]"); + .contains("cannot convert '' to BucketLayout"); String[] arg3 = new String[]{ @@ -2202,8 +2206,7 @@ public void testClientBucketLayoutValidation() { exception = assertThrows(ParameterException.class, () -> execute(ozoneShell, arg3)); assertThat(exception.getMessage()) - .contains("expected one of [FILE_SYSTEM_OPTIMIZED, OBJECT_STORE, " + - "LEGACY] "); + .contains("cannot convert 'INVALID' to BucketLayout"); } @Test @@ -2488,6 +2491,10 @@ public void testKeyDeleteLegacyWithEnableFileSystemPath() throws IOException { } private static String getKeyProviderURI(MiniKMS kms) { + // HDDS-11879 + if (kms == null) { + return ""; + } return KMSClientProvider.SCHEME_NAME + "://" + kms.getKMSUrl().toExternalForm().replace("://", "@"); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java index b1dcbc0576ea..37a514b16b46 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; /** @@ -34,22 +33,15 @@ public class TestOzoneShellHAWithFSO extends TestOzoneShellHA { * handler type. */ @BeforeAll - public static void init() throws Exception { + @Override + public void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED); conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true); conf.setBoolean("ozone.client.hbase.enhancements.allowed", true); conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true); - startKMS(); + // startKMS(); startCluster(conf); } - - /** - * shutdown MiniOzoneCluster. - */ - @AfterAll - public static void shutdownCluster() { - shutdown(); - } }