HDDS-8219. Run HA secure tests from s3g container

.github/workflows/ci.yml

@@ -248,7 +248,8 @@ jobs:
           - secure
           - unsecure
           - compat
-          - HA
+          - HA-secure
+          - HA-unsecure
           - MR
           - misc
       fail-fast: false

hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config

@@ -19,6 +19,7 @@ CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=*
 CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=*
 
 CORE-SITE.XML_fs.defaultFS=ofs://omservice/
+CORE-SITE.XML_fs.trash.interval=1
 
 OZONE-SITE.XML_ozone.om.service.ids=omservice
 OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
@@ -47,6 +48,7 @@ OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
 OZONE-SITE.XML_ozone.recon.address=recon:9891
 OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
 OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
+OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true
 
 OZONE_CONF_DIR=/etc/hadoop
 OZONE_LOG_DIR=/var/log/hadoop

hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#suite:HA
+#suite:HA-unsecure
 
 COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 export COMPOSE_DIR
@@ -28,11 +28,20 @@ export OM_SERVICE_ID=omservice
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
 
-start_docker_env
+start_docker_env 5
 
 execute_robot_test ${SCM} basic/ozone-shell-single.robot
 execute_robot_test ${SCM} basic/links.robot
-execute_robot_test ${SCM} s3
+
+execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link ozonefs/ozonefs.robot
+
+exclude=""
+for bucket in generated; do
+  execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
+  # some tests are independent of the bucket type, only need to be run once
+  exclude="--exclude no-bucket-type"
+done
+
 execute_robot_test ${SCM} freon
 execute_robot_test ${SCM} -v USERNAME:httpfs httpfs
 

hadoop-ozone/dist/src/main/compose/ozone/test.sh

@@ -40,7 +40,7 @@ execute_robot_test scm gdpr
 execute_robot_test scm security/ozone-secure-token.robot
 
 exclude=""
-for bucket in erasure link generated; do
+for bucket in erasure; do
   execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
   # some tests are independent of the bucket type, only need to be run once
   exclude="--exclude no-bucket-type"
@@ -58,7 +58,6 @@ execute_robot_test scm admincli
 execute_robot_test scm -v USERNAME:httpfs httpfs
 execute_debug_tests
 
-execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link ozonefs/ozonefs.robot
 execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-o3fs-bucket ozonefs/ozonefs.robot
 
 prefix=${RANDOM}

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config

@@ -19,6 +19,7 @@ CORE-SITE.XML_hadoop.proxyuser.httpfs.hosts=*
 CORE-SITE.XML_hadoop.proxyuser.httpfs.groups=*
 
 CORE-SITE.XML_fs.defaultFS=ofs://id1
+CORE-SITE.XML_fs.trash.interval=1
 
 OZONE-SITE.XML_ozone.om.service.ids=id1
 OZONE-SITE.XML_ozone.om.internal.service.id=id1
@@ -56,6 +57,7 @@ OZONE-SITE.XML_ozone.replication=3
 OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
 OZONE-SITE.XML_hdds.container.report.interval=60s
 OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
+OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true
 
 OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
 OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
@@ -142,6 +144,9 @@ HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/httpfs@EXAMPL
 HTTPFS-SITE.XML_httpfs.hadoop.authentication.type=kerberos
 HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
 HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal=httpfs/httpfs@EXAMPLE.COM
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=*
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=*
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=*
 
 #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
 #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#suite:HA
+#suite:HA-secure
 
 COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 export COMPOSE_DIR
@@ -24,25 +24,37 @@ export SECURITY_ENABLED=true
 export OM_SERVICE_ID="id1"
 export SCM=scm1.org
 
+: ${OZONE_BUCKET_KEY_NAME:=key1}
+
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
 
 start_docker_env
 
-execute_robot_test ${SCM} kinit.robot
+execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME}
+
+execute_robot_test s3g kinit.robot
 
-execute_robot_test ${SCM} freon
+execute_robot_test s3g freon
 
-execute_robot_test ${SCM} basic/links.robot
+execute_robot_test s3g -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link ozonefs/ozonefs.robot
 
-execute_robot_test ${SCM} s3
+execute_robot_test s3g basic/links.robot
 
-execute_robot_test ${SCM} admincli
+exclude=""
+for bucket in encrypted link; do
+  execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
+  # some tests are independent of the bucket type, only need to be run once
+  exclude="--exclude no-bucket-type"
+done
 
-execute_robot_test ${SCM} httpfs
+execute_robot_test s3g admincli
+
+execute_robot_test s3g httpfs
 
 export SCM=scm2.org
-execute_robot_test ${SCM} admincli
+execute_robot_test s3g admincli
+
 stop_docker_env
 
 generate_report

hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh

@@ -38,14 +38,6 @@ execute_robot_test scm basic
 execute_robot_test scm security
 
 execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot
-execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link ozonefs/ozonefs.robot
-
-exclude=""
-for bucket in encrypted link generated; do
-  execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
-  # some tests are independent of the bucket type, only need to be run once
-  exclude="--exclude no-bucket-type"
-done
 
 #expects 4 pipelines, should be run before
 #admincli which creates STANDALONE pipeline

hadoop-ozone/dist/src/main/smoketest/basic/links.robot

@@ -42,11 +42,12 @@ Setup ACL tests
     Execute             ozone sh bucket link ${source}/readable-bucket ${target}/readable-link
     Execute             ozone sh bucket link ${source}/readable-bucket ${target}/unreadable-link
     Execute             ozone sh bucket link ${source}/unreadable-bucket ${target}/link-to-unreadable-bucket
-    Execute             ozone sh volume addacl --acl user:testuser2/scm@EXAMPLE.COM:r ${target}
-    Execute             ozone sh volume addacl --acl user:testuser2/scm@EXAMPLE.COM:rl ${source}
-    Execute             ozone sh bucket addacl --acl user:testuser2/scm@EXAMPLE.COM:rl ${source}/readable-bucket
-    Execute             ozone sh bucket addacl --acl user:testuser2/scm@EXAMPLE.COM:r ${target}/readable-link
-    Execute             ozone sh bucket addacl --acl user:testuser2/scm@EXAMPLE.COM:r ${target}/link-to-unreadable-bucket
+    ${principal} =      Get test user principal    testuser2
+    Execute             ozone sh volume addacl --acl user:${principal}:r ${target}
+    Execute             ozone sh volume addacl --acl user:${principal}:rl ${source}
+    Execute             ozone sh bucket addacl --acl user:${principal}:rl ${source}/readable-bucket
+    Execute             ozone sh bucket addacl --acl user:${principal}:r ${target}/readable-link
+    Execute             ozone sh bucket addacl --acl user:${principal}:r ${target}/link-to-unreadable-bucket
 
 Can follow link with read access
     Execute             kdestroy

hadoop-ozone/dist/src/main/smoketest/commonlib.robot

@@ -26,15 +26,20 @@ ${OM_HA_PARAM}       ${EMPTY}
 ${OM_SERVICE_ID}     om
 
 *** Keywords ***
+Get test user principal
+    [arguments]         ${user}
+    ${instance} =       Execute                    hostname | sed 's/scm[0-9].org/scm/'
+    [return]            ${user}/${instance}@EXAMPLE.COM
+
 Kinit HTTP user
-    ${hostname} =       Execute                    hostname | sed 's/scm[0-9].org/scm/'
-    Wait Until Keyword Succeeds      2min       10sec      Execute            kinit -k HTTP/${hostname}@EXAMPLE.COM -t /etc/security/keytabs/HTTP.keytab
+    ${principal} =      Get test user principal    HTTP
+    Wait Until Keyword Succeeds      2min       10sec      Execute            kinit -k -t /etc/security/keytabs/HTTP.keytab ${principal}
 
 Kinit test user
     [arguments]                      ${user}       ${keytab}
-    ${instance} =       Execute                    hostname | sed 's/scm[0-9].org/scm/'
-    Set Suite Variable  ${TEST_USER}               ${user}/${instance}@EXAMPLE.COM
-    Wait Until Keyword Succeeds      2min       10sec      Execute            kinit -k ${user}/${instance}@EXAMPLE.COM -t /etc/security/keytabs/${keytab}
+    ${TEST_USER} =      Get test user principal    ${user}
+    Set Suite Variable  ${TEST_USER}
+    Wait Until Keyword Succeeds      2min       10sec      Execute            kinit -k -t /etc/security/keytabs/${keytab} ${TEST_USER}
 
 Access should be denied
     [arguments]    ${command}