HDDS-7975. Rebalance acceptance tests

.github/workflows/ci.yml

@@ -248,7 +248,8 @@ jobs:
           - secure
           - unsecure
           - compat
-          - HA
+          - HA-secure
+          - HA-unsecure
           - MR
           - misc
       fail-fast: false

hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config

@@ -19,6 +19,7 @@ CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=*
 CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=*
 
 CORE-SITE.XML_fs.defaultFS=ofs://omservice/
+CORE-SITE.XML_fs.trash.interval=1
 
 OZONE-SITE.XML_ozone.om.service.ids=omservice
 OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
@@ -47,6 +48,7 @@ OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
 OZONE-SITE.XML_ozone.recon.address=recon:9891
 OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
 OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
+OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true
 
 OZONE_CONF_DIR=/etc/hadoop
 OZONE_LOG_DIR=/var/log/hadoop

hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#suite:HA
+#suite:HA-unsecure
 
 COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 export COMPOSE_DIR
@@ -28,11 +28,20 @@ export OM_SERVICE_ID=omservice
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
 
-start_docker_env
+start_docker_env 5
 
 execute_robot_test ${SCM} basic/ozone-shell-single.robot
 execute_robot_test ${SCM} basic/links.robot
-execute_robot_test ${SCM} s3
+
+execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link ozonefs/ozonefs.robot
+
+exclude=""
+for bucket in generated; do
+  execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
+  # some tests are independent of the bucket type, only need to be run once
+  exclude="--exclude no-bucket-type"
+done
+
 execute_robot_test ${SCM} freon
 execute_robot_test ${SCM} -v USERNAME:httpfs httpfs
 

hadoop-ozone/dist/src/main/compose/ozone/test.sh

@@ -40,7 +40,7 @@ execute_robot_test scm gdpr
 execute_robot_test scm security/ozone-secure-token.robot
 
 exclude=""
-for bucket in erasure link generated; do
+for bucket in erasure; do
   execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
   # some tests are independent of the bucket type, only need to be run once
   exclude="--exclude no-bucket-type"
@@ -58,7 +58,6 @@ execute_robot_test scm admincli
 execute_robot_test scm -v USERNAME:httpfs httpfs
 execute_debug_tests
 
-execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link ozonefs/ozonefs.robot
 execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-o3fs-bucket ozonefs/ozonefs.robot
 
 prefix=${RANDOM}

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config

@@ -19,6 +19,7 @@ CORE-SITE.XML_hadoop.proxyuser.httpfs.hosts=*
 CORE-SITE.XML_hadoop.proxyuser.httpfs.groups=*
 
 CORE-SITE.XML_fs.defaultFS=ofs://id1
+CORE-SITE.XML_fs.trash.interval=1
 
 OZONE-SITE.XML_ozone.om.service.ids=id1
 OZONE-SITE.XML_ozone.om.internal.service.id=id1
@@ -56,6 +57,7 @@ OZONE-SITE.XML_ozone.replication=3
 OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
 OZONE-SITE.XML_hdds.container.report.interval=60s
 OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
+OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true
 
 OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
 OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
@@ -142,6 +144,9 @@ HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/httpfs@EXAMPL
 HTTPFS-SITE.XML_httpfs.hadoop.authentication.type=kerberos
 HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
 HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal=httpfs/[email protected]
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=*
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=*
+KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=*
 
 #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
 #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#suite:HA
+#suite:HA-secure
 
 COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 export COMPOSE_DIR
@@ -24,18 +24,29 @@ export SECURITY_ENABLED=true
 export OM_SERVICE_ID="id1"
 export SCM=scm1.org
 
+: ${OZONE_BUCKET_KEY_NAME:=key1}
+
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
 
 start_docker_env
 
+execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME}
+
 execute_robot_test ${SCM} kinit.robot
 
 execute_robot_test ${SCM} freon
 
+execute_robot_test ${SCM} -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link ozonefs/ozonefs.robot
+
 execute_robot_test ${SCM} basic/links.robot
 
-execute_robot_test ${SCM} s3
+exclude=""
+for bucket in encrypted link; do
+  execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
+  # some tests are independent of the bucket type, only need to be run once
+  exclude="--exclude no-bucket-type"
+done
 
 execute_robot_test ${SCM} admincli
 

hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh

@@ -38,14 +38,6 @@ execute_robot_test scm basic
 execute_robot_test scm security
 
 execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot
-execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link ozonefs/ozonefs.robot
-
-exclude=""
-for bucket in encrypted link generated; do
-  execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
-  # some tests are independent of the bucket type, only need to be run once
-  exclude="--exclude no-bucket-type"
-done
 
 #expects 4 pipelines, should be run before
 #admincli which creates STANDALONE pipeline