diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f540c1df77a3..2697a6e3f8ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -248,7 +248,8 @@ jobs: - secure - unsecure - compat - - HA + - HA-secure + - HA-unsecure - MR - misc fail-fast: false diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config index 6fb4003b6b3c..bfe0c03f77cf 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config @@ -19,6 +19,7 @@ CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=* CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=* CORE-SITE.XML_fs.defaultFS=ofs://omservice/ +CORE-SITE.XML_fs.trash.interval=1 OZONE-SITE.XML_ozone.om.service.ids=omservice OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3 @@ -47,6 +48,7 @@ OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.recon.address=recon:9891 OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888 OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889 +OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true OZONE_CONF_DIR=/etc/hadoop OZONE_LOG_DIR=/var/log/hadoop diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh index 867c6923f5dd..2c2834cc3ba5 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -#suite:HA +#suite:HA-unsecure COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR @@ -28,11 +28,20 @@ export OM_SERVICE_ID=omservice # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" -start_docker_env +start_docker_env 5 execute_robot_test ${SCM} basic/ozone-shell-single.robot execute_robot_test ${SCM} basic/links.robot -execute_robot_test ${SCM} s3 + +execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link ozonefs/ozonefs.robot + +exclude="" +for bucket in generated; do + execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + # some tests are independent of the bucket type, only need to be run once + exclude="--exclude no-bucket-type" +done + execute_robot_test ${SCM} freon execute_robot_test ${SCM} -v USERNAME:httpfs httpfs diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh index b61c6277c9f2..6c84c3054358 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@ -40,7 +40,7 @@ execute_robot_test scm gdpr execute_robot_test scm security/ozone-secure-token.robot exclude="" -for bucket in erasure link generated; do +for bucket in erasure; do execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 # some tests are independent of the bucket type, only need to be run once exclude="--exclude no-bucket-type" @@ -58,7 +58,6 @@ execute_robot_test scm admincli execute_robot_test scm -v USERNAME:httpfs httpfs execute_debug_tests -execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link ozonefs/ozonefs.robot execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-o3fs-bucket ozonefs/ozonefs.robot prefix=${RANDOM} diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config index ed82642df851..dce3cf4460c2 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config @@ -19,6 +19,7 @@ CORE-SITE.XML_hadoop.proxyuser.httpfs.hosts=* CORE-SITE.XML_hadoop.proxyuser.httpfs.groups=* CORE-SITE.XML_fs.defaultFS=ofs://id1 +CORE-SITE.XML_fs.trash.interval=1 OZONE-SITE.XML_ozone.om.service.ids=id1 OZONE-SITE.XML_ozone.om.internal.service.id=id1 @@ -56,6 +57,7 @@ OZONE-SITE.XML_ozone.replication=3 OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s OZONE-SITE.XML_hdds.container.report.interval=60s OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true +OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon @@ -142,6 +144,9 @@ HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/httpfs@EXAMPL HTTPFS-SITE.XML_httpfs.hadoop.authentication.type=kerberos HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal=httpfs/httpfs@EXAMPLE.COM +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=* +KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=* #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh index dcd16a802b68..c87e6dce23e6 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -#suite:HA +#suite:HA-secure COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR @@ -24,18 +24,29 @@ export SECURITY_ENABLED=true export OM_SERVICE_ID="id1" export SCM=scm1.org +: ${OZONE_BUCKET_KEY_NAME:=key1} + # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" start_docker_env +execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME} + execute_robot_test ${SCM} kinit.robot execute_robot_test ${SCM} freon +execute_robot_test ${SCM} -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link ozonefs/ozonefs.robot + execute_robot_test ${SCM} basic/links.robot -execute_robot_test ${SCM} s3 +exclude="" +for bucket in encrypted link; do + execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 + # some tests are independent of the bucket type, only need to be run once + exclude="--exclude no-bucket-type" +done execute_robot_test ${SCM} admincli diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh index df96734ed97c..a1e31830a151 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh @@ -38,14 +38,6 @@ execute_robot_test scm basic execute_robot_test scm security execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot -execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link ozonefs/ozonefs.robot - -exclude="" -for bucket in encrypted link generated; do - execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 - # some tests are independent of the bucket type, only need to be run once - exclude="--exclude no-bucket-type" -done #expects 4 pipelines, should be run before #admincli which creates STANDALONE pipeline