From 1dcb95409d2d641a7f2181428913b3c012b2d16f Mon Sep 17 00:00:00 2001 From: Zita Dombi Date: Mon, 19 May 2025 18:44:54 +0200 Subject: [PATCH 1/6] HDDS-12714. Create acceptance test framework for debug and repair tools Change-Id: I7d16dbd682bc08dba0396d4f9a32b80e8c80f199 --- .../dist/src/main/compose/ozonesecure-ha/.env | 2 + .../compose/ozonesecure-ha/debug-replicas.sh | 52 +++++ .../compose/ozonesecure-ha/debug-tools.yaml | 177 ++++++++++++++++++ .../ozonesecure-ha/docker-config-debug-tools | 15 ++ .../ozonesecure-ha/test-debug-tools.sh | 54 ++++++ 5 files changed, 300 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-replicas.sh create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-debug-tools create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env index c3a2c5329aa2..31345aaeb2ff 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env @@ -20,4 +20,6 @@ HADOOP_VERSION=${hadoop.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image} +OZONE_DIR=/opt/hadoop +OZONE_VOLUME=./data OZONE_OPTS= diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-replicas.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-replicas.sh new file mode 100644 index 000000000000..321d79204d5b --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-replicas.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +prefix=${RANDOM} + +volume="cli-debug-volume${prefix}" +bucket="cli-debug-bucket" +key="testfile" + +execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests.robot + +# get block locations for key +chunkinfo="${key}-blocks-${prefix}" +docker-compose exec -T ${SCM} bash -c "ozone debug replicas chunk-info ${volume}/${bucket}/${key}" > "$chunkinfo" +host="$(jq -r '.KeyLocations[0][0]["Datanode-HostName"]' ${chunkinfo})" +container="${host%%.*}" + +# corrupt the first block of key on one of the datanodes +datafile="$(jq -r '.KeyLocations[0][0].Locations.files[0]' ${chunkinfo})" +docker exec "${container}" sed -i -e '1s/^/a/' "${datafile}" + +execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "CORRUPT_DATANODE:${host}" debug/ozone-debug-corrupt-block.robot + +docker stop "${container}" + +wait_for_datanode "${container}" STALE 60 +execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "STALE_DATANODE:${host}" debug/ozone-debug-stale-datanode.robot + +wait_for_datanode "${container}" DEAD 60 +execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-dead-datanode.robot + +docker start "${container}" + +wait_for_datanode "${container}" HEALTHY 60 + +start_docker_env 9 +execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec3-2.robot +execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec6-3.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml new file mode 100644 index 000000000000..3214e61d1871 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +x-debug-tools-config: + &common-env-file + dns_search: . + image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} + env_file: + - ./docker-config + # - ./docker-config-debug-tools + +x-volumes: + - &keytabs ../_keytabs:/etc/security/keytabs + - &krb5conf ./krb5.conf:/etc/krb5.conf + - &ozone-dir ../..:${OZONE_DIR} + - &transformation ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py + +services: + kdc: + volumes: + - *keytabs + - ../..:/opt/hadoop + kms: + volumes: + - ${OZONE_VOLUME}/kms:/data + - *keytabs + - *krb5conf + - ../..:/opt/ozone + - *transformation + datanode1: + <<: *common-env-file + volumes: + - ${OZONE_VOLUME}/dn1:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + datanode2: + <<: *common-env-file + volumes: + - ${OZONE_VOLUME}/dn2:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + datanode3: + <<: *common-env-file + volumes: + - ${OZONE_VOLUME}/dn3:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + datanode4: + <<: *common-env-file + ports: + - 9870:9999 + command: ["/opt/hadoop/bin/ozone","datanode"] + extra_hosts: + - "scm1.org: 172.25.0.116" + - "scm2.org: 172.25.0.117" + - "scm3.org: 172.25.0.118" + - "recon: 172.25.0.115" + environment: + WAITFOR: scm3.org:9894 + OZONE_OPTS: + networks: + ozone_net: + ipv4_address: 172.25.0.105 + volumes: + - ${OZONE_VOLUME}/dn4:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + datanode5: + <<: *common-env-file + ports: + - 9872:9999 + command: [ "/opt/hadoop/bin/ozone","datanode" ] + extra_hosts: + - "scm1.org: 172.25.0.116" + - "scm2.org: 172.25.0.117" + - "scm3.org: 172.25.0.118" + - "recon: 172.25.0.115" + environment: + WAITFOR: scm3.org:9894 + OZONE_OPTS: + networks: + ozone_net: + ipv4_address: 172.25.0.106 + volumes: + - ${OZONE_VOLUME}/dn5:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + om1: + <<: *common-env-file + volumes: + - ${OZONE_VOLUME}/om1:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + om2: + <<: *common-env-file + volumes: + - ${OZONE_VOLUME}/om2:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + om3: + <<: *common-env-file + volumes: + - ${OZONE_VOLUME}/om3:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + s3g: + <<: *common-env-file + volumes: + - ${OZONE_VOLUME}/s3g:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + scm1.org: + <<: *common-env-file + volumes: + - ${OZONE_VOLUME}/scm1:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + scm2.org: + <<: *common-env-file + volumes: + - ${OZONE_VOLUME}/scm2:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + scm3.org: + <<: *common-env-file + volumes: + - ${OZONE_VOLUME}/scm3:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + recon: + <<: *common-env-file + volumes: + - ${OZONE_VOLUME}/recon:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + + diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-debug-tools b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-debug-tools new file mode 100644 index 000000000000..635f0d9e60e1 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-debug-tools @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh new file mode 100644 index 000000000000..32308f941afa --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#suite:HA-secure + +set -u -o pipefail + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +export SECURITY_ENABLED=true +export OM_SERVICE_ID="omservice" +export SCM=scm1.org +export COMPOSE_FILE=docker-compose.yaml:debug-tools.yaml + +: "${OZONE_VOLUME_OWNER:=}" +: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}" + +export OZONE_VOLUME + +# Clean up saved internal state from each container's volume for the next run. +rm -rf "${OZONE_VOLUME}" +mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,dn4,dn5,om1,om2,om3,scm1,scm2,scm3,recon,s3g,kms} + +if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then + current_user=$(whoami) + if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then + chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \ + || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" + fi +fi + +: ${OZONE_BUCKET_KEY_NAME:=key1} + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env + +source "$COMPOSE_DIR/../common/replicas-test.sh" From 916aca488ddf152140ca338c1c830da83ed34f73 Mon Sep 17 00:00:00 2001 From: Zita Dombi Date: Tue, 20 May 2025 08:46:41 +0200 Subject: [PATCH 2/6] Fix extra host ip parsing Change-Id: I0125f61a7d843ae5d65bdeccf907f316cebab92d --- .../main/compose/ozonesecure-ha/debug-tools.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml index 3214e61d1871..759def41a604 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml @@ -71,10 +71,10 @@ services: - 9870:9999 command: ["/opt/hadoop/bin/ozone","datanode"] extra_hosts: - - "scm1.org: 172.25.0.116" - - "scm2.org: 172.25.0.117" - - "scm3.org: 172.25.0.118" - - "recon: 172.25.0.115" + - "scm1.org=172.25.0.116" + - "scm2.org=172.25.0.117" + - "scm3.org=172.25.0.118" + - "recon=172.25.0.115" environment: WAITFOR: scm3.org:9894 OZONE_OPTS: @@ -93,10 +93,10 @@ services: - 9872:9999 command: [ "/opt/hadoop/bin/ozone","datanode" ] extra_hosts: - - "scm1.org: 172.25.0.116" - - "scm2.org: 172.25.0.117" - - "scm3.org: 172.25.0.118" - - "recon: 172.25.0.115" + - "scm1.org=172.25.0.116" + - "scm2.org=172.25.0.117" + - "scm3.org=172.25.0.118" + - "recon=172.25.0.115" environment: WAITFOR: scm3.org:9894 OZONE_OPTS: From 2730eacdba00c5df4da42230a91a9a4cb160300d Mon Sep 17 00:00:00 2001 From: Zita Dombi Date: Tue, 20 May 2025 13:02:58 +0200 Subject: [PATCH 3/6] Reduce code changes Change-Id: Ia138c3c54e87b567e6a697f097f8bdb60347e69e --- .../dist/src/main/compose/ozonesecure-ha/.env | 1 - .../compose/ozonesecure-ha/debug-replicas.sh | 52 ------------------- .../compose/ozonesecure-ha/debug-tools.yaml | 18 +------ .../ozonesecure-ha/test-debug-tools.sh | 3 +- 4 files changed, 3 insertions(+), 71 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-replicas.sh diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env index 31345aaeb2ff..df30e9989a32 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env @@ -20,6 +20,5 @@ HADOOP_VERSION=${hadoop.version} OZONE_RUNNER_VERSION=${docker.ozone-runner.version} OZONE_RUNNER_IMAGE=apache/ozone-runner OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image} -OZONE_DIR=/opt/hadoop OZONE_VOLUME=./data OZONE_OPTS= diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-replicas.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-replicas.sh deleted file mode 100644 index 321d79204d5b..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-replicas.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -prefix=${RANDOM} - -volume="cli-debug-volume${prefix}" -bucket="cli-debug-bucket" -key="testfile" - -execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests.robot - -# get block locations for key -chunkinfo="${key}-blocks-${prefix}" -docker-compose exec -T ${SCM} bash -c "ozone debug replicas chunk-info ${volume}/${bucket}/${key}" > "$chunkinfo" -host="$(jq -r '.KeyLocations[0][0]["Datanode-HostName"]' ${chunkinfo})" -container="${host%%.*}" - -# corrupt the first block of key on one of the datanodes -datafile="$(jq -r '.KeyLocations[0][0].Locations.files[0]' ${chunkinfo})" -docker exec "${container}" sed -i -e '1s/^/a/' "${datafile}" - -execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "CORRUPT_DATANODE:${host}" debug/ozone-debug-corrupt-block.robot - -docker stop "${container}" - -wait_for_datanode "${container}" STALE 60 -execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "STALE_DATANODE:${host}" debug/ozone-debug-stale-datanode.robot - -wait_for_datanode "${container}" DEAD 60 -execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-dead-datanode.robot - -docker start "${container}" - -wait_for_datanode "${container}" HEALTHY 60 - -start_docker_env 9 -execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec3-2.robot -execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec6-3.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml index 759def41a604..b71f13a94a0f 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml @@ -26,20 +26,19 @@ x-debug-tools-config: x-volumes: - &keytabs ../_keytabs:/etc/security/keytabs - &krb5conf ./krb5.conf:/etc/krb5.conf - - &ozone-dir ../..:${OZONE_DIR} + - &ozone-dir ../..:/opt/hadoop - &transformation ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py services: kdc: volumes: - *keytabs - - ../..:/opt/hadoop + - *ozone-dir kms: volumes: - ${OZONE_VOLUME}/kms:/data - *keytabs - *krb5conf - - ../..:/opt/ozone - *transformation datanode1: <<: *common-env-file @@ -48,7 +47,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation datanode2: <<: *common-env-file volumes: @@ -56,7 +54,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation datanode3: <<: *common-env-file volumes: @@ -64,7 +61,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation datanode4: <<: *common-env-file ports: @@ -86,7 +82,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation datanode5: <<: *common-env-file ports: @@ -108,7 +103,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation om1: <<: *common-env-file volumes: @@ -116,7 +110,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation om2: <<: *common-env-file volumes: @@ -124,7 +117,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation om3: <<: *common-env-file volumes: @@ -132,7 +124,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation s3g: <<: *common-env-file volumes: @@ -140,7 +131,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation scm1.org: <<: *common-env-file volumes: @@ -148,7 +138,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation scm2.org: <<: *common-env-file volumes: @@ -156,7 +145,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation scm3.org: <<: *common-env-file volumes: @@ -164,7 +152,6 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation recon: <<: *common-env-file volumes: @@ -172,6 +159,5 @@ services: - *keytabs - *krb5conf - *ozone-dir - - *transformation diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh index 32308f941afa..f25b7ce5b125 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh @@ -26,6 +26,7 @@ export SECURITY_ENABLED=true export OM_SERVICE_ID="omservice" export SCM=scm1.org export COMPOSE_FILE=docker-compose.yaml:debug-tools.yaml +export OZONE_DIR=/opt/hadoop : "${OZONE_VOLUME_OWNER:=}" : "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}" @@ -44,8 +45,6 @@ if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then fi fi -: ${OZONE_BUCKET_KEY_NAME:=key1} - # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" From 0561c2d0d2d67e9c5cddbdd789491fac76bd1c2d Mon Sep 17 00:00:00 2001 From: Zita Dombi Date: Thu, 22 May 2025 10:57:04 +0200 Subject: [PATCH 4/6] Fix om service id Change-Id: I598c55991b2ef10060d129650b194fdc6da61388 --- .../src/main/compose/ozonesecure-ha/test-debug-tools.sh | 6 ++++-- .../main/smoketest/debug/ozone-debug-tests-ec3-2.robot | 9 +++++---- .../main/smoketest/debug/ozone-debug-tests-ec6-3.robot | 5 +++-- .../src/main/smoketest/debug/ozone-debug-tests.robot | 9 +++++---- .../dist/src/main/smoketest/debug/ozone-debug.robot | 5 ++++- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh index f25b7ce5b125..56dac794b4af 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh @@ -21,9 +21,9 @@ set -u -o pipefail COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR - +§ export SECURITY_ENABLED=true -export OM_SERVICE_ID="omservice" +export OM_SERVICE_ID=omservice export SCM=scm1.org export COMPOSE_FILE=docker-compose.yaml:debug-tools.yaml export OZONE_DIR=/opt/hadoop @@ -50,4 +50,6 @@ source "$COMPOSE_DIR/../testlib.sh" start_docker_env +execute_robot_test ${SCM} kinit.robot + source "$COMPOSE_DIR/../common/replicas-test.sh" diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot index 0c310f62e89d..e10e03443d60 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot @@ -29,17 +29,18 @@ ${BUCKET} cli-debug-bucket ${TESTFILE} testfile ${EC_DATA} 3 ${EC_PARITY} 2 +${OM_SERVICE_ID} %{OM_SERVICE_ID} *** Keywords *** Create Volume Bucket - Execute ozone sh volume create o3://om/${VOLUME} - Execute ozone sh bucket create o3://om/${VOLUME}/${BUCKET} + Execute ozone sh volume create o3://${OM_SERVICE_ID}/${VOLUME} + Execute ozone sh bucket create o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET} Create EC key [arguments] ${bs} ${count} Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs} count=${count} - Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC + Execute ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC *** Test Cases *** 0 data block @@ -87,5 +88,5 @@ Create EC key Test ozone debug replicas chunk-info Create EC key 1048576 3 - ${count} = Execute ozone debug replicas chunk-info o3://om/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length' + ${count} = Execute ozone debug replicas chunk-info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length' Should Be Equal As Integers ${count} 5 diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot index 9c83cad6971a..c3b8c7ad1a3a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot @@ -28,13 +28,14 @@ ${BUCKET} cli-debug-bucket ${TESTFILE} testfile ${EC_DATA} 6 ${EC_PARITY} 3 +${OM_SERVICE_ID} %{OM_SERVICE_ID} *** Keywords *** Create EC key [arguments] ${bs} ${count} Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs} count=${count} - Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC + Execute ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC *** Test Cases *** 0 data block @@ -95,5 +96,5 @@ Create EC key Test ozone debug replicas chunk-info Create EC key 1048576 6 - ${count} = Execute ozone debug replicas chunk-info o3://om/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length' + ${count} = Execute ozone debug replicas chunk-info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length' Should Be Equal As Integers ${count} 9 diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot index 97fe36d2c50e..532f931bb5f6 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot @@ -27,17 +27,18 @@ ${VOLUME} cli-debug-volume${PREFIX} ${BUCKET} cli-debug-bucket ${DEBUGKEY} debugKey ${TESTFILE} testfile +${OM_SERVICE_ID} %{OM_SERVICE_ID} *** Keywords *** Write keys - Execute ozone sh volume create o3://om/${VOLUME} --space-quota 100TB --namespace-quota 100 - Execute ozone sh bucket create o3://om/${VOLUME}/${BUCKET} --space-quota 1TB + Execute ozone sh volume create o3://${OM_SERVICE_ID}/${VOLUME} --space-quota 100TB --namespace-quota 100 + Execute ozone sh bucket create o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET} --space-quota 1TB Execute dd if=/dev/urandom of=${TEMP_DIR}/${TESTFILE} bs=100000 count=15 - Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/${TESTFILE} ${TEMP_DIR}/${TESTFILE} + Execute ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} ${TEMP_DIR}/${TESTFILE} *** Test Cases *** Test ozone debug replicas verify checksums - ${output} = Execute ozone debug replicas verify --checksums o3://om/${VOLUME}/${BUCKET}/${TESTFILE} --output-dir ${TEMP_DIR} + ${output} = Execute ozone debug replicas verify --checksums o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} --output-dir ${TEMP_DIR} ${json} = Evaluate json.loads('''${output}''') json # 'keys' array should be empty if all keys and their replicas passed checksum verification diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot index b042f606cf2c..c8eb08924756 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot @@ -18,9 +18,12 @@ Documentation Keyword definitions for Ozone Debug CLI tests Library Collections Resource ../lib/os.robot +*** Variables *** +${OM_SERVICE_ID} %{OM_SERVICE_ID} + *** Keywords *** Execute replicas verify checksums CLI tool - Execute ozone debug -Dozone.network.topology.aware.read=true replicas verify --checksums --output-dir ${TEMP_DIR} o3://om/${VOLUME}/${BUCKET}/${TESTFILE} + Execute ozone debug -Dozone.network.topology.aware.read=true replicas verify --checksums --output-dir ${TEMP_DIR} o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} ${directory} = Execute ls -d ${TEMP_DIR}/${VOLUME}_${BUCKET}_${TESTFILE}_*/ | tail -n 1 Directory Should Exist ${directory} File Should Exist ${directory}/${TESTFILE}_manifest From 447682c843e5a65db2c84d5cf4b4890d5dec4fc7 Mon Sep 17 00:00:00 2001 From: Zita Dombi Date: Thu, 22 May 2025 12:20:24 +0200 Subject: [PATCH 5/6] Remove accidental character Change-Id: I7b664516af5bcb90944b60a287b5b34593b4970e --- .../dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh index 56dac794b4af..c54c17425b36 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh @@ -21,7 +21,7 @@ set -u -o pipefail COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export COMPOSE_DIR -§ + export SECURITY_ENABLED=true export OM_SERVICE_ID=omservice export SCM=scm1.org From 9f0804a037e295ae55efd01a4c1c364a0b46dd76 Mon Sep 17 00:00:00 2001 From: Zita Dombi Date: Tue, 27 May 2025 15:24:12 +0200 Subject: [PATCH 6/6] Remove docker-config-debug-tools Change-Id: I95dcac6a1706220b0f6accc7bbbf280a7c33e9b5 --- .../main/compose/ozonesecure-ha/debug-tools.yaml | 1 - .../ozonesecure-ha/docker-config-debug-tools | 15 --------------- 2 files changed, 16 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-debug-tools diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml index b71f13a94a0f..e07b41ad89de 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml @@ -21,7 +21,6 @@ x-debug-tools-config: image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION} env_file: - ./docker-config - # - ./docker-config-debug-tools x-volumes: - &keytabs ../_keytabs:/etc/security/keytabs diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-debug-tools b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-debug-tools deleted file mode 100644 index 635f0d9e60e1..000000000000 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-debug-tools +++ /dev/null @@ -1,15 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License.