HDDS-12714. Create acceptance test framework for debug and repair tools

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env

@@ -20,4 +20,5 @@ HADOOP_VERSION=${hadoop.version}
 OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
 OZONE_RUNNER_IMAGE=apache/ozone-runner
 OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
+OZONE_VOLUME=./data
 OZONE_OPTS=

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/debug-tools.yaml

@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+x-debug-tools-config:
+  &common-env-file
+  dns_search: .
+  image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+  env_file:
+    - ./docker-config
+
+x-volumes:
+  - &keytabs ../_keytabs:/etc/security/keytabs
+  - &krb5conf ./krb5.conf:/etc/krb5.conf
+  - &ozone-dir ../..:/opt/hadoop
+  - &transformation ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+
+services:
+  kdc:
+    volumes:
+      - *keytabs
+      - *ozone-dir
+  kms:
+    volumes:
+      - ${OZONE_VOLUME}/kms:/data
+      - *keytabs
+      - *krb5conf
+      - *transformation
+  datanode1:
+    <<: *common-env-file
+    volumes:
+      - ${OZONE_VOLUME}/dn1:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  datanode2:
+    <<: *common-env-file
+    volumes:
+      - ${OZONE_VOLUME}/dn2:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  datanode3:
+    <<: *common-env-file
+    volumes:
+      - ${OZONE_VOLUME}/dn3:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  datanode4:
+    <<: *common-env-file
+    ports:
+      - 9870:9999
+    command: ["/opt/hadoop/bin/ozone","datanode"]
+    extra_hosts:
+      - "scm1.org=172.25.0.116"
+      - "scm2.org=172.25.0.117"
+      - "scm3.org=172.25.0.118"
+      - "recon=172.25.0.115"
+    environment:
+      WAITFOR: scm3.org:9894
+      OZONE_OPTS:
+    networks:
+      ozone_net:
+        ipv4_address: 172.25.0.105
+    volumes:
+      - ${OZONE_VOLUME}/dn4:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  datanode5:
+    <<: *common-env-file
+    ports:
+      - 9872:9999
+    command: [ "/opt/hadoop/bin/ozone","datanode" ]
+    extra_hosts:
+      - "scm1.org=172.25.0.116"
+      - "scm2.org=172.25.0.117"
+      - "scm3.org=172.25.0.118"
+      - "recon=172.25.0.115"
+    environment:
+      WAITFOR: scm3.org:9894
+      OZONE_OPTS:
+    networks:
+      ozone_net:
+        ipv4_address: 172.25.0.106
+    volumes:
+      - ${OZONE_VOLUME}/dn5:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  om1:
+    <<: *common-env-file
+    volumes:
+      - ${OZONE_VOLUME}/om1:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  om2:
+    <<: *common-env-file
+    volumes:
+      - ${OZONE_VOLUME}/om2:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  om3:
+    <<: *common-env-file
+    volumes:
+      - ${OZONE_VOLUME}/om3:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  s3g:
+    <<: *common-env-file
+    volumes:
+      - ${OZONE_VOLUME}/s3g:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  scm1.org:
+    <<: *common-env-file
+    volumes:
+      - ${OZONE_VOLUME}/scm1:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  scm2.org:
+    <<: *common-env-file
+    volumes:
+      - ${OZONE_VOLUME}/scm2:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  scm3.org:
+    <<: *common-env-file
+    volumes:
+      - ${OZONE_VOLUME}/scm3:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+  recon:
+    <<: *common-env-file
+    volumes:
+      - ${OZONE_VOLUME}/recon:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+
+

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-debug-tools.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:HA-secure
+
+set -u -o pipefail
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=true
+export OM_SERVICE_ID=omservice
+export SCM=scm1.org
+export COMPOSE_FILE=docker-compose.yaml:debug-tools.yaml
+export OZONE_DIR=/opt/hadoop
+
+: "${OZONE_VOLUME_OWNER:=}"
+: "${OZONE_VOLUME:="${COMPOSE_DIR}/data"}"
+
+export OZONE_VOLUME
+
+# Clean up saved internal state from each container's volume for the next run.
+rm -rf "${OZONE_VOLUME}"
+mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,dn4,dn5,om1,om2,om3,scm1,scm2,scm3,recon,s3g,kms}
+
+if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
+  current_user=$(whoami)
+  if [[ "${OZONE_VOLUME_OWNER}" != "${current_user}" ]]; then
+    chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}" \
+      || sudo chown -R "${OZONE_VOLUME_OWNER}" "${OZONE_VOLUME}"
+  fi
+fi
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+execute_robot_test ${SCM} kinit.robot
+
+source "$COMPOSE_DIR/../common/replicas-test.sh"

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot

@@ -29,17 +29,18 @@ ${BUCKET}           cli-debug-bucket
 ${TESTFILE}         testfile
 ${EC_DATA}          3
 ${EC_PARITY}        2
+${OM_SERVICE_ID}    %{OM_SERVICE_ID}
 
 *** Keywords ***
 Create Volume Bucket
-    Execute             ozone sh volume create o3://om/${VOLUME}
-    Execute             ozone sh bucket create o3://om/${VOLUME}/${BUCKET}
+    Execute             ozone sh volume create o3://${OM_SERVICE_ID}/${VOLUME}
+    Execute             ozone sh bucket create o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}
 
 Create EC key
     [arguments]       ${bs}    ${count}    
 
     Execute           dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs} count=${count}
-    Execute           ozone sh key put o3://om/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
+    Execute           ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
 
 *** Test Cases ***
 0 data block
@@ -87,5 +88,5 @@ Create EC key
 
 Test ozone debug replicas chunk-info
     Create EC key     1048576    3
-    ${count} =        Execute           ozone debug replicas chunk-info o3://om/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length'
+    ${count} =        Execute           ozone debug replicas chunk-info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length'
     Should Be Equal As Integers         ${count}          5

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot

@@ -28,13 +28,14 @@ ${BUCKET}           cli-debug-bucket
 ${TESTFILE}         testfile
 ${EC_DATA}          6
 ${EC_PARITY}        3
+${OM_SERVICE_ID}    %{OM_SERVICE_ID}
 
 *** Keywords ***
 Create EC key
     [arguments]       ${bs}    ${count}
 
     Execute           dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs} count=${count}
-    Execute           ozone sh key put o3://om/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
+    Execute           ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
 
 *** Test Cases ***
 0 data block
@@ -95,5 +96,5 @@ Create EC key
 
 Test ozone debug replicas chunk-info
     Create EC key     1048576    6
-    ${count} =        Execute           ozone debug replicas chunk-info o3://om/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length'
+    ${count} =        Execute           ozone debug replicas chunk-info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length'
     Should Be Equal As Integers         ${count}           9

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot

@@ -27,17 +27,18 @@ ${VOLUME}           cli-debug-volume${PREFIX}
 ${BUCKET}           cli-debug-bucket
 ${DEBUGKEY}         debugKey
 ${TESTFILE}         testfile
+${OM_SERVICE_ID}    %{OM_SERVICE_ID}
 
 *** Keywords ***
 Write keys
-    Execute             ozone sh volume create o3://om/${VOLUME} --space-quota 100TB --namespace-quota 100
-    Execute             ozone sh bucket create o3://om/${VOLUME}/${BUCKET} --space-quota 1TB
+    Execute             ozone sh volume create o3://${OM_SERVICE_ID}/${VOLUME} --space-quota 100TB --namespace-quota 100
+    Execute             ozone sh bucket create o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET} --space-quota 1TB
     Execute             dd if=/dev/urandom of=${TEMP_DIR}/${TESTFILE} bs=100000 count=15
-    Execute             ozone sh key put o3://om/${VOLUME}/${BUCKET}/${TESTFILE} ${TEMP_DIR}/${TESTFILE}
+    Execute             ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} ${TEMP_DIR}/${TESTFILE}
 
 *** Test Cases ***
 Test ozone debug replicas verify checksums
-    ${output} =    Execute   ozone debug replicas verify --checksums o3://om/${VOLUME}/${BUCKET}/${TESTFILE} --output-dir ${TEMP_DIR}
+    ${output} =    Execute   ozone debug replicas verify --checksums o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} --output-dir ${TEMP_DIR}
     ${json} =      Evaluate  json.loads('''${output}''')      json
 
     # 'keys' array should be empty if all keys and their replicas passed checksum verification

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot

@@ -18,9 +18,12 @@ Documentation       Keyword definitions for Ozone Debug CLI tests
 Library             Collections
 Resource            ../lib/os.robot
 
+*** Variables ***
+${OM_SERVICE_ID}                    %{OM_SERVICE_ID}
+
 *** Keywords ***
 Execute replicas verify checksums CLI tool
-    Execute                         ozone debug -Dozone.network.topology.aware.read=true replicas verify --checksums --output-dir ${TEMP_DIR} o3://om/${VOLUME}/${BUCKET}/${TESTFILE}
+    Execute                         ozone debug -Dozone.network.topology.aware.read=true replicas verify --checksums --output-dir ${TEMP_DIR} o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE}
     ${directory} =                  Execute     ls -d ${TEMP_DIR}/${VOLUME}_${BUCKET}_${TESTFILE}_*/ | tail -n 1
     Directory Should Exist          ${directory}
     File Should Exist               ${directory}/${TESTFILE}_manifest