HDDS-12890. Add acceptance robot tests for ozone debug replicas verify

hadoop-ozone/dist/src/main/compose/common/replicas-test.sh

@@ -21,33 +21,45 @@ volume="cli-debug-volume${prefix}"
 bucket="cli-debug-bucket"
 key="testfile"
 
+dn_container="ozonesecure-ha-datanode1-1"
+container_db_path="/data/hdds/hdds/"
+local_db_backup_path="${COMPOSE_DIR}/container_db_backup"
+mkdir -p "${local_db_backup_path}"
+
+echo "Taking a backup of container.db"
+docker exec "${dn_container}" find "${container_db_path}" -name "container.db" | while read -r db; do
+  docker cp "${dn_container}:${db}" "${local_db_backup_path}/container.db"
+done
+
 execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests.robot
 
-#TODO HDDS-12890: Add acceptance robot tests for ozone debug replicas verify
 # get block locations for key
-#chunkinfo="${key}-blocks-${prefix}"
-#docker-compose exec -T ${SCM} bash -c "ozone debug replicas chunk-info ${volume}/${bucket}/${key}" > "$chunkinfo"
-#host="$(jq -r '.KeyLocations[0][0]["Datanode-HostName"]' ${chunkinfo})"
-#container="${host%%.*}"
-#
-## corrupt the first block of key on one of the datanodes
-#datafile="$(jq -r '.KeyLocations[0][0].Locations.files[0]' ${chunkinfo})"
-#docker exec "${container}" sed -i -e '1s/^/a/' "${datafile}"
-#
-#execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "CORRUPT_DATANODE:${host}" debug/ozone-debug-corrupt-block.robot
-#
-#docker stop "${container}"
-#
-#wait_for_datanode "${container}" STALE 60
-#execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "STALE_DATANODE:${host}" debug/ozone-debug-stale-datanode.robot
-#
-#wait_for_datanode "${container}" DEAD 60
-#execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-dead-datanode.robot
-#
-#docker start "${container}"
-#
-#wait_for_datanode "${container}" HEALTHY 60
+chunkinfo="${key}-blocks-${prefix}"
+docker-compose exec -T ${SCM} bash -c "ozone debug replicas chunk-info ${volume}/${bucket}/${key}" > "$chunkinfo"
+host="$(jq -r '.keyLocations[0][0].datanode["hostname"]' ${chunkinfo})"
+container="${host%%.*}"
+
+# corrupt the first block of key on one of the datanodes
+datafile="$(jq -r '.keyLocations[0][0].file' ${chunkinfo})"
+docker exec "${container}" sed -i -e '1s/^/a/' "${datafile}"
+
+execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "CORRUPT_DATANODE:${host}" debug/corrupt-block-checksum.robot
+
+echo "Overwriting container.db with the backup db"
+target_container_dir=$(docker exec "${container}" find "${container_db_path}" -name "container.db" | xargs dirname)
+docker cp "${local_db_backup_path}/container.db" "${container}:${target_container_dir}/"
+docker exec "${container}" sudo chown -R hadoop:hadoop "${target_container_dir}"
+
+docker stop "${container}"
+
+wait_for_datanode "${container}" STALE 60
+execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "STALE_DATANODE:${host}" debug/stale-datanode-checksum.robot
+
+docker start "${container}"
+
+wait_for_datanode "${container}" HEALTHY 60
+
+execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "DATANODE:${host}" debug/block-existence-check.robot
 
-#start_docker_env 9
-#execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec3-2.robot
-#execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec6-3.robot
+execute_robot_test ${OM} kinit.robot
+execute_robot_test ${OM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec3-2.robot

hadoop-ozone/dist/src/main/compose/ozone/test.sh

@@ -49,7 +49,6 @@ execute_robot_test scm cli
 execute_robot_test scm admincli
 
 execute_robot_test scm -v USERNAME:httpfs httpfs
-source "$COMPOSE_DIR/../common/replicas-test.sh"
 
 execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-o3fs-bucket ozonefs/ozonefs.robot
 execute_robot_test scm -v SCHEME:ofs -N ozonefs-obs ozonefs/ozonefs-obs.robot

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config

@@ -47,6 +47,8 @@ OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
 OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
 OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.scm.stale.node.interval=30s
+OZONE-SITE.XML_hdds.heartbeat.interval=5s
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_ozone.scm.client.address=scm

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot → hadoop-ozone/dist/src/main/smoketest/debug/block-existence-check.robot

@@ -14,21 +14,22 @@
 # limitations under the License.
 
 *** Settings ***
-Documentation       Test checksums in case of one datanode is dead
+Documentation       Test existence of a block on a datanode
 Library             OperatingSystem
 Resource            ../lib/os.robot
-Resource            ozone-debug.robot
+Resource            ozone-debug-keywords.robot
 Test Timeout        5 minute
+
 *** Variables ***
 ${PREFIX}           ${EMPTY}
+${DATANODE}         ${EMPTY}
 ${VOLUME}           cli-debug-volume${PREFIX}
 ${BUCKET}           cli-debug-bucket
 ${TESTFILE}         testfile
+${CHECK_TYPE}       blockExistence
 
 *** Test Cases ***
-Test ozone debug checksums with one datanode DEAD
-    ${directory} =                 Execute replicas verify checksums CLI tool
-    Set Test Variable    ${DIR}         ${directory}
-
-    ${count_files} =               Count Files In Directory    ${directory}
-    Should Be Equal As Integers    ${count_files}     1
+Test block existence with a block missing on a replica
+    ${output} =         Execute replicas verify block existence debug tool
+    ${json} =           Parse replicas verify JSON output    ${output}
+    Check to Verify Replicas    ${json}  ${CHECK_TYPE}  ${DATANODE}  Unable to find the block

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot → hadoop-ozone/dist/src/main/smoketest/debug/corrupt-block-checksum.robot

@@ -14,33 +14,23 @@
 # limitations under the License.
 
 *** Settings ***
-Documentation       Test read-replicas in case of a corrupt replica
+Documentation       Test checksums on a corrupt block replica
 Library             OperatingSystem
 Resource            ../lib/os.robot
-Resource            ozone-debug.robot
+Resource            ozone-debug-keywords.robot
 Test Timeout        5 minute
+
 *** Variables ***
 ${PREFIX}              ${EMPTY}
+${CORRUPT_DATANODE}    ${EMPTY}
 ${VOLUME}              cli-debug-volume${PREFIX}
 ${BUCKET}              cli-debug-bucket
 ${TESTFILE}            testfile
-${CORRUPT_DATANODE}    ozone_datanode_1.ozone_default
-
-*** Test Cases ***
-Test ozone debug checksums with corrupt block replica
-    ${directory} =                      Execute replicas verify checksums CLI tool
-    Set Test Variable    ${DIR}         ${directory}
+${CHECK_TYPE}          checksum
 
-    ${count_files} =                    Count Files In Directory    ${directory}
-    Should Be Equal As Integers         ${count_files}     1
 
-    ${json} =                           Read Replicas Manifest
-    ${md5sum} =                         Execute     md5sum ${TEMP_DIR}/${TESTFILE} | awk '{print $1}'
-
-    FOR    ${replica}    IN RANGE    3
-        ${datanode} =    Set Variable    ${json}[blocks][0][replicas][${replica}][hostname]
-
-        IF    '${datanode}' == '${CORRUPT_DATANODE}'
-            Should Contain           ${json}[blocks][0][replicas][${replica}][exception]    Checksum mismatch
-        END
-    END
+*** Test Cases ***
+Test checksums with a corrupt block replica
+    ${output} =         Execute replicas verify checksums debug tool
+    ${json} =           Parse replicas verify JSON output    ${output}
+    Check to Verify Replicas    ${json}  ${CHECK_TYPE}  ${CORRUPT_DATANODE}  Checksum mismatched

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Keyword definitions for Ozone Debug CLI tests
+Library             Collections
+Resource            ../lib/os.robot
+
+*** Variables ***
+${OM_SERVICE_ID}                    %{OM_SERVICE_ID}
+
+*** Keywords ***
+Execute replicas verify checksums debug tool
+    ${output}      Execute          ozone debug replicas verify --checksums o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} --all-results
+    [Return]       ${output}
+
+Execute replicas verify block existence debug tool
+    ${output}      Execute          ozone debug replicas verify --block-existence o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} --all-results
+    [Return]       ${output}
+
+Parse replicas verify JSON output
+    [Arguments]    ${output}
+    ${json} =      Evaluate    json.loads('''${output}''')    json
+    [Return]       ${json}
+
+Check to Verify Replicas
+    [Arguments]    ${json}  ${check_type}  ${faulty_datanode}  ${expected_message}
+    ${replicas} =    Get From Dictionary    ${json['keys'][0]['blocks'][0]}    replicas
+    FOR    ${replica}    IN    @{replicas}
+        ${datanode} =     Get From Dictionary    ${replica}    datanode
+        ${hostname} =     Get From Dictionary    ${datanode}   hostname
+        Run Keyword If    '${hostname}' == '${faulty_datanode}'    Check Replica Failed    ${replica}  ${check_type}  ${expected_message}
+        Run Keyword If    '${hostname}' != '${faulty_datanode}'    Check Replica Passed    ${replica}  ${check_type}
+    END
+
+Check Replica Failed
+    [Arguments]    ${replica}  ${check_type}  ${expected_message}
+    ${checks} =     Get From Dictionary    ${replica}    checks
+    ${check} =      Get From List          ${checks}     0
+    Should Be Equal    ${check['type']}    ${check_type}
+    Should Be Equal    ${check['pass']}    ${False}
+    Should Contain     ${check['failures'][0]['message']}    ${expected_message}
+
+Check Replica Passed
+    [Arguments]    ${replica}    ${check_type}
+    ${checks} =    Get From Dictionary    ${replica}    checks
+    ${check} =     Get From List          ${checks}     0
+    Should Be Equal   ${check['type']}    ${check_type}
+    Should Be True    ${check['completed']}
+    Should Be True    ${check['pass']}
+    Should Be Empty   ${check['failures']}

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot

@@ -18,14 +18,14 @@ Documentation       Test ozone Debug CLI for EC(3,2) replicated keys
 Library             OperatingSystem
 Library             Process
 Resource            ../lib/os.robot
-Resource            ozone-debug.robot
+Resource            ozone-debug-keywords.robot
 Test Timeout        5 minute
 Suite Setup         Create Volume Bucket
 
 *** Variables ***
 ${PREFIX}           ${EMPTY}
-${VOLUME}           cli-debug-volume${PREFIX}
-${BUCKET}           cli-debug-bucket
+${VOLUME}           cli-debug-ec-volume${PREFIX}
+${BUCKET}           cli-debug-ec-bucket
 ${TESTFILE}         testfile
 ${EC_DATA}          3
 ${EC_PARITY}        2
@@ -43,49 +43,6 @@ Create EC key
     Execute           ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
 
 *** Test Cases ***
-0 data block
-    Create EC key     1000    0
-    ${directory} =                      Execute replicas verify checksums CLI tool
-    ${count_files} =                    Count Files In Directory    ${directory}
-    Should Be Equal As Integers         ${count_files}     1
-
-1 data block
-    Create EC key     1048576    1
-    ${directory} =                      Execute replicas verify checksums CLI tool
-    ${count_files} =                    Count Files In Directory    ${directory}
-    Should Be Equal As Integers         ${count_files}     1
-
-2 data blocks
-    Create EC key     1048576    2
-    ${directory} =                      Execute replicas verify checksums CLI tool
-    ${count_files} =                    Count Files In Directory    ${directory}
-    Should Be Equal As Integers         ${count_files}     1
-
-3 data blocks
-    Create EC key     1048576    3
-    ${directory} =                      Execute replicas verify checksums CLI tool
-    ${count_files} =                    Count Files In Directory    ${directory}
-    Should Be Equal As Integers         ${count_files}     1
-
-3 data blocks and partial stripe
-    Create EC key     1000000    4
-    ${directory} =                      Execute replicas verify checksums CLI tool
-    ${count_files} =                    Count Files In Directory    ${directory}
-    ${sum_size_last_stripe} =           Evaluate     ((1000000 * 4) % 1048576) * 3
-    Should Be Equal As Integers         ${count_files}     1
-
-4 data blocks and partial stripe
-    Create EC key     1000000    5
-    ${directory} =                      Execute replicas verify checksums CLI tool
-    ${count_files} =                    Count Files In Directory    ${directory}
-    Should Be Equal As Integers         ${count_files}     1
-
-6 data blocks
-    Create EC key     1048576    6
-    ${directory} =                      Execute replicas verify checksums CLI tool
-    ${count_files} =                    Count Files In Directory    ${directory}
-    Should Be Equal As Integers         ${count_files}     1
-
 Test ozone debug replicas chunk-info
     Create EC key     1048576    3
     ${count} =        Execute           ozone debug replicas chunk-info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length'