diff --git a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh index b875bfcafc8e..520ab8c47909 100755 --- a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh @@ -21,33 +21,45 @@ volume="cli-debug-volume${prefix}" bucket="cli-debug-bucket" key="testfile" +dn_container="ozonesecure-ha-datanode1-1" +container_db_path="/data/hdds/hdds/" +local_db_backup_path="${COMPOSE_DIR}/container_db_backup" +mkdir -p "${local_db_backup_path}" + +echo "Taking a backup of container.db" +docker exec "${dn_container}" find "${container_db_path}" -name "container.db" | while read -r db; do + docker cp "${dn_container}:${db}" "${local_db_backup_path}/container.db" +done + execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests.robot -#TODO HDDS-12890: Add acceptance robot tests for ozone debug replicas verify # get block locations for key -#chunkinfo="${key}-blocks-${prefix}" -#docker-compose exec -T ${SCM} bash -c "ozone debug replicas chunk-info ${volume}/${bucket}/${key}" > "$chunkinfo" -#host="$(jq -r '.KeyLocations[0][0]["Datanode-HostName"]' ${chunkinfo})" -#container="${host%%.*}" -# -## corrupt the first block of key on one of the datanodes -#datafile="$(jq -r '.KeyLocations[0][0].Locations.files[0]' ${chunkinfo})" -#docker exec "${container}" sed -i -e '1s/^/a/' "${datafile}" -# -#execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "CORRUPT_DATANODE:${host}" debug/ozone-debug-corrupt-block.robot -# -#docker stop "${container}" -# -#wait_for_datanode "${container}" STALE 60 -#execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "STALE_DATANODE:${host}" debug/ozone-debug-stale-datanode.robot -# -#wait_for_datanode "${container}" DEAD 60 -#execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-dead-datanode.robot -# -#docker start "${container}" -# -#wait_for_datanode "${container}" HEALTHY 60 +chunkinfo="${key}-blocks-${prefix}" +docker-compose exec -T ${SCM} bash -c "ozone debug replicas chunk-info ${volume}/${bucket}/${key}" > "$chunkinfo" +host="$(jq -r '.keyLocations[0][0].datanode["hostname"]' ${chunkinfo})" +container="${host%%.*}" + +# corrupt the first block of key on one of the datanodes +datafile="$(jq -r '.keyLocations[0][0].file' ${chunkinfo})" +docker exec "${container}" sed -i -e '1s/^/a/' "${datafile}" + +execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "CORRUPT_DATANODE:${host}" debug/corrupt-block-checksum.robot + +echo "Overwriting container.db with the backup db" +target_container_dir=$(docker exec "${container}" find "${container_db_path}" -name "container.db" | xargs dirname) +docker cp "${local_db_backup_path}/container.db" "${container}:${target_container_dir}/" +docker exec "${container}" sudo chown -R hadoop:hadoop "${target_container_dir}" + +docker stop "${container}" + +wait_for_datanode "${container}" STALE 60 +execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "STALE_DATANODE:${host}" debug/stale-datanode-checksum.robot + +docker start "${container}" + +wait_for_datanode "${container}" HEALTHY 60 + +execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "DATANODE:${host}" debug/block-existence-check.robot -#start_docker_env 9 -#execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec3-2.robot -#execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec6-3.robot +execute_robot_test ${OM} kinit.robot +execute_robot_test ${OM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec3-2.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh index 929632a07d01..653a0aaf766e 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@ -49,7 +49,6 @@ execute_robot_test scm cli execute_robot_test scm admincli execute_robot_test scm -v USERNAME:httpfs httpfs -source "$COMPOSE_DIR/../common/replicas-test.sh" execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-o3fs-bucket ozonefs/ozonefs.robot execute_robot_test scm -v SCHEME:ofs -N ozonefs-obs ozonefs/ozonefs-obs.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config index 9e2e7cbf0a04..c5ab0bf9443f 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config @@ -47,6 +47,8 @@ OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.scm.stale.node.interval=30s +OZONE-SITE.XML_hdds.heartbeat.interval=5s OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot b/hadoop-ozone/dist/src/main/smoketest/debug/block-existence-check.robot similarity index 69% rename from hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot rename to hadoop-ozone/dist/src/main/smoketest/debug/block-existence-check.robot index 33a2b78c3df0..4c319f169d3e 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/block-existence-check.robot @@ -14,21 +14,22 @@ # limitations under the License. *** Settings *** -Documentation Test checksums in case of one datanode is dead +Documentation Test existence of a block on a datanode Library OperatingSystem Resource ../lib/os.robot -Resource ozone-debug.robot +Resource ozone-debug-keywords.robot Test Timeout 5 minute + *** Variables *** ${PREFIX} ${EMPTY} +${DATANODE} ${EMPTY} ${VOLUME} cli-debug-volume${PREFIX} ${BUCKET} cli-debug-bucket ${TESTFILE} testfile +${CHECK_TYPE} blockExistence *** Test Cases *** -Test ozone debug checksums with one datanode DEAD - ${directory} = Execute replicas verify checksums CLI tool - Set Test Variable ${DIR} ${directory} - - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 +Test block existence with a block missing on a replica + ${output} = Execute replicas verify block existence debug tool + ${json} = Parse replicas verify JSON output ${output} + Check to Verify Replicas ${json} ${CHECK_TYPE} ${DATANODE} Unable to find the block diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot b/hadoop-ozone/dist/src/main/smoketest/debug/corrupt-block-checksum.robot similarity index 52% rename from hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot rename to hadoop-ozone/dist/src/main/smoketest/debug/corrupt-block-checksum.robot index ba17ca9f085a..4bd06e1e5f94 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/corrupt-block-checksum.robot @@ -14,33 +14,23 @@ # limitations under the License. *** Settings *** -Documentation Test read-replicas in case of a corrupt replica +Documentation Test checksums on a corrupt block replica Library OperatingSystem Resource ../lib/os.robot -Resource ozone-debug.robot +Resource ozone-debug-keywords.robot Test Timeout 5 minute + *** Variables *** ${PREFIX} ${EMPTY} +${CORRUPT_DATANODE} ${EMPTY} ${VOLUME} cli-debug-volume${PREFIX} ${BUCKET} cli-debug-bucket ${TESTFILE} testfile -${CORRUPT_DATANODE} ozone_datanode_1.ozone_default - -*** Test Cases *** -Test ozone debug checksums with corrupt block replica - ${directory} = Execute replicas verify checksums CLI tool - Set Test Variable ${DIR} ${directory} +${CHECK_TYPE} checksum - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - ${json} = Read Replicas Manifest - ${md5sum} = Execute md5sum ${TEMP_DIR}/${TESTFILE} | awk '{print $1}' - - FOR ${replica} IN RANGE 3 - ${datanode} = Set Variable ${json}[blocks][0][replicas][${replica}][hostname] - - IF '${datanode}' == '${CORRUPT_DATANODE}' - Should Contain ${json}[blocks][0][replicas][${replica}][exception] Checksum mismatch - END - END +*** Test Cases *** +Test checksums with a corrupt block replica + ${output} = Execute replicas verify checksums debug tool + ${json} = Parse replicas verify JSON output ${output} + Check to Verify Replicas ${json} ${CHECK_TYPE} ${CORRUPT_DATANODE} Checksum mismatched diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot new file mode 100644 index 000000000000..19257a62c72d --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Keyword definitions for Ozone Debug CLI tests +Library Collections +Resource ../lib/os.robot + +*** Variables *** +${OM_SERVICE_ID} %{OM_SERVICE_ID} + +*** Keywords *** +Execute replicas verify checksums debug tool + ${output} Execute ozone debug replicas verify --checksums o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} --all-results + [Return] ${output} + +Execute replicas verify block existence debug tool + ${output} Execute ozone debug replicas verify --block-existence o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} --all-results + [Return] ${output} + +Parse replicas verify JSON output + [Arguments] ${output} + ${json} = Evaluate json.loads('''${output}''') json + [Return] ${json} + +Check to Verify Replicas + [Arguments] ${json} ${check_type} ${faulty_datanode} ${expected_message} + ${replicas} = Get From Dictionary ${json['keys'][0]['blocks'][0]} replicas + FOR ${replica} IN @{replicas} + ${datanode} = Get From Dictionary ${replica} datanode + ${hostname} = Get From Dictionary ${datanode} hostname + Run Keyword If '${hostname}' == '${faulty_datanode}' Check Replica Failed ${replica} ${check_type} ${expected_message} + Run Keyword If '${hostname}' != '${faulty_datanode}' Check Replica Passed ${replica} ${check_type} + END + +Check Replica Failed + [Arguments] ${replica} ${check_type} ${expected_message} + ${checks} = Get From Dictionary ${replica} checks + ${check} = Get From List ${checks} 0 + Should Be Equal ${check['type']} ${check_type} + Should Be Equal ${check['pass']} ${False} + Should Contain ${check['failures'][0]['message']} ${expected_message} + +Check Replica Passed + [Arguments] ${replica} ${check_type} + ${checks} = Get From Dictionary ${replica} checks + ${check} = Get From List ${checks} 0 + Should Be Equal ${check['type']} ${check_type} + Should Be True ${check['completed']} + Should Be True ${check['pass']} + Should Be Empty ${check['failures']} diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot deleted file mode 100644 index 69448f21e6b9..000000000000 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation Test checksums in case of one datanode is stale -Library OperatingSystem -Resource ../lib/os.robot -Resource ozone-debug.robot -Test Timeout 5 minute -*** Variables *** -${PREFIX} ${EMPTY} -${VOLUME} cli-debug-volume${PREFIX} -${BUCKET} cli-debug-bucket -${TESTFILE} testfile -${STALE_DATANODE} ozone_datanode_1.ozone_default - -*** Test Cases *** -Test ozone debug checksums with one datanode STALE - ${directory} = Execute replicas verify checksums CLI tool - Set Test Variable ${DIR} ${directory} - - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - - ${json} = Read Replicas Manifest - ${md5sum} = Execute md5sum ${TEMP_DIR}/${TESTFILE} | awk '{print $1}' - - FOR ${replica} IN RANGE 3 - ${datanode} = Set Variable ${json}[blocks][0][replicas][${replica}][hostname] - - IF '${datanode}' == '${STALE_DATANODE}' - Verify Stale Replica ${json} ${replica} - END - END diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot index e10e03443d60..7b88f97254c9 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot @@ -18,14 +18,14 @@ Documentation Test ozone Debug CLI for EC(3,2) replicated keys Library OperatingSystem Library Process Resource ../lib/os.robot -Resource ozone-debug.robot +Resource ozone-debug-keywords.robot Test Timeout 5 minute Suite Setup Create Volume Bucket *** Variables *** ${PREFIX} ${EMPTY} -${VOLUME} cli-debug-volume${PREFIX} -${BUCKET} cli-debug-bucket +${VOLUME} cli-debug-ec-volume${PREFIX} +${BUCKET} cli-debug-ec-bucket ${TESTFILE} testfile ${EC_DATA} 3 ${EC_PARITY} 2 @@ -43,49 +43,6 @@ Create EC key Execute ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC *** Test Cases *** -0 data block - Create EC key 1000 0 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -1 data block - Create EC key 1048576 1 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -2 data blocks - Create EC key 1048576 2 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -3 data blocks - Create EC key 1048576 3 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -3 data blocks and partial stripe - Create EC key 1000000 4 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - ${sum_size_last_stripe} = Evaluate ((1000000 * 4) % 1048576) * 3 - Should Be Equal As Integers ${count_files} 1 - -4 data blocks and partial stripe - Create EC key 1000000 5 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -6 data blocks - Create EC key 1048576 6 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - Test ozone debug replicas chunk-info Create EC key 1048576 3 ${count} = Execute ozone debug replicas chunk-info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length' diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot deleted file mode 100644 index c3b8c7ad1a3a..000000000000 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot +++ /dev/null @@ -1,100 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation Test ozone Debug CLI for EC(6,3) replicated key -Library OperatingSystem -Library Process -Resource ../lib/os.robot -Resource ozone-debug.robot -Test Timeout 5 minute - -*** Variables *** -${PREFIX} ${EMPTY} -${VOLUME} cli-debug-volume${PREFIX} -${BUCKET} cli-debug-bucket -${TESTFILE} testfile -${EC_DATA} 6 -${EC_PARITY} 3 -${OM_SERVICE_ID} %{OM_SERVICE_ID} - -*** Keywords *** -Create EC key - [arguments] ${bs} ${count} - - Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs} count=${count} - Execute ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC - -*** Test Cases *** -0 data block - Create EC key 1048576 0 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -1 data block - Create EC key 1048576 1 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -2 data blocks - Create EC key 1048576 2 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -3 data blocks - Create EC key 1048576 3 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -4 data blocks - Create EC key 1048576 4 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -5 data blocks - Create EC key 1048576 5 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -6 data blocks - Create EC key 1048576 6 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 - -6 data blocks and partial stripe - Create EC key 1000000 7 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - ${sum_size_last_stripe} = Evaluate ((1000000 * 7) % 1048576) * 4 - Should Be Equal As Integers ${count_files} 1 - -7 data blocks and partial stripe - Create EC key 1000000 8 - ${directory} = Execute replicas verify checksums CLI tool - ${count_files} = Count Files In Directory ${directory} - ${sum_size_last_stripe} = Evaluate 1048576 * 4 + ((1000000 * 8) % 1048576) - Should Be Equal As Integers ${count_files} 1 - -Test ozone debug replicas chunk-info - Create EC key 1048576 6 - ${count} = Execute ozone debug replicas chunk-info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length' - Should Be Equal As Integers ${count} 9 diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot index 9364c98a6ce0..60e7b69a2ceb 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot @@ -14,10 +14,10 @@ # limitations under the License. *** Settings *** -Documentation Test ozone Debug CLI +Documentation Test ozone debug CLI Library OperatingSystem Resource ../lib/os.robot -Resource ozone-debug.robot +Resource ozone-debug-keywords.robot Test Timeout 5 minute Suite Setup Write keys @@ -38,9 +38,9 @@ Write keys *** Test Cases *** Test ozone debug replicas verify checksums ${output} = Execute ozone debug replicas verify --checksums --block-existence --container-state o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} - ${json} = Evaluate json.loads('''${output}''') json + ${json} = Parse replicas verify JSON output ${output} - # 'keys' array should be empty if all keys and their replicas passed checksum verification + # 'keys' array should be empty if all keys and their replicas passed Should Be Empty ${json}[keys] Should Be True ${json}[pass] ${True} diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot deleted file mode 100644 index c3cb8d2cf15f..000000000000 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot +++ /dev/null @@ -1,98 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -*** Settings *** -Documentation Keyword definitions for Ozone Debug CLI tests -Library Collections -Resource ../lib/os.robot - -*** Variables *** -${OM_SERVICE_ID} %{OM_SERVICE_ID} - -*** Keywords *** -Execute replicas verify checksums CLI tool - Execute ozone debug -Dozone.network.topology.aware.read=true replicas verify --checksums o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} - ${directory} = Execute ls -d ${TEMP_DIR}/${VOLUME}_${BUCKET}_${TESTFILE}_*/ | tail -n 1 - Directory Should Exist ${directory} - File Should Exist ${directory}/${TESTFILE}_manifest - [Return] ${directory} - -Read Replicas Manifest - ${manifest} = Get File ${DIR}/${TESTFILE}_manifest - ${json} = Evaluate json.loads('''${manifest}''') json - Validate JSON ${json} - [return] ${json} - -Validate JSON - [arguments] ${json} - Should Be Equal ${json}[filename] ${VOLUME}/${BUCKET}/${TESTFILE} - ${file_size} = Get File Size ${TEMP_DIR}/${TESTFILE} - Should Be Equal ${json}[datasize] ${file_size} - Should Be Equal As Integers ${json}[blocks][0][blockIndex] 1 - Should Not Be Empty Convert To String ${json}[blocks][0][containerId] - Should Not Be Empty Convert To String ${json}[blocks][0][localId] - Should Be Equal As Integers ${json}[blocks][0][length] 1048576 - Should Not Be Empty Convert To String ${json}[blocks][0][offset] - Should Be Equal As Integers ${json}[blocks][1][blockIndex] 2 - Should Not Be Empty Convert To String ${json}[blocks][1][containerId] - Should Not Be Empty Convert To String ${json}[blocks][1][localId] - Should Be Equal As Integers ${json}[blocks][1][length] 451424 - Should Not Be Empty Convert To String ${json}[blocks][1][offset] - -Get Replica Filenames - [arguments] ${json} ${replica} - - ${list} = Create List - - FOR ${block} IN RANGE 2 - ${datanode} = Set Variable ${json}[blocks][${block}][replicas][${replica}][hostname] - ${n} = Evaluate ${block} + 1 - Append To List ${list} ${DIR}/${TESTFILE}_block${n}_${datanode} - END - - ${filenames} = Catenate @{list} - - [return] ${filenames} - -Verify Healthy Replica - [arguments] ${json} ${replica} ${expected_md5sum} - - ${block_filenames} = Get Replica Filenames ${json} ${replica} - ${md5sum} = Execute cat ${block_filenames} | md5sum | awk '{print $1}' - Should Be Equal ${md5sum} ${expected_md5sum} - -Verify Healthy EC Replica - [arguments] ${directory} ${block} ${expected_block_size} - - ${block_size} = Execute ls -l ${directory} | grep "testfile_block${block}_ozone-datanode-.*\.ozone_default" | awk '{sum += $5} END {print sum}' - Should Be Equal As Integers ${block_size} ${expected_block_size} - -Verify Corrupt Replica - [arguments] ${json} ${replica} ${valid_md5sum} - - ${block_filenames} = Get Replica Filenames ${json} ${replica} - ${md5sum} = Execute cat ${block_filenames} | md5sum | awk '{print $1}' - Should Not Be Equal ${md5sum} ${valid_md5sum} - -Verify Stale Replica - [arguments] ${json} ${replica} - - FOR ${block} IN RANGE 2 - ${datanode} = Set Variable ${json}[blocks][${block}][replicas][${replica}][hostname] - - IF '${datanode}' == '${STALE_DATANODE}' - Should Contain ${json}[blocks][${block}][replicas][${replica}][exception] UNAVAILABLE - END - END diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/stale-datanode-checksum.robot b/hadoop-ozone/dist/src/main/smoketest/debug/stale-datanode-checksum.robot new file mode 100644 index 000000000000..56f2133cd0fc --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/debug/stale-datanode-checksum.robot @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test checksums in case of a stale datanode +Library OperatingSystem +Resource ../lib/os.robot +Resource ozone-debug-keywords.robot +Test Timeout 5 minute + +*** Variables *** +${PREFIX} ${EMPTY} +${STALE_DATANODE} ${EMPTY} +${VOLUME} cli-debug-volume${PREFIX} +${BUCKET} cli-debug-bucket +${TESTFILE} testfile +${CHECK_TYPE} checksum + +*** Test Cases *** +Test checksums with a stale datanode + ${output} = Execute replicas verify checksums debug tool + ${json} = Parse replicas verify JSON output ${output} + Check to Verify Replicas ${json} ${CHECK_TYPE} ${STALE_DATANODE} UNAVAILABLE