diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot index 20689b7c0f5e..ba17ca9f085a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot @@ -32,7 +32,7 @@ Test ozone debug checksums with corrupt block replica Set Test Variable ${DIR} ${directory} ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 7 + Should Be Equal As Integers ${count_files} 1 ${json} = Read Replicas Manifest ${md5sum} = Execute md5sum ${TEMP_DIR}/${TESTFILE} | awk '{print $1}' @@ -41,9 +41,6 @@ Test ozone debug checksums with corrupt block replica ${datanode} = Set Variable ${json}[blocks][0][replicas][${replica}][hostname] IF '${datanode}' == '${CORRUPT_DATANODE}' - Verify Corrupt Replica ${json} ${replica} ${md5sum} Should Contain ${json}[blocks][0][replicas][${replica}][exception] Checksum mismatch - ELSE - Verify Healthy Replica ${json} ${replica} ${md5sum} END END diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot index 42ae5dec7e92..33a2b78c3df0 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-dead-datanode.robot @@ -31,11 +31,4 @@ Test ozone debug checksums with one datanode DEAD Set Test Variable ${DIR} ${directory} ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 5 - - ${json} = Read Replicas Manifest - ${md5sum} = Execute md5sum ${TEMP_DIR}/${TESTFILE} | awk '{print $1}' - - FOR ${replica} IN RANGE 2 - Verify Healthy Replica ${json} ${replica} ${md5sum} - END + Should Be Equal As Integers ${count_files} 1 diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot index 36cef5e66518..69448f21e6b9 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot @@ -32,7 +32,7 @@ Test ozone debug checksums with one datanode STALE Set Test Variable ${DIR} ${directory} ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 7 + Should Be Equal As Integers ${count_files} 1 ${json} = Read Replicas Manifest ${md5sum} = Execute md5sum ${TEMP_DIR}/${TESTFILE} | awk '{print $1}' @@ -42,7 +42,5 @@ Test ozone debug checksums with one datanode STALE IF '${datanode}' == '${STALE_DATANODE}' Verify Stale Replica ${json} ${replica} - ELSE - Verify Healthy Replica ${json} ${replica} ${md5sum} END END diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot index 57227458cc15..c578ac76cc29 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot @@ -52,52 +52,35 @@ Create EC key Create EC key 1048576 1 ${directory} = Execute replicas verify checksums CLI tool ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 6 - ${sum_size} = Evaluate 1048576 * 3 - Verify Healthy EC Replica ${directory} 1 ${sum_size} + Should Be Equal As Integers ${count_files} 1 2 data blocks Create EC key 1048576 2 ${directory} = Execute replicas verify checksums CLI tool - ${sum_size} = Evaluate 1048576 * 4 ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 6 - Verify Healthy EC Replica ${directory} 1 ${sum_size} + Should Be Equal As Integers ${count_files} 1 3 data blocks Create EC key 1048576 3 ${directory} = Execute replicas verify checksums CLI tool - ${sum_size} = Evaluate 1048576 * 5 ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 6 - Verify Healthy EC Replica ${directory} 1 ${sum_size} - + Should Be Equal As Integers ${count_files} 1 + 3 data blocks and partial stripe Create EC key 1000000 4 ${directory} = Execute replicas verify checksums CLI tool ${count_files} = Count Files In Directory ${directory} - ${sum_size} = Evaluate 1048576 * 5 ${sum_size_last_stripe} = Evaluate ((1000000 * 4) % 1048576) * 3 - Should Be Equal As Integers ${count_files} 11 - Verify Healthy EC Replica ${directory} 1 ${sum_size} - Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} + Should Be Equal As Integers ${count_files} 1 4 data blocks and partial stripe Create EC key 1000000 5 ${directory} = Execute replicas verify checksums CLI tool ${count_files} = Count Files In Directory ${directory} - ${sum_size} = Evaluate 1048576 * 5 - ${sum_size_last_stripe} = Evaluate 1048576 * 3 + ((1000000 * 5) % 1048576) - Should Be Equal As Integers ${count_files} 11 - Verify Healthy EC Replica ${directory} 1 ${sum_size} - Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} + Should Be Equal As Integers ${count_files} 1 6 data blocks Create EC key 1048576 6 ${directory} = Execute replicas verify checksums CLI tool ${count_files} = Count Files In Directory ${directory} - ${sum_size} = Evaluate 1048576 * 5 - Should Be Equal As Integers ${count_files} 11 - FOR ${block} IN RANGE 1 3 - Verify Healthy EC Replica ${directory} ${block} ${sum_size} - END + Should Be Equal As Integers ${count_files} 1 diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot index 52d48c25f77d..7815e8ef4f75 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot @@ -47,66 +47,48 @@ Create EC key Create EC key 1048576 1 ${directory} = Execute replicas verify checksums CLI tool ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 10 - ${sum_size} = Evaluate 1048576 * 4 - Verify Healthy EC Replica ${directory} 1 ${sum_size} + Should Be Equal As Integers ${count_files} 1 2 data blocks Create EC key 1048576 2 ${directory} = Execute replicas verify checksums CLI tool - ${sum_size} = Evaluate 1048576 * 5 ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 10 - Verify Healthy EC Replica ${directory} 1 ${sum_size} + Should Be Equal As Integers ${count_files} 1 3 data blocks Create EC key 1048576 3 ${directory} = Execute replicas verify checksums CLI tool - ${sum_size} = Evaluate 1048576 * 6 ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 10 - Verify Healthy EC Replica ${directory} 1 ${sum_size} - + Should Be Equal As Integers ${count_files} 1 + 4 data blocks Create EC key 1048576 4 ${directory} = Execute replicas verify checksums CLI tool ${count_files} = Count Files In Directory ${directory} - ${sum_size} = Evaluate 1048576 * 7 - Should Be Equal As Integers ${count_files} 10 - Verify Healthy EC Replica ${directory} 1 ${sum_size} + Should Be Equal As Integers ${count_files} 1 5 data blocks Create EC key 1048576 5 ${directory} = Execute replicas verify checksums CLI tool ${count_files} = Count Files In Directory ${directory} - ${sum_size} = Evaluate 1048576 * 8 - Should Be Equal As Integers ${count_files} 10 - Verify Healthy EC Replica ${directory} 1 ${sum_size} + Should Be Equal As Integers ${count_files} 1 6 data blocks Create EC key 1048576 6 ${directory} = Execute replicas verify checksums CLI tool ${count_files} = Count Files In Directory ${directory} - ${sum_size} = Evaluate 1048576 * 9 - Should Be Equal As Integers ${count_files} 10 - Verify Healthy EC Replica ${directory} 1 ${sum_size} + Should Be Equal As Integers ${count_files} 1 6 data blocks and partial stripe Create EC key 1000000 7 ${directory} = Execute replicas verify checksums CLI tool ${count_files} = Count Files In Directory ${directory} - ${sum_size} = Evaluate 1048576 * 9 ${sum_size_last_stripe} = Evaluate ((1000000 * 7) % 1048576) * 4 - Should Be Equal As Integers ${count_files} 19 - Verify Healthy EC Replica ${directory} 1 ${sum_size} - Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} + Should Be Equal As Integers ${count_files} 1 7 data blocks and partial stripe Create EC key 1000000 8 ${directory} = Execute replicas verify checksums CLI tool ${count_files} = Count Files In Directory ${directory} - ${sum_size} = Evaluate 1048576 * 9 ${sum_size_last_stripe} = Evaluate 1048576 * 4 + ((1000000 * 8) % 1048576) - Should Be Equal As Integers ${count_files} 19 - Verify Healthy EC Replica ${directory} 1 ${sum_size} - Verify Healthy EC Replica ${directory} 2 ${sum_size_last_stripe} + Should Be Equal As Integers ${count_files} 1 diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot index 803ab19ade84..02de6794b237 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot @@ -41,14 +41,7 @@ Test ozone debug read-replicas Set Test Variable ${DIR} ${directory} ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 7 - - ${json} = Read Replicas Manifest - ${md5sum} = Execute md5sum ${TEMP_DIR}/${TESTFILE} | awk '{print $1}' - - FOR ${replica} IN RANGE 3 - Verify Healthy Replica ${json} ${replica} ${md5sum} - END + Should Be Equal As Integers ${count_files} 1 Test ozone debug version diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot index 9bb77d00d6d3..b042f606cf2c 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot @@ -87,15 +87,9 @@ Verify Stale Replica [arguments] ${json} ${replica} FOR ${block} IN RANGE 2 - ${n} = Evaluate ${block} + 1 ${datanode} = Set Variable ${json}[blocks][${block}][replicas][${replica}][hostname] - ${filename} = Set Variable ${DIR}/${TESTFILE}_block${n}_${datanode} IF '${datanode}' == '${STALE_DATANODE}' - File Should Be Empty ${filename} Should Contain ${json}[blocks][${block}][replicas][${replica}][exception] UNAVAILABLE - ELSE - ${filesize} = Get File Size ${filename} - Should Be Equal As Integers ${json}[blocks][${block}][length] ${filesize} END END diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java index de6aa05de73c..38f6f0103017 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java @@ -17,35 +17,26 @@ package org.apache.hadoop.ozone.debug.replicas; -import static java.util.Collections.emptyMap; - import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import jakarta.annotation.Nonnull; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Map; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.NullOutputStream; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.server.JsonUtils; -import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; -import org.apache.hadoop.ozone.client.rpc.RpcClient; -import org.apache.hadoop.ozone.common.OzoneChecksumException; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.slf4j.Logger; /** * Class that downloads every replica for all the blocks associated with a @@ -69,24 +60,16 @@ public class Checksums implements ReplicaVerifier { private String outputDir; - private RpcClient rpcClient = null; private OzoneClient client; - private Logger log; - private OzoneConfiguration ozoneConfiguration; - public Checksums(OzoneClient client, String outputDir, Logger log, OzoneConfiguration conf) { + public Checksums(OzoneClient client, String outputDir) { this.client = client; this.outputDir = outputDir; - this.log = log; - this.ozoneConfiguration = conf; } private void downloadReplicasAndCreateManifest( - String keyName, Map> replicas, - Map> - replicasWithoutChecksum, - File dir, ArrayNode blocks) throws IOException { + ArrayNode blocks) throws IOException { int blockIndex = 0; for (Map.Entry> @@ -97,16 +80,11 @@ private void downloadReplicasAndCreateManifest( blockIndex += 1; OmKeyLocationInfo locationInfo = block.getKey(); blockJson.put(JSON_PROPERTY_BLOCK_INDEX, blockIndex); - blockJson.put(JSON_PROPERTY_BLOCK_CONTAINERID, - locationInfo.getContainerID()); + blockJson.put(JSON_PROPERTY_BLOCK_CONTAINERID, locationInfo.getContainerID()); blockJson.put(JSON_PROPERTY_BLOCK_LOCALID, locationInfo.getLocalID()); blockJson.put(JSON_PROPERTY_BLOCK_LENGTH, locationInfo.getLength()); blockJson.put(JSON_PROPERTY_BLOCK_OFFSET, locationInfo.getOffset()); - BlockID blockID = locationInfo.getBlockID(); - Map blockReplicasWithoutChecksum = - replicasOf(blockID, replicasWithoutChecksum); - for (Map.Entry replica : block.getValue().entrySet()) { DatanodeDetails datanode = replica.getKey(); @@ -116,47 +94,16 @@ private void downloadReplicasAndCreateManifest( replicaJson.put(JSON_PROPERTY_REPLICA_HOSTNAME, datanode.getHostName()); replicaJson.put(JSON_PROPERTY_REPLICA_UUID, datanode.getUuidString()); - String fileName = keyName + "_block" + blockIndex + "_" + - datanode.getHostName(); - Path path = new File(dir, fileName).toPath(); - try (InputStream is = replica.getValue()) { - Files.copy(is, path, StandardCopyOption.REPLACE_EXISTING); + IOUtils.copyLarge(is, NullOutputStream.INSTANCE); } catch (IOException e) { - Throwable cause = e.getCause(); replicaJson.put(JSON_PROPERTY_REPLICA_EXCEPTION, e.getMessage()); - if (cause instanceof OzoneChecksumException) { - try (InputStream is = getReplica( - blockReplicasWithoutChecksum, datanode)) { - Files.copy(is, path, StandardCopyOption.REPLACE_EXISTING); - } - } } replicasJson.add(replicaJson); } blockJson.set(JSON_PROPERTY_BLOCK_REPLICAS, replicasJson); blocks.add(blockJson); - - IOUtils.close(log, blockReplicasWithoutChecksum.values()); - } - } - - private Map replicasOf(BlockID blockID, - Map> replicas) { - for (Map.Entry> - block : replicas.entrySet()) { - if (block.getKey().getBlockID().equals(blockID)) { - return block.getValue(); - } } - return emptyMap(); - } - - private InputStream getReplica( - Map replicas, DatanodeDetails datanode - ) { - InputStream input = replicas.remove(datanode); - return input != null ? input : new ByteArrayInputStream(new byte[0]); } @Nonnull @@ -185,21 +132,8 @@ public void verifyKey(OzoneKeyDetails keyDetails) { String bucketName = keyDetails.getBucketName(); String keyName = keyDetails.getName(); System.out.println("Processing key : " + volumeName + "/" + bucketName + "/" + keyName); - boolean isChecksumVerifyEnabled = ozoneConfiguration.getBoolean("ozone.client.verify.checksum", true); - RpcClient newClient = null; try { - OzoneConfiguration configuration = new OzoneConfiguration(ozoneConfiguration); - configuration.setBoolean("ozone.client.verify.checksum", !isChecksumVerifyEnabled); - newClient = getClient(isChecksumVerifyEnabled); - ClientProtocol noChecksumClient; - ClientProtocol checksumClient; - if (isChecksumVerifyEnabled) { - checksumClient = client.getObjectStore().getClientProxy(); - noChecksumClient = newClient; - } else { - checksumClient = newClient; - noChecksumClient = client.getObjectStore().getClientProxy(); - } + ClientProtocol checksumClient = client.getObjectStore().getClientProxy(); // Multilevel keys will have a '/' in their names. This interferes with // directory and file creation process. Flatten the keys to fix this. @@ -209,15 +143,13 @@ public void verifyKey(OzoneKeyDetails keyDetails) { OzoneKeyDetails keyInfoDetails = checksumClient.getKeyDetails(volumeName, bucketName, keyName); Map> replicas = checksumClient.getKeysEveryReplicas(volumeName, bucketName, keyName); - Map> replicasWithoutChecksum = - noChecksumClient.getKeysEveryReplicas(volumeName, bucketName, keyName); ObjectNode result = JsonUtils.createObjectNode(null); result.put(JSON_PROPERTY_FILE_NAME, volumeName + "/" + bucketName + "/" + keyName); result.put(JSON_PROPERTY_FILE_SIZE, keyInfoDetails.getDataSize()); ArrayNode blocks = JsonUtils.createArrayNode(); - downloadReplicasAndCreateManifest(sanitizedKeyName, replicas, replicasWithoutChecksum, dir, blocks); + downloadReplicasAndCreateManifest(replicas, blocks); result.set(JSON_PROPERTY_FILE_BLOCKS, blocks); String prettyJson = JsonUtils.toJsonStringWithDefaultPrettyPrinter(result); @@ -229,15 +161,4 @@ public void verifyKey(OzoneKeyDetails keyDetails) { throw new RuntimeException(e); } } - - private RpcClient getClient(boolean isChecksumVerifyEnabled) throws IOException { - if (rpcClient != null) { - return rpcClient; - } - - OzoneConfiguration configuration = new OzoneConfiguration(ozoneConfiguration); - configuration.setBoolean("ozone.client.verify.checksum", !isChecksumVerifyEnabled); - rpcClient = new RpcClient(configuration, null); - return rpcClient; - } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java index a5d474219e89..dcdaebc0d6fe 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java @@ -77,7 +77,7 @@ protected void execute(OzoneClient client, OzoneAddress address) throws IOExcept replicaVerifiers = new ArrayList<>(); if (verification.doExecuteChecksums) { - replicaVerifiers.add(new Checksums(client, outputDir, LOG, getConf())); + replicaVerifiers.add(new Checksums(client, outputDir)); } if (verification.doExecutePadding) {