From 8f59ac0a4d5e351889ce9597ce41a67b3e99af9c Mon Sep 17 00:00:00 2001 From: echonesis Date: Fri, 11 Jul 2025 09:37:19 +0800 Subject: [PATCH 01/13] HDDS-13305. Create wrapper object for container checksums --- .../AbstractContainerReportHandler.java | 2 +- .../scm/container/ContainerChecksums.java | 74 +++++++++++++++++++ .../hdds/scm/container/ContainerReplica.java | 22 +++--- .../scm/server/SCMClientProtocolServer.java | 2 +- .../scm/container/ContainerChecksumsTest.java | 55 ++++++++++++++ .../container/TestContainerReportHandler.java | 10 +-- ...TestIncrementalContainerReportHandler.java | 10 +-- .../hadoop/hdds/scm/TestCloseContainer.java | 8 +- ...groundContainerDataScannerIntegration.java | 4 +- ...ndContainerMetadataScannerIntegration.java | 10 +-- ...DemandContainerDataScannerIntegration.java | 4 +- .../recon/fsck/ContainerHealthStatus.java | 4 +- .../recon/scm/ContainerReplicaHistory.java | 21 +++--- .../recon/scm/ReconContainerManager.java | 21 +++--- .../recon/api/TestContainerEndpoint.java | 19 ++--- .../recon/fsck/TestContainerHealthStatus.java | 5 +- .../recon/fsck/TestContainerHealthTask.java | 5 +- ...estContainerHealthTaskRecordGenerator.java | 5 +- .../recon/scm/TestReconContainerManager.java | 11 ++- 19 files changed, 219 insertions(+), 73 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/ContainerChecksumsTest.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java index 7f2030f6e74b..2e7cf56888cb 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java @@ -362,7 +362,7 @@ private void updateContainerReplica(final DatanodeDetails datanodeDetails, .setReplicaIndex(replicaProto.getReplicaIndex()) .setBytesUsed(replicaProto.getUsed()) .setEmpty(replicaProto.getIsEmpty()) - .setDataChecksum(replicaProto.getDataChecksum()) + .setChecksums(new ContainerChecksums(replicaProto.getDataChecksum())) .build(); if (replica.getState().equals(State.DELETED)) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java new file mode 100644 index 000000000000..6bf50f5b73c5 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container; + +import java.util.Objects; + +/** + * Wrapper for container checksums (data, metadata, etc.). + * Provides equality, hash, and hex string rendering. + */ +public class ContainerChecksums { + private final long dataChecksum; + private final Long metadataChecksum; // nullable for future use + + public ContainerChecksums(long dataChecksum) { + this(dataChecksum, null); + } + + public ContainerChecksums(long dataChecksum, Long metadataChecksum) { + this.dataChecksum = dataChecksum; + this.metadataChecksum = metadataChecksum; + } + + public long getDataChecksum() { + return dataChecksum; + } + + public Long getMetadataChecksum() { + return metadataChecksum; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof ContainerChecksums)) { + return false; + } + ContainerChecksums that = (ContainerChecksums) obj; + return dataChecksum == that.dataChecksum && + Objects.equals(metadataChecksum, that.metadataChecksum); + } + + @Override + public int hashCode() { + return Objects.hash(dataChecksum, metadataChecksum); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("data=").append(Long.toHexString(dataChecksum)); + if (metadataChecksum != null) { + sb.append(", metadata=").append(Long.toHexString(metadataChecksum)); + } + return sb.toString(); + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java index 6cba1e8f1c31..77fe24fd14c2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java @@ -45,7 +45,7 @@ public final class ContainerReplica implements Comparable { private final long keyCount; private final long bytesUsed; private final boolean isEmpty; - private final long dataChecksum; + private final ContainerChecksums checksums; private ContainerReplica(ContainerReplicaBuilder b) { this.containerID = Objects.requireNonNull(b.containerID, "containerID == null"); @@ -57,7 +57,7 @@ private ContainerReplica(ContainerReplicaBuilder b) { this.replicaIndex = b.replicaIndex; this.isEmpty = b.isEmpty; this.sequenceId = b.sequenceId; - this.dataChecksum = b.dataChecksum; + this.checksums = Objects.requireNonNull(b.checksums, "checksums == null"); } public ContainerID getContainerID() { @@ -122,8 +122,8 @@ public boolean isEmpty() { return isEmpty; } - public long getDataChecksum() { - return dataChecksum; + public ContainerChecksums getChecksums() { + return checksums; } @Override @@ -180,7 +180,8 @@ public ContainerReplicaBuilder toBuilder() { .setOriginNodeId(originDatanodeId) .setReplicaIndex(replicaIndex) .setSequenceId(sequenceId) - .setEmpty(isEmpty); + .setEmpty(isEmpty) + .setChecksums(checksums); } @Override @@ -194,7 +195,7 @@ public String toString() { + ", keyCount=" + keyCount + ", bytesUsed=" + bytesUsed + ", " + (isEmpty ? "empty" : "non-empty") - + ", dataChecksum=" + dataChecksum + + ", checksums=" + checksums + '}'; } @@ -212,7 +213,7 @@ public static class ContainerReplicaBuilder { private long keyCount; private int replicaIndex; private boolean isEmpty; - private long dataChecksum; + private ContainerChecksums checksums; /** * Set Container Id. @@ -287,8 +288,8 @@ public ContainerReplicaBuilder setEmpty(boolean empty) { return this; } - public ContainerReplicaBuilder setDataChecksum(long dataChecksum) { - this.dataChecksum = dataChecksum; + public ContainerReplicaBuilder setChecksums(ContainerChecksums checksums) { + this.checksums = checksums; return this; } @@ -298,6 +299,9 @@ public ContainerReplicaBuilder setDataChecksum(long dataChecksum) { * @return ContainerReplicaBuilder */ public ContainerReplica build() { + if (this.checksums == null) { + this.checksums = new ContainerChecksums(0L); + } return new ContainerReplica(this); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index 03774c48699c..a261f8c253fd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -360,7 +360,7 @@ public List getContainerReplicas( .setKeyCount(r.getKeyCount()) .setSequenceID(r.getSequenceId()) .setReplicaIndex(r.getReplicaIndex()) - .setDataChecksum(r.getDataChecksum()) + .setDataChecksum(r.getChecksums().getDataChecksum()) .build() ); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/ContainerChecksumsTest.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/ContainerChecksumsTest.java new file mode 100644 index 000000000000..2e7ba3887c6f --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/ContainerChecksumsTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class ContainerChecksumsTest { + @Test + void testEqualsAndHashCode() { + ContainerChecksums c1 = new ContainerChecksums(123L); + ContainerChecksums c2 = new ContainerChecksums(123L); + ContainerChecksums c3 = new ContainerChecksums(456L); + ContainerChecksums c4 = new ContainerChecksums(123L, 789L); + ContainerChecksums c5 = new ContainerChecksums(123L, 789L); + ContainerChecksums c6 = new ContainerChecksums(123L, 790L); + + assertEquals(c1, c2); + assertEquals(c1.hashCode(), c2.hashCode()); + assertNotEquals(c1, c3); + assertNotEquals(c1, c4); + assertEquals(c4, c5); + assertNotEquals(c4, c6); + } + + @Test + void testToString() { + ContainerChecksums c1 = new ContainerChecksums(0x1234ABCDL); + assertTrue(c1.toString().contains("data=1234abcd")); + assertFalse(c1.toString().contains("metadata=")); + + ContainerChecksums c2 = new ContainerChecksums(0x1234ABCDL, 0xDEADBEEFL); + assertTrue(c2.toString().contains("data=1234abcd")); + assertTrue(c2.toString().contains("metadata=deadbeef")); + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java index a3db471334de..0af693649fe8 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java @@ -1205,7 +1205,7 @@ public void testWithNoContainerDataChecksum() throws Exception { // All replicas should start with an empty data checksum in SCM. boolean contOneDataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getDataChecksum() == 0); + .allMatch(r -> r.getChecksums().getDataChecksum() == 0); assertTrue(contOneDataChecksumsEmpty, "Replicas of container one should not yet have any data checksums."); // Send a report to SCM from one datanode that still does not have a data checksum. @@ -1222,7 +1222,7 @@ public void testWithNoContainerDataChecksum() throws Exception { // Regardless of which datanode sent the report, none of them have checksums, so all replica's data checksums // should remain empty. boolean containerDataChecksumEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getDataChecksum() == 0); + .allMatch(r -> r.getChecksums().getDataChecksum() == 0); assertTrue(containerDataChecksumEmpty, "Replicas of the container should not have any data checksums."); } @@ -1254,7 +1254,7 @@ public void testWithContainerDataChecksum() throws Exception { // All replicas should start with an empty data checksum in SCM. boolean dataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getDataChecksum() == 0); + .allMatch(r -> r.getChecksums().getDataChecksum() == 0); assertTrue(dataChecksumsEmpty, "Replicas of container one should not yet have any data checksums."); // For each datanode, send a container report with a mismatched checksum. @@ -1278,7 +1278,7 @@ public void testWithContainerDataChecksum() throws Exception { int numReplicasChecked = 0; for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) { long expectedChecksum = createUniqueDataChecksumForReplica(contID, replica.getDatanodeDetails().getUuidString()); - assertEquals(expectedChecksum, replica.getDataChecksum()); + assertEquals(expectedChecksum, replica.getChecksums().getDataChecksum()); numReplicasChecked++; } assertEquals(numNodes, numReplicasChecked); @@ -1304,7 +1304,7 @@ public void testWithContainerDataChecksum() throws Exception { numReplicasChecked = 0; for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) { long expectedChecksum = createMatchingDataChecksumForReplica(contID); - assertEquals(expectedChecksum, replica.getDataChecksum()); + assertEquals(expectedChecksum, replica.getChecksums().getDataChecksum()); numReplicasChecked++; } assertEquals(numNodes, numReplicasChecked); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java index 41fbe2032a7b..1306d1bcbfae 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java @@ -648,7 +648,7 @@ public void testWithNoContainerDataChecksum() throws Exception { // All replicas should start with an empty data checksum in SCM. boolean contOneDataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getDataChecksum() == 0); + .allMatch(r -> r.getChecksums().getDataChecksum() == 0); assertTrue(contOneDataChecksumsEmpty, "Replicas of container one should not yet have any data checksums."); // Send a report to SCM from one datanode that still does not have a data checksum. @@ -663,7 +663,7 @@ public void testWithNoContainerDataChecksum() throws Exception { // Regardless of which datanode sent the report, none of them have checksums, so all replica's data checksums // should remain empty. boolean containerDataChecksumEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getDataChecksum() == 0); + .allMatch(r -> r.getChecksums().getDataChecksum() == 0); assertTrue(containerDataChecksumEmpty, "Replicas of the container should not have any data checksums."); } @@ -695,7 +695,7 @@ public void testWithContainerDataChecksum() throws Exception { // All replicas should start with a zero data checksum in SCM. boolean dataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getDataChecksum() == 0); + .allMatch(r -> r.getChecksums().getDataChecksum() == 0); assertTrue(dataChecksumsEmpty, "Replicas of container one should not yet have any data checksums."); // For each datanode, send a container report with a mismatched checksum. @@ -721,7 +721,7 @@ public void testWithContainerDataChecksum() throws Exception { for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) { long expectedChecksum = createUniqueDataChecksumForReplica( contID, replica.getDatanodeDetails().getUuidString()); - assertEquals(expectedChecksum, replica.getDataChecksum()); + assertEquals(expectedChecksum, replica.getChecksums().getDataChecksum()); numReplicasChecked++; } assertEquals(numNodes, numReplicasChecked); @@ -748,7 +748,7 @@ public void testWithContainerDataChecksum() throws Exception { numReplicasChecked = 0; for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) { long expectedChecksum = createMatchingDataChecksumForReplica(contID); - assertEquals(expectedChecksum, replica.getDataChecksum()); + assertEquals(expectedChecksum, replica.getChecksums().getDataChecksum()); numReplicasChecked++; } assertEquals(numNodes, numReplicasChecked); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java index 99cbde901c75..40440f13ac7c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java @@ -159,7 +159,7 @@ public void testReplicasAreReportedForClosedContainerAfterRestart() GenericTestUtils.waitFor(() -> getContainerReplicas(newContainer).size() == 3, 200, 30000); for (ContainerReplica replica : getContainerReplicas(newContainer)) { - assertNotEquals(0, replica.getDataChecksum()); + assertNotEquals(0, replica.getChecksums().getDataChecksum()); } } @@ -201,7 +201,7 @@ public void testCloseClosedContainer() } for (ContainerReplica replica : getContainerReplicas(container)) { - assertNotEquals(0, replica.getDataChecksum()); + assertNotEquals(0, replica.getChecksums().getDataChecksum()); } assertThrows(IOException.class, @@ -275,10 +275,10 @@ public void testContainerChecksumForClosedContainer() throws Exception { assertNotEquals(prevExpectedChecksumInfo1.getContainerMerkleTree().getDataChecksum(), prevExpectedChecksumInfo2.getContainerMerkleTree().getDataChecksum()); for (ContainerReplica replica : getContainerReplicas(containerInfo1)) { - assertNotEquals(0, replica.getDataChecksum()); + assertNotEquals(0, replica.getChecksums().getDataChecksum()); } for (ContainerReplica replica : getContainerReplicas(containerInfo2)) { - assertNotEquals(0, replica.getDataChecksum()); + assertNotEquals(0, replica.getChecksums().getDataChecksum()); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java index 2306af221c43..68dac0baab01 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java @@ -85,7 +85,7 @@ void testCorruptionDetected(TestContainerCorruptions corruption) assertTrue(containerChecksumFileExists(containerID)); waitForScmToSeeReplicaState(containerID, CLOSED); - long initialReportedDataChecksum = getContainerReplica(containerID).getDataChecksum(); + long initialReportedDataChecksum = getContainerReplica(containerID).getChecksums().getDataChecksum(); corruption.applyTo(container); @@ -98,7 +98,7 @@ void testCorruptionDetected(TestContainerCorruptions corruption) // Wait for SCM to get a report of the unhealthy replica with a different checksum than before. waitForScmToSeeReplicaState(containerID, UNHEALTHY); - long newReportedDataChecksum = getContainerReplica(containerID).getDataChecksum(); + long newReportedDataChecksum = getContainerReplica(containerID).getChecksums().getDataChecksum(); if (corruption == TestContainerCorruptions.MISSING_METADATA_DIR || corruption == TestContainerCorruptions.MISSING_CONTAINER_DIR) { // In these cases, the new tree will not be able to be written since it exists in the metadata directory. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java index b25df7e11369..bc324dad9522 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java @@ -98,7 +98,7 @@ void testCorruptionDetected(TestContainerCorruptions corruption) assertEquals(State.CLOSED, closedContainer.getContainerState()); assertTrue(containerChecksumFileExists(closedContainerID)); waitForScmToSeeReplicaState(closedContainerID, CLOSED); - long initialClosedChecksum = getContainerReplica(closedContainerID).getDataChecksum(); + long initialClosedChecksum = getContainerReplica(closedContainerID).getChecksums().getDataChecksum(); assertNotEquals(0, initialClosedChecksum); long openContainerID = writeDataToOpenContainer(); @@ -106,7 +106,7 @@ void testCorruptionDetected(TestContainerCorruptions corruption) assertEquals(State.OPEN, openContainer.getContainerState()); waitForScmToSeeReplicaState(openContainerID, OPEN); // Open containers should not yet have a checksum generated. - assertEquals(0, getContainerReplica(openContainerID).getDataChecksum()); + assertEquals(0, getContainerReplica(openContainerID).getChecksums().getDataChecksum()); // Corrupt both containers. corruption.applyTo(closedContainer); @@ -124,17 +124,17 @@ void testCorruptionDetected(TestContainerCorruptions corruption) // The metadata scanner does not generate data checksums and the other scanners have been turned off for this // test, so the data checksums should not change. waitForScmToSeeReplicaState(closedContainerID, UNHEALTHY); - assertEquals(initialClosedChecksum, getContainerReplica(closedContainerID).getDataChecksum()); + assertEquals(initialClosedChecksum, getContainerReplica(closedContainerID).getChecksums().getDataChecksum()); waitForScmToSeeReplicaState(openContainerID, UNHEALTHY); if (corruption == MISSING_METADATA_DIR || corruption == MISSING_CONTAINER_DIR) { // In these cases the tree cannot be generated when the container is marked unhealthy and the checksum should // remain at 0. // The tree is generated from metadata by the container changing to unhealthy, not by the metadata scanner. - assertEquals(0, getContainerReplica(openContainerID).getDataChecksum()); + assertEquals(0, getContainerReplica(openContainerID).getChecksums().getDataChecksum()); } else { // The checksum will be generated for the first time when the container is marked unhealthy. // The tree is generated from metadata by the container changing to unhealthy, not by the metadata scanner. - assertNotEquals(0, getContainerReplica(openContainerID).getDataChecksum()); + assertNotEquals(0, getContainerReplica(openContainerID).getChecksums().getDataChecksum()); } // Once the unhealthy replica is reported, the open container's lifecycle diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java index d3b3ed46fdeb..ea0f93168335 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java @@ -105,7 +105,7 @@ void testCorruptionDetected(TestContainerCorruptions corruption) assertTrue(containerChecksumFileExists(containerID)); waitForScmToSeeReplicaState(containerID, CLOSED); - long initialReportedDataChecksum = getContainerReplica(containerID).getDataChecksum(); + long initialReportedDataChecksum = getContainerReplica(containerID).getChecksums().getDataChecksum(); // Corrupt the container. corruption.applyTo(container); @@ -121,7 +121,7 @@ void testCorruptionDetected(TestContainerCorruptions corruption) // Wait for SCM to get a report of the unhealthy replica. waitForScmToSeeReplicaState(containerID, UNHEALTHY); corruption.assertLogged(containerID, 1, logCapturer); - long newReportedDataChecksum = getContainerReplica(containerID).getDataChecksum(); + long newReportedDataChecksum = getContainerReplica(containerID).getChecksums().getDataChecksum(); if (corruption == TestContainerCorruptions.MISSING_METADATA_DIR || corruption == TestContainerCorruptions.MISSING_CONTAINER_DIR) { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java index 4c28806dfa8a..8e6bcb1f0784 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.ContainerPlacementStatus; import org.apache.hadoop.hdds.scm.PlacementPolicy; +import org.apache.hadoop.hdds.scm.container.ContainerChecksums; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaCount; @@ -162,7 +163,8 @@ public boolean isEmpty() { public boolean isDataChecksumMismatched() { return !replicas.isEmpty() && replicas.stream() - .map(ContainerReplica::getDataChecksum) + .map(ContainerReplica::getChecksums) + .map(ContainerChecksums::getDataChecksum) .distinct() .count() != 1; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java index d47c7010a2aa..a9e3077b3de3 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java @@ -19,6 +19,7 @@ import java.util.UUID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerReplicaHistoryProto; +import org.apache.hadoop.hdds.scm.container.ContainerChecksums; /** * A ContainerReplica timestamp class that tracks first and last seen time. @@ -39,16 +40,16 @@ public class ContainerReplicaHistory { private long bcsId; private String state; - private long dataChecksum; + private ContainerChecksums checksums; public ContainerReplicaHistory(UUID id, Long firstSeenTime, - Long lastSeenTime, long bcsId, String state, long dataChecksum) { + Long lastSeenTime, long bcsId, String state, ContainerChecksums checksums) { this.uuid = id; this.firstSeenTime = firstSeenTime; this.lastSeenTime = lastSeenTime; this.bcsId = bcsId; this.state = state; - this.dataChecksum = dataChecksum; + this.checksums = checksums; } public long getBcsId() { @@ -83,24 +84,26 @@ public void setState(String state) { this.state = state; } - public long getDataChecksum() { - return dataChecksum; + public ContainerChecksums getChecksums() { + return checksums; } - public void setDataChecksum(long dataChecksum) { - this.dataChecksum = dataChecksum; + public void setChecksums(ContainerChecksums checksums) { + this.checksums = checksums; } public static ContainerReplicaHistory fromProto( ContainerReplicaHistoryProto proto) { return new ContainerReplicaHistory(UUID.fromString(proto.getUuid()), proto.getFirstSeenTime(), proto.getLastSeenTime(), proto.getBcsId(), - proto.getState(), proto.getDataChecksum()); + proto.getState(), new ContainerChecksums(proto.getDataChecksum())); } public ContainerReplicaHistoryProto toProto() { return ContainerReplicaHistoryProto.newBuilder().setUuid(uuid.toString()) .setFirstSeenTime(firstSeenTime).setLastSeenTime(lastSeenTime) - .setBcsId(bcsId).setState(state).setDataChecksum(dataChecksum).build(); + .setBcsId(bcsId).setState(state) + .setDataChecksum(checksums != null ? checksums.getDataChecksum() : 0L) + .build(); } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java index 59436cb72b2a..e21add7e84fd 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; +import org.apache.hadoop.hdds.scm.container.ContainerChecksums; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerManagerImpl; @@ -278,7 +279,7 @@ public void updateContainerReplica(ContainerID containerID, boolean flushToDB = false; long bcsId = replica.getSequenceId() != null ? replica.getSequenceId() : -1; String state = replica.getState().toString(); - long dataChecksum = replica.getDataChecksum(); + ContainerChecksums checksums = replica.getChecksums(); // If replica doesn't exist in in-memory map, add to DB and add to map if (replicaLastSeenMap == null) { @@ -286,7 +287,7 @@ public void updateContainerReplica(ContainerID containerID, replicaHistoryMap.putIfAbsent(id, new ConcurrentHashMap() {{ put(uuid, new ContainerReplicaHistory(uuid, currTime, currTime, - bcsId, state, dataChecksum)); + bcsId, state, checksums)); }}); flushToDB = true; } else { @@ -296,18 +297,19 @@ public void updateContainerReplica(ContainerID containerID, // New Datanode replicaLastSeenMap.put(uuid, new ContainerReplicaHistory(uuid, currTime, currTime, bcsId, - state, dataChecksum)); + state, checksums)); flushToDB = true; } else { // if the object exists, only update the last seen time & bcsId fields ts.setLastSeenTime(currTime); ts.setBcsId(bcsId); ts.setState(state); + ts.setChecksums(checksums); } } if (flushToDB) { - upsertContainerHistory(id, uuid, currTime, bcsId, state, dataChecksum); + upsertContainerHistory(id, uuid, currTime, bcsId, state, checksums); } } @@ -324,7 +326,6 @@ public void removeContainerReplica(ContainerID containerID, final DatanodeDetails dnInfo = replica.getDatanodeDetails(); final UUID uuid = dnInfo.getUuid(); String state = replica.getState().toString(); - long dataChecksum = replica.getDataChecksum(); final Map replicaLastSeenMap = replicaHistoryMap.get(id); @@ -333,7 +334,7 @@ public void removeContainerReplica(ContainerID containerID, if (ts != null) { // Flush to DB, then remove from in-memory map upsertContainerHistory(id, uuid, ts.getLastSeenTime(), ts.getBcsId(), - state, dataChecksum); + state, ts.getChecksums()); replicaLastSeenMap.remove(uuid); } } @@ -392,7 +393,7 @@ public List getAllContainerHistory(long containerID) { final long lastSeenTime = entry.getValue().getLastSeenTime(); long bcsId = entry.getValue().getBcsId(); String state = entry.getValue().getState(); - long dataChecksum = entry.getValue().getDataChecksum(); + long dataChecksum = entry.getValue().getChecksums().getDataChecksum(); resList.add(new ContainerHistory(containerID, uuid.toString(), hostname, firstSeenTime, lastSeenTime, bcsId, state, dataChecksum)); @@ -430,7 +431,7 @@ public void flushReplicaHistoryMapToDB(boolean clearMap) { } public void upsertContainerHistory(long containerID, UUID uuid, long time, - long bcsId, String state, long dataChecksum) { + long bcsId, String state, ContainerChecksums checksums) { Map tsMap; try { tsMap = cdbServiceProvider.getContainerReplicaHistory(containerID); @@ -438,12 +439,12 @@ public void upsertContainerHistory(long containerID, UUID uuid, long time, if (ts == null) { // New entry tsMap.put(uuid, new ContainerReplicaHistory(uuid, time, time, bcsId, - state, dataChecksum)); + state, checksums)); } else { // Entry exists, update last seen time and put it back to DB. ts.setLastSeenTime(time); ts.setState(state); - ts.setDataChecksum(dataChecksum); + ts.setChecksums(checksums); } cdbServiceProvider.storeContainerReplicaHistory(containerID, tsMap); } catch (IOException e) { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java index 72376c7f923a..a40f3b5d3022 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java @@ -60,6 +60,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.scm.container.ContainerChecksums; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerStateManager; @@ -1035,12 +1036,12 @@ public void testGetReplicaHistoryForContainer() throws IOException { final UUID u2 = newDatanode("host2", "127.0.0.2"); final UUID u3 = newDatanode("host3", "127.0.0.3"); final UUID u4 = newDatanode("host4", "127.0.0.4"); - reconContainerManager.upsertContainerHistory(1L, u1, 1L, 1L, "OPEN", 1234L); - reconContainerManager.upsertContainerHistory(1L, u2, 2L, 1L, "OPEN", 1234L); - reconContainerManager.upsertContainerHistory(1L, u3, 3L, 1L, "OPEN", 1234L); - reconContainerManager.upsertContainerHistory(1L, u4, 4L, 1L, "OPEN", 1234L); + reconContainerManager.upsertContainerHistory(1L, u1, 1L, 1L, "OPEN", new ContainerChecksums(1234L)); + reconContainerManager.upsertContainerHistory(1L, u2, 2L, 1L, "OPEN", new ContainerChecksums(1234L)); + reconContainerManager.upsertContainerHistory(1L, u3, 3L, 1L, "OPEN", new ContainerChecksums(1234L)); + reconContainerManager.upsertContainerHistory(1L, u4, 4L, 1L, "OPEN", new ContainerChecksums(1234L)); - reconContainerManager.upsertContainerHistory(1L, u1, 5L, 1L, "OPEN", 1234L); + reconContainerManager.upsertContainerHistory(1L, u1, 5L, 1L, "OPEN", new ContainerChecksums(1234L)); Response response = containerEndpoint.getReplicaHistoryForContainer(1L); List histories = @@ -1150,13 +1151,13 @@ private void createUnhealthyRecord(int id, String state, int expected, long differentChecksum = dataChecksumMismatch ? 2345L : 1234L; reconContainerManager.upsertContainerHistory(cID, uuid1, 1L, 1L, - "UNHEALTHY", differentChecksum); + "UNHEALTHY", new ContainerChecksums(differentChecksum)); reconContainerManager.upsertContainerHistory(cID, uuid2, 2L, 1L, - "UNHEALTHY", differentChecksum); + "UNHEALTHY", new ContainerChecksums(differentChecksum)); reconContainerManager.upsertContainerHistory(cID, uuid3, 3L, 1L, - "UNHEALTHY", 1234L); + "UNHEALTHY", new ContainerChecksums(1234L)); reconContainerManager.upsertContainerHistory(cID, uuid4, 4L, 1L, - "UNHEALTHY", 1234L); + "UNHEALTHY", new ContainerChecksums(1234L)); } protected ContainerWithPipeline getTestContainer( diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java index dc488ea303ca..30adb7e89acb 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.PlacementPolicy; +import org.apache.hadoop.hdds.scm.container.ContainerChecksums; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; @@ -416,7 +417,7 @@ private Set generateReplicas(ContainerInfo cont, replicas.add(new ContainerReplica.ContainerReplicaBuilder() .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) - .setDataChecksum(1234L) + .setChecksums(new ContainerChecksums(1234L)) .setContainerState(s) .build()); } @@ -432,7 +433,7 @@ private Set generateMismatchedReplicas(ContainerInfo cont, .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) .setContainerState(s) - .setDataChecksum(checksum) + .setChecksums(new ContainerChecksums(checksum)) .build()); checksum++; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java index fc659b49fdef..fd7795e66ad7 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; import org.apache.hadoop.hdds.scm.ContainerPlacementStatus; import org.apache.hadoop.hdds.scm.PlacementPolicy; +import org.apache.hadoop.hdds.scm.container.ContainerChecksums; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerManager; @@ -690,7 +691,7 @@ private Set getMockReplicas( .setContainerState(s) .setContainerID(ContainerID.valueOf(containerId)) .setSequenceId(1) - .setDataChecksum(1234L) + .setChecksums(new ContainerChecksums(1234L)) .build()); } return replicas; @@ -706,7 +707,7 @@ private Set getMockReplicasChecksumMismatch( .setContainerState(s) .setContainerID(ContainerID.valueOf(containerId)) .setSequenceId(1) - .setDataChecksum(checksum) + .setChecksums(new ContainerChecksums(checksum)) .build()); checksum++; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java index d020a548fc8c..5bd2783ec1de 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.PlacementPolicy; +import org.apache.hadoop.hdds.scm.container.ContainerChecksums; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; @@ -642,8 +643,8 @@ private Set generateReplicas(ContainerInfo cont, replicas.add(new ContainerReplica.ContainerReplicaBuilder() .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) + .setChecksums(new ContainerChecksums(1234L)) .setContainerState(s) - .setDataChecksum(1234L) .build()); } return replicas; @@ -658,7 +659,7 @@ private Set generateMismatchedReplicas(ContainerInfo cont, .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) .setContainerState(s) - .setDataChecksum(checksum) + .setChecksums(new ContainerChecksums(checksum)) .build()); checksum++; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java index 81c9166b13a2..b7bcdd8329bf 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State; +import org.apache.hadoop.hdds.scm.container.ContainerChecksums; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; @@ -209,7 +210,8 @@ public void testUpdateAndRemoveContainerReplica() .setUuid(uuid1).setHostName("host1").setIpAddress("127.0.0.1").build(); ContainerReplica containerReplica1 = ContainerReplica.newBuilder() .setContainerID(containerID1).setContainerState(State.OPEN) - .setDatanodeDetails(datanodeDetails1).setSequenceId(1001L).setDataChecksum(1234L).build(); + .setDatanodeDetails(datanodeDetails1).setSequenceId(1001L) + .setChecksums(new ContainerChecksums(1234L)).build(); final ReconContainerManager containerManager = getContainerManager(); final Map> repHistMap = @@ -237,7 +239,7 @@ public void testUpdateAndRemoveContainerReplica() assertEquals(repHist1.getLastSeenTime(), repHist1.getFirstSeenTime()); assertEquals(containerReplica1.getSequenceId().longValue(), repHist1.getBcsId()); - assertEquals(containerReplica1.getDataChecksum(), repHist1.getDataChecksum()); + assertEquals(containerReplica1.getChecksums().getDataChecksum(), repHist1.getChecksums().getDataChecksum()); // Let's update the entry again containerReplica1 = ContainerReplica.newBuilder() @@ -256,7 +258,8 @@ public void testUpdateAndRemoveContainerReplica() .setUuid(uuid2).setHostName("host2").setIpAddress("127.0.0.2").build(); final ContainerReplica containerReplica2 = ContainerReplica.newBuilder() .setContainerID(containerID1).setContainerState(State.OPEN) - .setDatanodeDetails(datanodeDetails2).setSequenceId(1051L).setDataChecksum(1234L).build(); + .setDatanodeDetails(datanodeDetails2).setSequenceId(1051L) + .setChecksums(new ContainerChecksums(1234L)).build(); // Add replica to DN02 containerManager.updateContainerReplica(containerID1, containerReplica2); @@ -270,7 +273,7 @@ public void testUpdateAndRemoveContainerReplica() // Because this is a new entry, first seen time equals last seen time assertEquals(repHist2.getLastSeenTime(), repHist2.getFirstSeenTime()); assertEquals(1051L, repHist2.getBcsId()); - assertEquals(containerReplica2.getDataChecksum(), repHist2.getDataChecksum()); + assertEquals(containerReplica2.getChecksums().getDataChecksum(), repHist2.getChecksums().getDataChecksum()); // Remove replica from DN01 containerManager.removeContainerReplica(containerID1, containerReplica1); From 1299510c2d70618d03cba73bef8a1924a2091820 Mon Sep 17 00:00:00 2001 From: echonesis Date: Wed, 23 Jul 2025 11:18:57 +0800 Subject: [PATCH 02/13] PR review modification update --- hadoop-hdds/server-scm/pom.xml | 4 +++ .../AbstractContainerReportHandler.java | 2 +- .../scm/container/ContainerChecksums.java | 35 ++++++++++++++----- .../hdds/scm/container/ContainerReplica.java | 2 +- ...sTest.java => TestContainerChecksums.java} | 27 +++++++------- .../recon/scm/ContainerReplicaHistory.java | 2 +- .../recon/api/TestContainerEndpoint.java | 18 +++++----- .../recon/fsck/TestContainerHealthStatus.java | 4 +-- .../recon/fsck/TestContainerHealthTask.java | 4 +-- ...estContainerHealthTaskRecordGenerator.java | 4 +-- .../recon/scm/TestReconContainerManager.java | 4 +-- 11 files changed, 63 insertions(+), 43 deletions(-) rename hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/{ContainerChecksumsTest.java => TestContainerChecksums.java} (59%) diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml index 80a48482773f..6a0525e517b6 100644 --- a/hadoop-hdds/server-scm/pom.xml +++ b/hadoop-hdds/server-scm/pom.xml @@ -43,6 +43,10 @@ jackson-databind + + com.github.stephenc.jcip + jcip-annotations + com.google.guava guava diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java index 2e7cf56888cb..3020b1dfeaa4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java @@ -362,7 +362,7 @@ private void updateContainerReplica(final DatanodeDetails datanodeDetails, .setReplicaIndex(replicaProto.getReplicaIndex()) .setBytesUsed(replicaProto.getUsed()) .setEmpty(replicaProto.getIsEmpty()) - .setChecksums(new ContainerChecksums(replicaProto.getDataChecksum())) + .setChecksums(ContainerChecksums.dataOnly(replicaProto.getDataChecksum())) .build(); if (replica.getState().equals(State.DELETED)) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java index 6bf50f5b73c5..80567ed3104e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java @@ -19,23 +19,42 @@ import java.util.Objects; +import net.jcip.annotations.Immutable; + /** * Wrapper for container checksums (data, metadata, etc.). * Provides equality, hash, and hex string rendering. */ -public class ContainerChecksums { +@Immutable +public final class ContainerChecksums { + + private static final ContainerChecksums UNKNOWN = + new ContainerChecksums(0, null); + private final long dataChecksum; private final Long metadataChecksum; // nullable for future use - public ContainerChecksums(long dataChecksum) { - this(dataChecksum, null); - } - - public ContainerChecksums(long dataChecksum, Long metadataChecksum) { + private ContainerChecksums(long dataChecksum, Long metadataChecksum) { this.dataChecksum = dataChecksum; this.metadataChecksum = metadataChecksum; } + public static ContainerChecksums unknown() { + return UNKNOWN; + } + + public static ContainerChecksums dataOnly(long dataChecksum) { + return new ContainerChecksums(dataChecksum, null); + } + + public static ContainerChecksums metadataOnly(long metadataChecksum) { + return new ContainerChecksums(0, metadataChecksum); + } + + public static ContainerChecksums of(long dataChecksum, long metadataChecksum) { + return new ContainerChecksums(dataChecksum, metadataChecksum); + } + public long getDataChecksum() { return dataChecksum; } @@ -54,7 +73,7 @@ public boolean equals(Object obj) { } ContainerChecksums that = (ContainerChecksums) obj; return dataChecksum == that.dataChecksum && - Objects.equals(metadataChecksum, that.metadataChecksum); + Objects.equals(metadataChecksum, that.metadataChecksum); } @Override @@ -71,4 +90,4 @@ public String toString() { } return sb.toString(); } -} +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java index 77fe24fd14c2..6968aaea8837 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java @@ -300,7 +300,7 @@ public ContainerReplicaBuilder setChecksums(ContainerChecksums checksums) { */ public ContainerReplica build() { if (this.checksums == null) { - this.checksums = new ContainerChecksums(0L); + this.checksums = ContainerChecksums.unknown(); } return new ContainerReplica(this); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/ContainerChecksumsTest.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java similarity index 59% rename from hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/ContainerChecksumsTest.java rename to hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java index 2e7ba3887c6f..4db5248a3538 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/ContainerChecksumsTest.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java @@ -17,22 +17,21 @@ package org.apache.hadoop.hdds.scm.container; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; -class ContainerChecksumsTest { +class TestContainerChecksums { @Test void testEqualsAndHashCode() { - ContainerChecksums c1 = new ContainerChecksums(123L); - ContainerChecksums c2 = new ContainerChecksums(123L); - ContainerChecksums c3 = new ContainerChecksums(456L); - ContainerChecksums c4 = new ContainerChecksums(123L, 789L); - ContainerChecksums c5 = new ContainerChecksums(123L, 789L); - ContainerChecksums c6 = new ContainerChecksums(123L, 790L); + ContainerChecksums c1 = ContainerChecksums.dataOnly(123L); + ContainerChecksums c2 = ContainerChecksums.dataOnly(123L); + ContainerChecksums c3 = ContainerChecksums.dataOnly(456L); + ContainerChecksums c4 = ContainerChecksums.of(123L, 789L); + ContainerChecksums c5 = ContainerChecksums.of(123L, 789L); + ContainerChecksums c6 = ContainerChecksums.of(123L, 790L); assertEquals(c1, c2); assertEquals(c1.hashCode(), c2.hashCode()); @@ -44,12 +43,10 @@ void testEqualsAndHashCode() { @Test void testToString() { - ContainerChecksums c1 = new ContainerChecksums(0x1234ABCDL); - assertTrue(c1.toString().contains("data=1234abcd")); - assertFalse(c1.toString().contains("metadata=")); + ContainerChecksums c1 = ContainerChecksums.dataOnly(0x1234ABCDL); + assertThat(c1.toString()).contains("data=1234abcd").doesNotContain("metadata="); - ContainerChecksums c2 = new ContainerChecksums(0x1234ABCDL, 0xDEADBEEFL); - assertTrue(c2.toString().contains("data=1234abcd")); - assertTrue(c2.toString().contains("metadata=deadbeef")); + ContainerChecksums c2 = ContainerChecksums.of(0x1234ABCDL, 0xDEADBEEFL); + assertThat(c2.toString()).contains("data=1234abcd").contains("metadata=deadbeef"); } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java index a9e3077b3de3..ba0b0eda7dd2 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java @@ -96,7 +96,7 @@ public static ContainerReplicaHistory fromProto( ContainerReplicaHistoryProto proto) { return new ContainerReplicaHistory(UUID.fromString(proto.getUuid()), proto.getFirstSeenTime(), proto.getLastSeenTime(), proto.getBcsId(), - proto.getState(), new ContainerChecksums(proto.getDataChecksum())); + proto.getState(), ContainerChecksums.dataOnly(proto.getDataChecksum())); } public ContainerReplicaHistoryProto toProto() { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java index a40f3b5d3022..dd5020dc98f5 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java @@ -1036,12 +1036,12 @@ public void testGetReplicaHistoryForContainer() throws IOException { final UUID u2 = newDatanode("host2", "127.0.0.2"); final UUID u3 = newDatanode("host3", "127.0.0.3"); final UUID u4 = newDatanode("host4", "127.0.0.4"); - reconContainerManager.upsertContainerHistory(1L, u1, 1L, 1L, "OPEN", new ContainerChecksums(1234L)); - reconContainerManager.upsertContainerHistory(1L, u2, 2L, 1L, "OPEN", new ContainerChecksums(1234L)); - reconContainerManager.upsertContainerHistory(1L, u3, 3L, 1L, "OPEN", new ContainerChecksums(1234L)); - reconContainerManager.upsertContainerHistory(1L, u4, 4L, 1L, "OPEN", new ContainerChecksums(1234L)); + reconContainerManager.upsertContainerHistory(1L, u1, 1L, 1L, "OPEN", ContainerChecksums.dataOnly(1234L)); + reconContainerManager.upsertContainerHistory(1L, u2, 2L, 1L, "OPEN", ContainerChecksums.dataOnly(1234L)); + reconContainerManager.upsertContainerHistory(1L, u3, 3L, 1L, "OPEN", ContainerChecksums.dataOnly(1234L)); + reconContainerManager.upsertContainerHistory(1L, u4, 4L, 1L, "OPEN", ContainerChecksums.dataOnly(1234L)); - reconContainerManager.upsertContainerHistory(1L, u1, 5L, 1L, "OPEN", new ContainerChecksums(1234L)); + reconContainerManager.upsertContainerHistory(1L, u1, 5L, 1L, "OPEN", ContainerChecksums.dataOnly(1234L)); Response response = containerEndpoint.getReplicaHistoryForContainer(1L); List histories = @@ -1151,13 +1151,13 @@ private void createUnhealthyRecord(int id, String state, int expected, long differentChecksum = dataChecksumMismatch ? 2345L : 1234L; reconContainerManager.upsertContainerHistory(cID, uuid1, 1L, 1L, - "UNHEALTHY", new ContainerChecksums(differentChecksum)); + "UNHEALTHY", ContainerChecksums.dataOnly(differentChecksum)); reconContainerManager.upsertContainerHistory(cID, uuid2, 2L, 1L, - "UNHEALTHY", new ContainerChecksums(differentChecksum)); + "UNHEALTHY", ContainerChecksums.dataOnly(differentChecksum)); reconContainerManager.upsertContainerHistory(cID, uuid3, 3L, 1L, - "UNHEALTHY", new ContainerChecksums(1234L)); + "UNHEALTHY", ContainerChecksums.dataOnly(1234L)); reconContainerManager.upsertContainerHistory(cID, uuid4, 4L, 1L, - "UNHEALTHY", new ContainerChecksums(1234L)); + "UNHEALTHY", ContainerChecksums.dataOnly(1234L)); } protected ContainerWithPipeline getTestContainer( diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java index 30adb7e89acb..b9dd49095d35 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java @@ -417,7 +417,7 @@ private Set generateReplicas(ContainerInfo cont, replicas.add(new ContainerReplica.ContainerReplicaBuilder() .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) - .setChecksums(new ContainerChecksums(1234L)) + .setChecksums(ContainerChecksums.dataOnly(1234L)) .setContainerState(s) .build()); } @@ -433,7 +433,7 @@ private Set generateMismatchedReplicas(ContainerInfo cont, .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) .setContainerState(s) - .setChecksums(new ContainerChecksums(checksum)) + .setChecksums(ContainerChecksums.dataOnly(checksum)) .build()); checksum++; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java index fd7795e66ad7..f01d690f0df6 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java @@ -691,7 +691,7 @@ private Set getMockReplicas( .setContainerState(s) .setContainerID(ContainerID.valueOf(containerId)) .setSequenceId(1) - .setChecksums(new ContainerChecksums(1234L)) + .setChecksums(ContainerChecksums.dataOnly(1234L)) .build()); } return replicas; @@ -707,7 +707,7 @@ private Set getMockReplicasChecksumMismatch( .setContainerState(s) .setContainerID(ContainerID.valueOf(containerId)) .setSequenceId(1) - .setChecksums(new ContainerChecksums(checksum)) + .setChecksums(ContainerChecksums.dataOnly(checksum)) .build()); checksum++; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java index 5bd2783ec1de..c7dc3bdcba73 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java @@ -643,7 +643,7 @@ private Set generateReplicas(ContainerInfo cont, replicas.add(new ContainerReplica.ContainerReplicaBuilder() .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) - .setChecksums(new ContainerChecksums(1234L)) + .setChecksums(ContainerChecksums.dataOnly(1234L)) .setContainerState(s) .build()); } @@ -659,7 +659,7 @@ private Set generateMismatchedReplicas(ContainerInfo cont, .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) .setContainerState(s) - .setChecksums(new ContainerChecksums(checksum)) + .setChecksums(ContainerChecksums.dataOnly(checksum)) .build()); checksum++; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java index b7bcdd8329bf..ba9df7e3cd0f 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java @@ -211,7 +211,7 @@ public void testUpdateAndRemoveContainerReplica() ContainerReplica containerReplica1 = ContainerReplica.newBuilder() .setContainerID(containerID1).setContainerState(State.OPEN) .setDatanodeDetails(datanodeDetails1).setSequenceId(1001L) - .setChecksums(new ContainerChecksums(1234L)).build(); + .setChecksums(ContainerChecksums.dataOnly(1234L)).build(); final ReconContainerManager containerManager = getContainerManager(); final Map> repHistMap = @@ -259,7 +259,7 @@ public void testUpdateAndRemoveContainerReplica() final ContainerReplica containerReplica2 = ContainerReplica.newBuilder() .setContainerID(containerID1).setContainerState(State.OPEN) .setDatanodeDetails(datanodeDetails2).setSequenceId(1051L) - .setChecksums(new ContainerChecksums(1234L)).build(); + .setChecksums(ContainerChecksums.dataOnly(1234L)).build(); // Add replica to DN02 containerManager.updateContainerReplica(containerID1, containerReplica2); From f0e922f63d52a35ee6e33cfb3be9b9bb8a00ade5 Mon Sep 17 00:00:00 2001 From: echonesis Date: Wed, 23 Jul 2025 11:23:09 +0800 Subject: [PATCH 03/13] checkstyle update --- .../apache/hadoop/hdds/scm/container/ContainerChecksums.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java index 80567ed3104e..ed35de81e022 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdds.scm.container; import java.util.Objects; - import net.jcip.annotations.Immutable; /** @@ -90,4 +89,4 @@ public String toString() { } return sb.toString(); } -} \ No newline at end of file +} From 0253d2d0379c5ae4659f7ac3ad144eefc670c1df Mon Sep 17 00:00:00 2001 From: echonesis Date: Wed, 23 Jul 2025 13:51:56 +0800 Subject: [PATCH 04/13] Replica modification update --- .../hadoop/hdds/scm/container/ContainerReplica.java | 4 ++++ .../hdds/scm/container/TestContainerReportHandler.java | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java index 6968aaea8837..5c9bd57cd881 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplica.java @@ -126,6 +126,10 @@ public ContainerChecksums getChecksums() { return checksums; } + public long getDataChecksum() { + return checksums.getDataChecksum(); + } + @Override public int hashCode() { return new HashCodeBuilder(61, 71) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java index 0af693649fe8..a3db471334de 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java @@ -1205,7 +1205,7 @@ public void testWithNoContainerDataChecksum() throws Exception { // All replicas should start with an empty data checksum in SCM. boolean contOneDataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getChecksums().getDataChecksum() == 0); + .allMatch(r -> r.getDataChecksum() == 0); assertTrue(contOneDataChecksumsEmpty, "Replicas of container one should not yet have any data checksums."); // Send a report to SCM from one datanode that still does not have a data checksum. @@ -1222,7 +1222,7 @@ public void testWithNoContainerDataChecksum() throws Exception { // Regardless of which datanode sent the report, none of them have checksums, so all replica's data checksums // should remain empty. boolean containerDataChecksumEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getChecksums().getDataChecksum() == 0); + .allMatch(r -> r.getDataChecksum() == 0); assertTrue(containerDataChecksumEmpty, "Replicas of the container should not have any data checksums."); } @@ -1254,7 +1254,7 @@ public void testWithContainerDataChecksum() throws Exception { // All replicas should start with an empty data checksum in SCM. boolean dataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getChecksums().getDataChecksum() == 0); + .allMatch(r -> r.getDataChecksum() == 0); assertTrue(dataChecksumsEmpty, "Replicas of container one should not yet have any data checksums."); // For each datanode, send a container report with a mismatched checksum. @@ -1278,7 +1278,7 @@ public void testWithContainerDataChecksum() throws Exception { int numReplicasChecked = 0; for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) { long expectedChecksum = createUniqueDataChecksumForReplica(contID, replica.getDatanodeDetails().getUuidString()); - assertEquals(expectedChecksum, replica.getChecksums().getDataChecksum()); + assertEquals(expectedChecksum, replica.getDataChecksum()); numReplicasChecked++; } assertEquals(numNodes, numReplicasChecked); @@ -1304,7 +1304,7 @@ public void testWithContainerDataChecksum() throws Exception { numReplicasChecked = 0; for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) { long expectedChecksum = createMatchingDataChecksumForReplica(contID); - assertEquals(expectedChecksum, replica.getChecksums().getDataChecksum()); + assertEquals(expectedChecksum, replica.getDataChecksum()); numReplicasChecked++; } assertEquals(numNodes, numReplicasChecked); From c9425f4f6b1dfe731bda1ccba9492ef72fb1aa2c Mon Sep 17 00:00:00 2001 From: echonesis Date: Thu, 24 Jul 2025 18:24:56 +0800 Subject: [PATCH 05/13] review update --- .../scm/container/ContainerChecksums.java | 28 ++++++------------- .../scm/container/TestContainerChecksums.java | 10 +++---- hadoop-hdds/server-scm/pom.xml | 4 --- .../AbstractContainerReportHandler.java | 2 +- ...groundContainerDataScannerIntegration.java | 17 +++++------ .../recon/fsck/ContainerHealthStatus.java | 3 +- .../ozone/recon/fsck/ContainerHealthTask.java | 8 +++--- .../recon/scm/ContainerReplicaHistory.java | 2 +- .../recon/api/TestContainerEndpoint.java | 18 ++++++------ .../recon/fsck/TestContainerHealthStatus.java | 8 +++--- .../recon/fsck/TestContainerHealthTask.java | 4 +-- ...estContainerHealthTaskRecordGenerator.java | 4 +-- .../recon/scm/TestReconContainerManager.java | 4 +-- 13 files changed, 48 insertions(+), 64 deletions(-) rename hadoop-hdds/{server-scm => common}/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java (72%) rename hadoop-hdds/{server-scm => common}/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java (83%) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java similarity index 72% rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java index ed35de81e022..65ad22d18fca 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java @@ -28,12 +28,12 @@ public final class ContainerChecksums { private static final ContainerChecksums UNKNOWN = - new ContainerChecksums(0, null); + new ContainerChecksums(0, 0L); private final long dataChecksum; - private final Long metadataChecksum; // nullable for future use + private final long metadataChecksum; - private ContainerChecksums(long dataChecksum, Long metadataChecksum) { + private ContainerChecksums(long dataChecksum, long metadataChecksum) { this.dataChecksum = dataChecksum; this.metadataChecksum = metadataChecksum; } @@ -41,15 +41,7 @@ private ContainerChecksums(long dataChecksum, Long metadataChecksum) { public static ContainerChecksums unknown() { return UNKNOWN; } - - public static ContainerChecksums dataOnly(long dataChecksum) { - return new ContainerChecksums(dataChecksum, null); - } - - public static ContainerChecksums metadataOnly(long metadataChecksum) { - return new ContainerChecksums(0, metadataChecksum); - } - + public static ContainerChecksums of(long dataChecksum, long metadataChecksum) { return new ContainerChecksums(dataChecksum, metadataChecksum); } @@ -58,7 +50,7 @@ public long getDataChecksum() { return dataChecksum; } - public Long getMetadataChecksum() { + public long getMetadataChecksum() { return metadataChecksum; } @@ -72,7 +64,7 @@ public boolean equals(Object obj) { } ContainerChecksums that = (ContainerChecksums) obj; return dataChecksum == that.dataChecksum && - Objects.equals(metadataChecksum, that.metadataChecksum); + metadataChecksum == that.metadataChecksum; } @Override @@ -82,11 +74,7 @@ public int hashCode() { @Override public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("data=").append(Long.toHexString(dataChecksum)); - if (metadataChecksum != null) { - sb.append(", metadata=").append(Long.toHexString(metadataChecksum)); - } - return sb.toString(); + return "data=" + Long.toHexString(dataChecksum) + + ", metadata=" + Long.toHexString(metadataChecksum); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java similarity index 83% rename from hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java rename to hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java index 4db5248a3538..b9827bed813a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java @@ -26,9 +26,9 @@ class TestContainerChecksums { @Test void testEqualsAndHashCode() { - ContainerChecksums c1 = ContainerChecksums.dataOnly(123L); - ContainerChecksums c2 = ContainerChecksums.dataOnly(123L); - ContainerChecksums c3 = ContainerChecksums.dataOnly(456L); + ContainerChecksums c1 = ContainerChecksums.of(123L, 0L); + ContainerChecksums c2 = ContainerChecksums.of(123L, 0L); + ContainerChecksums c3 = ContainerChecksums.of(456L, 0L); ContainerChecksums c4 = ContainerChecksums.of(123L, 789L); ContainerChecksums c5 = ContainerChecksums.of(123L, 789L); ContainerChecksums c6 = ContainerChecksums.of(123L, 790L); @@ -43,8 +43,8 @@ void testEqualsAndHashCode() { @Test void testToString() { - ContainerChecksums c1 = ContainerChecksums.dataOnly(0x1234ABCDL); - assertThat(c1.toString()).contains("data=1234abcd").doesNotContain("metadata="); + ContainerChecksums c1 = ContainerChecksums.of(0x1234ABCDL, 0L); + assertThat(c1.toString()).contains("data=1234abcd", "metadata=0"); ContainerChecksums c2 = ContainerChecksums.of(0x1234ABCDL, 0xDEADBEEFL); assertThat(c2.toString()).contains("data=1234abcd").contains("metadata=deadbeef"); diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml index 6a0525e517b6..80a48482773f 100644 --- a/hadoop-hdds/server-scm/pom.xml +++ b/hadoop-hdds/server-scm/pom.xml @@ -43,10 +43,6 @@ jackson-databind - - com.github.stephenc.jcip - jcip-annotations - com.google.guava guava diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java index 3020b1dfeaa4..b24c4ea02337 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java @@ -362,7 +362,7 @@ private void updateContainerReplica(final DatanodeDetails datanodeDetails, .setReplicaIndex(replicaProto.getReplicaIndex()) .setBytesUsed(replicaProto.getUsed()) .setEmpty(replicaProto.getIsEmpty()) - .setChecksums(ContainerChecksums.dataOnly(replicaProto.getDataChecksum())) + .setChecksums(ContainerChecksums.of(replicaProto.getDataChecksum(), 0L)) .build(); if (replica.getState().equals(State.DELETED)) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java index 7acf9e472484..fff38740160b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java @@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State; +import org.apache.hadoop.hdds.scm.container.ContainerChecksums; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.keyvalue.TestContainerCorruptions; import org.apache.hadoop.ozone.container.ozoneimpl.BackgroundContainerDataScanner; @@ -82,8 +83,8 @@ void testCorruptionDetected(TestContainerCorruptions corruption) assertNotEquals(0, container.getContainerData().getDataChecksum()); waitForScmToSeeReplicaState(containerID, CLOSED); - long initialReportedDataChecksum = getContainerReplica(containerID).getChecksums().getDataChecksum(); - assertNotEquals(0, initialReportedDataChecksum); + ContainerChecksums initialReportedChecksum = getContainerReplica(containerID).getChecksums(); + assertNotEquals(ContainerChecksums.unknown(), initialReportedChecksum); corruption.applyTo(container); resumeScanner(); @@ -95,16 +96,16 @@ void testCorruptionDetected(TestContainerCorruptions corruption) // Wait for SCM to get a report of the unhealthy replica with a different checksum than before. waitForScmToSeeReplicaState(containerID, UNHEALTHY); - long newReportedDataChecksum = getContainerReplica(containerID).getChecksums().getDataChecksum(); + ContainerChecksums newReportedChecksum = getContainerReplica(containerID).getChecksums(); if (corruption == TestContainerCorruptions.MISSING_METADATA_DIR || corruption == TestContainerCorruptions.MISSING_CONTAINER_DIR) { // In these cases, the new tree will not be able to be written since it exists in the metadata directory. // When the tree write fails, the in-memory checksum should remain at its original value. - assertEquals(checksumToString(initialReportedDataChecksum), checksumToString(newReportedDataChecksum)); + assertEquals(initialReportedChecksum, newReportedChecksum); } else { - assertNotEquals(checksumToString(initialReportedDataChecksum), checksumToString(newReportedDataChecksum)); + assertNotEquals(initialReportedChecksum, newReportedChecksum); // Test that the scanner wrote updated checksum info to the disk. - assertReplicaChecksumMatches(container, newReportedDataChecksum); + assertReplicaChecksumMatches(container, newReportedChecksum); assertFalse(container.getContainerData().needsDataChecksum()); } @@ -118,10 +119,10 @@ void testCorruptionDetected(TestContainerCorruptions corruption) } } - private void assertReplicaChecksumMatches(Container container, long expectedChecksum) throws Exception { + private void assertReplicaChecksumMatches(Container container, ContainerChecksums expectedChecksum) throws Exception { assertTrue(containerChecksumFileExists(container.getContainerData().getContainerID())); long dataChecksumFromFile = readChecksumFile(container.getContainerData()) .getContainerMerkleTree().getDataChecksum(); - assertEquals(checksumToString(expectedChecksum), checksumToString(dataChecksumFromFile)); + assertEquals(checksumToString(expectedChecksum.getDataChecksum()), checksumToString(dataChecksumFromFile)); } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java index 8e6bcb1f0784..3465e7809ea8 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java @@ -161,10 +161,9 @@ public boolean isEmpty() { return numKeys == 0; } - public boolean isDataChecksumMismatched() { + public boolean areChecksumsMismatched() { return !replicas.isEmpty() && replicas.stream() .map(ContainerReplica::getChecksums) - .map(ContainerChecksums::getDataChecksum) .distinct() .count() != 1; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java index 56fc47132b29..a6b6f3a8c30f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthTask.java @@ -385,7 +385,7 @@ private void processContainer(ContainerInfo container, long currentTime, containerReplicas, placementPolicy, reconContainerMetadataManager, conf); - if ((h.isHealthilyReplicated() && !h.isDataChecksumMismatched()) || h.isDeleted()) { + if ((h.isHealthilyReplicated() && !h.areChecksumsMismatched()) || h.isDeleted()) { return; } // For containers deleted in SCM, we sync the container state here. @@ -563,7 +563,7 @@ public static List generateUnhealthyRecords( Map> unhealthyContainerStateStatsMap) { List records = new ArrayList<>(); - if ((container.isHealthilyReplicated() && !container.isDataChecksumMismatched()) || container.isDeleted()) { + if ((container.isHealthilyReplicated() && !container.areChecksumsMismatched()) || container.isDeleted()) { return records; } @@ -610,7 +610,7 @@ public static List generateUnhealthyRecords( populateContainerStats(container, UnHealthyContainerStates.OVER_REPLICATED, unhealthyContainerStateStatsMap); } - if (container.isDataChecksumMismatched() + if (container.areChecksumsMismatched() && !recordForStateExists.contains( UnHealthyContainerStates.REPLICA_MISMATCH.toString())) { records.add(recordForState( @@ -686,7 +686,7 @@ private static boolean keepMisReplicatedRecord( private static boolean keepReplicaMismatchRecord( ContainerHealthStatus container, UnhealthyContainersRecord rec) { - if (container.isDataChecksumMismatched()) { + if (container.areChecksumsMismatched()) { updateExpectedReplicaCount(rec, container.getReplicationFactor()); updateActualReplicaCount(rec, container.getReplicaCount()); updateReplicaDelta(rec, container.replicaDelta()); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java index ba0b0eda7dd2..239ca565adff 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java @@ -96,7 +96,7 @@ public static ContainerReplicaHistory fromProto( ContainerReplicaHistoryProto proto) { return new ContainerReplicaHistory(UUID.fromString(proto.getUuid()), proto.getFirstSeenTime(), proto.getLastSeenTime(), proto.getBcsId(), - proto.getState(), ContainerChecksums.dataOnly(proto.getDataChecksum())); + proto.getState(), ContainerChecksums.of(proto.getDataChecksum(), 0L)); } public ContainerReplicaHistoryProto toProto() { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java index dd5020dc98f5..85159e6daaba 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java @@ -1036,12 +1036,12 @@ public void testGetReplicaHistoryForContainer() throws IOException { final UUID u2 = newDatanode("host2", "127.0.0.2"); final UUID u3 = newDatanode("host3", "127.0.0.3"); final UUID u4 = newDatanode("host4", "127.0.0.4"); - reconContainerManager.upsertContainerHistory(1L, u1, 1L, 1L, "OPEN", ContainerChecksums.dataOnly(1234L)); - reconContainerManager.upsertContainerHistory(1L, u2, 2L, 1L, "OPEN", ContainerChecksums.dataOnly(1234L)); - reconContainerManager.upsertContainerHistory(1L, u3, 3L, 1L, "OPEN", ContainerChecksums.dataOnly(1234L)); - reconContainerManager.upsertContainerHistory(1L, u4, 4L, 1L, "OPEN", ContainerChecksums.dataOnly(1234L)); + reconContainerManager.upsertContainerHistory(1L, u1, 1L, 1L, "OPEN", ContainerChecksums.of(1234L, 0L)); + reconContainerManager.upsertContainerHistory(1L, u2, 2L, 1L, "OPEN", ContainerChecksums.of(1234L, 0L)); + reconContainerManager.upsertContainerHistory(1L, u3, 3L, 1L, "OPEN", ContainerChecksums.of(1234L, 0L)); + reconContainerManager.upsertContainerHistory(1L, u4, 4L, 1L, "OPEN", ContainerChecksums.of(1234L, 0L)); - reconContainerManager.upsertContainerHistory(1L, u1, 5L, 1L, "OPEN", ContainerChecksums.dataOnly(1234L)); + reconContainerManager.upsertContainerHistory(1L, u1, 5L, 1L, "OPEN", ContainerChecksums.of(1234L, 0L)); Response response = containerEndpoint.getReplicaHistoryForContainer(1L); List histories = @@ -1151,13 +1151,13 @@ private void createUnhealthyRecord(int id, String state, int expected, long differentChecksum = dataChecksumMismatch ? 2345L : 1234L; reconContainerManager.upsertContainerHistory(cID, uuid1, 1L, 1L, - "UNHEALTHY", ContainerChecksums.dataOnly(differentChecksum)); + "UNHEALTHY", ContainerChecksums.of(differentChecksum, 0L)); reconContainerManager.upsertContainerHistory(cID, uuid2, 2L, 1L, - "UNHEALTHY", ContainerChecksums.dataOnly(differentChecksum)); + "UNHEALTHY", ContainerChecksums.of(differentChecksum, 0L)); reconContainerManager.upsertContainerHistory(cID, uuid3, 3L, 1L, - "UNHEALTHY", ContainerChecksums.dataOnly(1234L)); + "UNHEALTHY", ContainerChecksums.of(1234L, 0L)); reconContainerManager.upsertContainerHistory(cID, uuid4, 4L, 1L, - "UNHEALTHY", ContainerChecksums.dataOnly(1234L)); + "UNHEALTHY", ContainerChecksums.of(1234L, 0L)); } protected ContainerWithPipeline getTestContainer( diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java index b9dd49095d35..6fcf5c5fb4f6 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthStatus.java @@ -190,7 +190,7 @@ public void testSameDataChecksumContainer() { assertFalse(status.isUnderReplicated()); assertFalse(status.isOverReplicated()); assertFalse(status.isMisReplicated()); - assertFalse(status.isDataChecksumMismatched()); + assertFalse(status.areChecksumsMismatched()); } @Test @@ -207,7 +207,7 @@ public void testDataChecksumMismatchContainer() { assertFalse(status.isUnderReplicated()); assertFalse(status.isOverReplicated()); assertFalse(status.isMisReplicated()); - assertTrue(status.isDataChecksumMismatched()); + assertTrue(status.areChecksumsMismatched()); } /** @@ -417,7 +417,7 @@ private Set generateReplicas(ContainerInfo cont, replicas.add(new ContainerReplica.ContainerReplicaBuilder() .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) - .setChecksums(ContainerChecksums.dataOnly(1234L)) + .setChecksums(ContainerChecksums.of(1234L, 0L)) .setContainerState(s) .build()); } @@ -433,7 +433,7 @@ private Set generateMismatchedReplicas(ContainerInfo cont, .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) .setContainerState(s) - .setChecksums(ContainerChecksums.dataOnly(checksum)) + .setChecksums(ContainerChecksums.of(checksum, 0L)) .build()); checksum++; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java index f01d690f0df6..93bf09a6d071 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTask.java @@ -691,7 +691,7 @@ private Set getMockReplicas( .setContainerState(s) .setContainerID(ContainerID.valueOf(containerId)) .setSequenceId(1) - .setChecksums(ContainerChecksums.dataOnly(1234L)) + .setChecksums(ContainerChecksums.of(1234L, 0L)) .build()); } return replicas; @@ -707,7 +707,7 @@ private Set getMockReplicasChecksumMismatch( .setContainerState(s) .setContainerID(ContainerID.valueOf(containerId)) .setSequenceId(1) - .setChecksums(ContainerChecksums.dataOnly(checksum)) + .setChecksums(ContainerChecksums.of(checksum, 0L)) .build()); checksum++; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java index c7dc3bdcba73..9e8b3905a58a 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestContainerHealthTaskRecordGenerator.java @@ -643,7 +643,7 @@ private Set generateReplicas(ContainerInfo cont, replicas.add(new ContainerReplica.ContainerReplicaBuilder() .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) - .setChecksums(ContainerChecksums.dataOnly(1234L)) + .setChecksums(ContainerChecksums.of(1234L, 0L)) .setContainerState(s) .build()); } @@ -659,7 +659,7 @@ private Set generateMismatchedReplicas(ContainerInfo cont, .setContainerID(cont.containerID()) .setDatanodeDetails(MockDatanodeDetails.randomDatanodeDetails()) .setContainerState(s) - .setChecksums(ContainerChecksums.dataOnly(checksum)) + .setChecksums(ContainerChecksums.of(checksum, 0L)) .build()); checksum++; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java index ba9df7e3cd0f..cbbc6aa6c88f 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java @@ -211,7 +211,7 @@ public void testUpdateAndRemoveContainerReplica() ContainerReplica containerReplica1 = ContainerReplica.newBuilder() .setContainerID(containerID1).setContainerState(State.OPEN) .setDatanodeDetails(datanodeDetails1).setSequenceId(1001L) - .setChecksums(ContainerChecksums.dataOnly(1234L)).build(); + .setChecksums(ContainerChecksums.of(1234L, 0L)).build(); final ReconContainerManager containerManager = getContainerManager(); final Map> repHistMap = @@ -259,7 +259,7 @@ public void testUpdateAndRemoveContainerReplica() final ContainerReplica containerReplica2 = ContainerReplica.newBuilder() .setContainerID(containerID1).setContainerState(State.OPEN) .setDatanodeDetails(datanodeDetails2).setSequenceId(1051L) - .setChecksums(ContainerChecksums.dataOnly(1234L)).build(); + .setChecksums(ContainerChecksums.of(1234L, 0L)).build(); // Add replica to DN02 containerManager.updateContainerReplica(containerID1, containerReplica2); From 242f2e410943ea9503c8c90fe98fc3e56d594ed9 Mon Sep 17 00:00:00 2001 From: echonesis Date: Fri, 25 Jul 2025 11:30:50 +0800 Subject: [PATCH 06/13] placeholder update --- .../scm/container/ContainerChecksums.java | 48 ++++++++++++++++--- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java index 65ad22d18fca..b1d37db39979 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java @@ -28,10 +28,15 @@ public final class ContainerChecksums { private static final ContainerChecksums UNKNOWN = - new ContainerChecksums(0, 0L); + new ContainerChecksums(-1L, -1L); - private final long dataChecksum; - private final long metadataChecksum; + // Checksum of the data within the wrapper. + private long dataChecksum; + private static final long UNSET_DATA_CHECKSUM = -1; + + // Checksum of the metadata within the wrapper. + private long metadataChecksum; + private static final long UNSET_METADATA_CHECKSUM = -1; private ContainerChecksums(long dataChecksum, long metadataChecksum) { this.dataChecksum = dataChecksum; @@ -45,15 +50,44 @@ public static ContainerChecksums unknown() { public static ContainerChecksums of(long dataChecksum, long metadataChecksum) { return new ContainerChecksums(dataChecksum, metadataChecksum); } + public void setDataChecksum(long dataChecksum) { + if (dataChecksum < 0) { + throw new IllegalArgumentException("Data checksum cannot be set to a negative number."); + } + this.dataChecksum = dataChecksum; + } public long getDataChecksum() { + // UNSET_DATA_CHECKSUM is an internal placeholder, it should not be used outside this class. + if (needsDataChecksum()) { + return 0; + } return dataChecksum; } + public boolean needsDataChecksum() { + return dataChecksum == UNSET_DATA_CHECKSUM; + } + + public void setMetadataChecksum(long metadataChecksum) { + if (metadataChecksum < 0) { + throw new IllegalArgumentException("Metadata checksum cannot be set to a negative number."); + } + this.metadataChecksum = metadataChecksum; + } + public long getMetadataChecksum() { + // UNSET_DATA_CHECKSUM is an internal placeholder, it should not be used outside this class. + if (needsMetadataChecksum()) { + return 0; + } return metadataChecksum; } + public boolean needsMetadataChecksum() { + return metadataChecksum == UNSET_METADATA_CHECKSUM; + } + @Override public boolean equals(Object obj) { if (this == obj) { @@ -63,8 +97,8 @@ public boolean equals(Object obj) { return false; } ContainerChecksums that = (ContainerChecksums) obj; - return dataChecksum == that.dataChecksum && - metadataChecksum == that.metadataChecksum; + return getDataChecksum() == that.getDataChecksum() && + getMetadataChecksum() == that.getMetadataChecksum(); } @Override @@ -74,7 +108,7 @@ public int hashCode() { @Override public String toString() { - return "data=" + Long.toHexString(dataChecksum) + - ", metadata=" + Long.toHexString(metadataChecksum); + return "data=" + Long.toHexString(getDataChecksum()) + + ", metadata=" + Long.toHexString(getMetadataChecksum()); } } From cffc5c2d0c10fdba9746866af9f5a4213c2e4697 Mon Sep 17 00:00:00 2001 From: echonesis Date: Fri, 25 Jul 2025 11:40:27 +0800 Subject: [PATCH 07/13] checkstyle update --- .../apache/hadoop/hdds/scm/container/ContainerChecksums.java | 1 + .../scanner/TestBackgroundContainerDataScannerIntegration.java | 3 ++- .../apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java index b1d37db39979..a31c965b0dbd 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java @@ -50,6 +50,7 @@ public static ContainerChecksums unknown() { public static ContainerChecksums of(long dataChecksum, long metadataChecksum) { return new ContainerChecksums(dataChecksum, metadataChecksum); } + public void setDataChecksum(long dataChecksum) { if (dataChecksum < 0) { throw new IllegalArgumentException("Data checksum cannot be set to a negative number."); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java index fff38740160b..cd6b3581c10c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java @@ -119,7 +119,8 @@ void testCorruptionDetected(TestContainerCorruptions corruption) } } - private void assertReplicaChecksumMatches(Container container, ContainerChecksums expectedChecksum) throws Exception { + private void assertReplicaChecksumMatches( + Container container, ContainerChecksums expectedChecksum) throws Exception { assertTrue(containerChecksumFileExists(container.getContainerData().getContainerID())); long dataChecksumFromFile = readChecksumFile(container.getContainerData()) .getContainerMerkleTree().getDataChecksum(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java index 3465e7809ea8..7a69c403050d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ContainerHealthStatus.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.ContainerPlacementStatus; import org.apache.hadoop.hdds.scm.PlacementPolicy; -import org.apache.hadoop.hdds.scm.container.ContainerChecksums; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaCount; From 620efb040b9637cbb96724a0c3a004534e47b94e Mon Sep 17 00:00:00 2001 From: echonesis Date: Fri, 25 Jul 2025 13:43:21 +0800 Subject: [PATCH 08/13] conflict fix --- .../hdds/scm/container/ContainerChecksums.java | 18 ++---------------- ...kgroundContainerDataScannerIntegration.java | 2 +- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java index a31c965b0dbd..36d89bcbdd56 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java @@ -31,11 +31,11 @@ public final class ContainerChecksums { new ContainerChecksums(-1L, -1L); // Checksum of the data within the wrapper. - private long dataChecksum; + private final long dataChecksum; private static final long UNSET_DATA_CHECKSUM = -1; // Checksum of the metadata within the wrapper. - private long metadataChecksum; + private final long metadataChecksum; private static final long UNSET_METADATA_CHECKSUM = -1; private ContainerChecksums(long dataChecksum, long metadataChecksum) { @@ -50,13 +50,6 @@ public static ContainerChecksums unknown() { public static ContainerChecksums of(long dataChecksum, long metadataChecksum) { return new ContainerChecksums(dataChecksum, metadataChecksum); } - - public void setDataChecksum(long dataChecksum) { - if (dataChecksum < 0) { - throw new IllegalArgumentException("Data checksum cannot be set to a negative number."); - } - this.dataChecksum = dataChecksum; - } public long getDataChecksum() { // UNSET_DATA_CHECKSUM is an internal placeholder, it should not be used outside this class. @@ -70,13 +63,6 @@ public boolean needsDataChecksum() { return dataChecksum == UNSET_DATA_CHECKSUM; } - public void setMetadataChecksum(long metadataChecksum) { - if (metadataChecksum < 0) { - throw new IllegalArgumentException("Metadata checksum cannot be set to a negative number."); - } - this.metadataChecksum = metadataChecksum; - } - public long getMetadataChecksum() { // UNSET_DATA_CHECKSUM is an internal placeholder, it should not be used outside this class. if (needsMetadataChecksum()) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java index cd6b3581c10c..1a3355f8fd4c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java @@ -120,7 +120,7 @@ void testCorruptionDetected(TestContainerCorruptions corruption) } private void assertReplicaChecksumMatches( - Container container, ContainerChecksums expectedChecksum) throws Exception { + Container container, ContainerChecksums expectedChecksum) throws Exception { assertTrue(containerChecksumFileExists(container.getContainerData().getContainerID())); long dataChecksumFromFile = readChecksumFile(container.getContainerData()) .getContainerMerkleTree().getDataChecksum(); From 42d61cf883825e41ec140e85eac3ef24ba0506c2 Mon Sep 17 00:00:00 2001 From: echonesis Date: Fri, 25 Jul 2025 16:38:23 +0800 Subject: [PATCH 09/13] reduce mod test --- .../hdds/scm/container/TestContainerChecksums.java | 3 +++ ...tBackgroundContainerMetadataScannerIntegration.java | 10 +++++----- .../TestOnDemandContainerScannerIntegration.java | 4 ++-- .../ozone/recon/scm/ContainerReplicaHistory.java | 4 ++++ .../hadoop/ozone/recon/scm/ReconContainerManager.java | 2 +- .../ozone/recon/scm/TestReconContainerManager.java | 4 ++-- 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java index b9827bed813a..6e9d34c5562b 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerChecksums.java @@ -48,5 +48,8 @@ void testToString() { ContainerChecksums c2 = ContainerChecksums.of(0x1234ABCDL, 0xDEADBEEFL); assertThat(c2.toString()).contains("data=1234abcd").contains("metadata=deadbeef"); + + ContainerChecksums c3 = ContainerChecksums.unknown(); + assertThat(c3.toString()).contains("data=0").contains("metadata=0"); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java index bc324dad9522..b25df7e11369 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java @@ -98,7 +98,7 @@ void testCorruptionDetected(TestContainerCorruptions corruption) assertEquals(State.CLOSED, closedContainer.getContainerState()); assertTrue(containerChecksumFileExists(closedContainerID)); waitForScmToSeeReplicaState(closedContainerID, CLOSED); - long initialClosedChecksum = getContainerReplica(closedContainerID).getChecksums().getDataChecksum(); + long initialClosedChecksum = getContainerReplica(closedContainerID).getDataChecksum(); assertNotEquals(0, initialClosedChecksum); long openContainerID = writeDataToOpenContainer(); @@ -106,7 +106,7 @@ void testCorruptionDetected(TestContainerCorruptions corruption) assertEquals(State.OPEN, openContainer.getContainerState()); waitForScmToSeeReplicaState(openContainerID, OPEN); // Open containers should not yet have a checksum generated. - assertEquals(0, getContainerReplica(openContainerID).getChecksums().getDataChecksum()); + assertEquals(0, getContainerReplica(openContainerID).getDataChecksum()); // Corrupt both containers. corruption.applyTo(closedContainer); @@ -124,17 +124,17 @@ void testCorruptionDetected(TestContainerCorruptions corruption) // The metadata scanner does not generate data checksums and the other scanners have been turned off for this // test, so the data checksums should not change. waitForScmToSeeReplicaState(closedContainerID, UNHEALTHY); - assertEquals(initialClosedChecksum, getContainerReplica(closedContainerID).getChecksums().getDataChecksum()); + assertEquals(initialClosedChecksum, getContainerReplica(closedContainerID).getDataChecksum()); waitForScmToSeeReplicaState(openContainerID, UNHEALTHY); if (corruption == MISSING_METADATA_DIR || corruption == MISSING_CONTAINER_DIR) { // In these cases the tree cannot be generated when the container is marked unhealthy and the checksum should // remain at 0. // The tree is generated from metadata by the container changing to unhealthy, not by the metadata scanner. - assertEquals(0, getContainerReplica(openContainerID).getChecksums().getDataChecksum()); + assertEquals(0, getContainerReplica(openContainerID).getDataChecksum()); } else { // The checksum will be generated for the first time when the container is marked unhealthy. // The tree is generated from metadata by the container changing to unhealthy, not by the metadata scanner. - assertNotEquals(0, getContainerReplica(openContainerID).getChecksums().getDataChecksum()); + assertNotEquals(0, getContainerReplica(openContainerID).getDataChecksum()); } // Once the unhealthy replica is reported, the open container's lifecycle diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerScannerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerScannerIntegration.java index eed26f8a2c50..136a307e0ab8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerScannerIntegration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerScannerIntegration.java @@ -114,7 +114,7 @@ void testCorruptionDetected(TestContainerCorruptions corruption) assertTrue(containerChecksumFileExists(containerID)); waitForScmToSeeReplicaState(containerID, CLOSED); - long initialReportedDataChecksum = getContainerReplica(containerID).getChecksums().getDataChecksum(); + long initialReportedDataChecksum = getContainerReplica(containerID).getDataChecksum(); // Corrupt the container. corruption.applyTo(container); @@ -130,7 +130,7 @@ void testCorruptionDetected(TestContainerCorruptions corruption) // Wait for SCM to get a report of the unhealthy replica. waitForScmToSeeReplicaState(containerID, UNHEALTHY); corruption.assertLogged(containerID, 1, logCapturer); - long newReportedDataChecksum = getContainerReplica(containerID).getChecksums().getDataChecksum(); + long newReportedDataChecksum = getContainerReplica(containerID).getDataChecksum(); if (corruption == TestContainerCorruptions.MISSING_METADATA_DIR || corruption == TestContainerCorruptions.MISSING_CONTAINER_DIR) { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java index 239ca565adff..d143f0365beb 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java @@ -84,6 +84,10 @@ public void setState(String state) { this.state = state; } + public long getDataChecksum() { + return getChecksums().getDataChecksum(); + } + public ContainerChecksums getChecksums() { return checksums; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java index e21add7e84fd..586aad5fd68f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java @@ -393,7 +393,7 @@ public List getAllContainerHistory(long containerID) { final long lastSeenTime = entry.getValue().getLastSeenTime(); long bcsId = entry.getValue().getBcsId(); String state = entry.getValue().getState(); - long dataChecksum = entry.getValue().getChecksums().getDataChecksum(); + long dataChecksum = entry.getValue().getDataChecksum(); resList.add(new ContainerHistory(containerID, uuid.toString(), hostname, firstSeenTime, lastSeenTime, bcsId, state, dataChecksum)); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java index cbbc6aa6c88f..1d871b9974b9 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java @@ -239,7 +239,7 @@ public void testUpdateAndRemoveContainerReplica() assertEquals(repHist1.getLastSeenTime(), repHist1.getFirstSeenTime()); assertEquals(containerReplica1.getSequenceId().longValue(), repHist1.getBcsId()); - assertEquals(containerReplica1.getChecksums().getDataChecksum(), repHist1.getChecksums().getDataChecksum()); + assertEquals(containerReplica1.getDataChecksum(), repHist1.getDataChecksum()); // Let's update the entry again containerReplica1 = ContainerReplica.newBuilder() @@ -273,7 +273,7 @@ public void testUpdateAndRemoveContainerReplica() // Because this is a new entry, first seen time equals last seen time assertEquals(repHist2.getLastSeenTime(), repHist2.getFirstSeenTime()); assertEquals(1051L, repHist2.getBcsId()); - assertEquals(containerReplica2.getChecksums().getDataChecksum(), repHist2.getChecksums().getDataChecksum()); + assertEquals(containerReplica2.getDataChecksum(), repHist2.getDataChecksum()); // Remove replica from DN01 containerManager.removeContainerReplica(containerID1, containerReplica1); From 5fd256efb80d8f40b8b7bad10df9cbe9d0033dd4 Mon Sep 17 00:00:00 2001 From: echonesis Date: Fri, 25 Jul 2025 17:02:41 +0800 Subject: [PATCH 10/13] ra test --- .../hdds/scm/server/SCMClientProtocolServer.java | 2 +- .../TestIncrementalContainerReportHandler.java | 10 +++++----- .../org/apache/hadoop/hdds/scm/TestCloseContainer.java | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java index a261f8c253fd..03774c48699c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java @@ -360,7 +360,7 @@ public List getContainerReplicas( .setKeyCount(r.getKeyCount()) .setSequenceID(r.getSequenceId()) .setReplicaIndex(r.getReplicaIndex()) - .setDataChecksum(r.getChecksums().getDataChecksum()) + .setDataChecksum(r.getDataChecksum()) .build() ); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java index 1306d1bcbfae..41fbe2032a7b 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java @@ -648,7 +648,7 @@ public void testWithNoContainerDataChecksum() throws Exception { // All replicas should start with an empty data checksum in SCM. boolean contOneDataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getChecksums().getDataChecksum() == 0); + .allMatch(r -> r.getDataChecksum() == 0); assertTrue(contOneDataChecksumsEmpty, "Replicas of container one should not yet have any data checksums."); // Send a report to SCM from one datanode that still does not have a data checksum. @@ -663,7 +663,7 @@ public void testWithNoContainerDataChecksum() throws Exception { // Regardless of which datanode sent the report, none of them have checksums, so all replica's data checksums // should remain empty. boolean containerDataChecksumEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getChecksums().getDataChecksum() == 0); + .allMatch(r -> r.getDataChecksum() == 0); assertTrue(containerDataChecksumEmpty, "Replicas of the container should not have any data checksums."); } @@ -695,7 +695,7 @@ public void testWithContainerDataChecksum() throws Exception { // All replicas should start with a zero data checksum in SCM. boolean dataChecksumsEmpty = containerManager.getContainerReplicas(contID).stream() - .allMatch(r -> r.getChecksums().getDataChecksum() == 0); + .allMatch(r -> r.getDataChecksum() == 0); assertTrue(dataChecksumsEmpty, "Replicas of container one should not yet have any data checksums."); // For each datanode, send a container report with a mismatched checksum. @@ -721,7 +721,7 @@ public void testWithContainerDataChecksum() throws Exception { for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) { long expectedChecksum = createUniqueDataChecksumForReplica( contID, replica.getDatanodeDetails().getUuidString()); - assertEquals(expectedChecksum, replica.getChecksums().getDataChecksum()); + assertEquals(expectedChecksum, replica.getDataChecksum()); numReplicasChecked++; } assertEquals(numNodes, numReplicasChecked); @@ -748,7 +748,7 @@ public void testWithContainerDataChecksum() throws Exception { numReplicasChecked = 0; for (ContainerReplica replica: containerManager.getContainerReplicas(contID)) { long expectedChecksum = createMatchingDataChecksumForReplica(contID); - assertEquals(expectedChecksum, replica.getChecksums().getDataChecksum()); + assertEquals(expectedChecksum, replica.getDataChecksum()); numReplicasChecked++; } assertEquals(numNodes, numReplicasChecked); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java index d7e6395ab874..7910b6908c96 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestCloseContainer.java @@ -159,7 +159,7 @@ public void testReplicasAreReportedForClosedContainerAfterRestart() GenericTestUtils.waitFor(() -> getContainerReplicas(newContainer).size() == 3, 200, 30000); for (ContainerReplica replica : getContainerReplicas(newContainer)) { - assertNotEquals(0, replica.getChecksums().getDataChecksum()); + assertNotEquals(0, replica.getDataChecksum()); } } @@ -201,7 +201,7 @@ public void testCloseClosedContainer() } for (ContainerReplica replica : getContainerReplicas(container)) { - assertNotEquals(0, replica.getChecksums().getDataChecksum()); + assertNotEquals(0, replica.getDataChecksum()); } assertThrows(IOException.class, @@ -277,9 +277,9 @@ public void testContainerChecksumForClosedContainer() throws Exception { // Wait for SCM to receive container reports with non-zero checksums for all replicas GenericTestUtils.waitFor(() -> getContainerReplicas(containerInfo1).stream() - .allMatch(r -> r.getChecksums().getDataChecksum() != 0), 200, 5000); + .allMatch(r -> r.getDataChecksum() != 0), 200, 5000); GenericTestUtils.waitFor(() -> getContainerReplicas(containerInfo2).stream() - .allMatch(r -> r.getChecksums().getDataChecksum() != 0), 200, 5000); + .allMatch(r -> r.getDataChecksum() != 0), 200, 5000); } private boolean checkContainerCloseInDatanode(HddsDatanodeService hddsDatanode, From f4ef042a14a85413c8aa750233f682376bd8129a Mon Sep 17 00:00:00 2001 From: echonesis Date: Wed, 17 Dec 2025 01:15:01 +0800 Subject: [PATCH 11/13] fix: CR update --- .../hadoop/hdds/scm/container/ContainerChecksums.java | 11 +++++------ .../ozone/recon/scm/ContainerReplicaHistory.java | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java index 36d89bcbdd56..66e6de327165 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java @@ -26,18 +26,19 @@ */ @Immutable public final class ContainerChecksums { - - private static final ContainerChecksums UNKNOWN = - new ContainerChecksums(-1L, -1L); - // Checksum of the data within the wrapper. private final long dataChecksum; + // UNSET_DATA_CHECKSUM is an internal placeholder, it should not be used outside this class. private static final long UNSET_DATA_CHECKSUM = -1; // Checksum of the metadata within the wrapper. private final long metadataChecksum; + // UNSET_DATA_CHECKSUM is an internal placeholder, it should not be used outside this class. private static final long UNSET_METADATA_CHECKSUM = -1; + private static final ContainerChecksums UNKNOWN = + new ContainerChecksums(UNSET_DATA_CHECKSUM, UNSET_METADATA_CHECKSUM); + private ContainerChecksums(long dataChecksum, long metadataChecksum) { this.dataChecksum = dataChecksum; this.metadataChecksum = metadataChecksum; @@ -52,7 +53,6 @@ public static ContainerChecksums of(long dataChecksum, long metadataChecksum) { } public long getDataChecksum() { - // UNSET_DATA_CHECKSUM is an internal placeholder, it should not be used outside this class. if (needsDataChecksum()) { return 0; } @@ -64,7 +64,6 @@ public boolean needsDataChecksum() { } public long getMetadataChecksum() { - // UNSET_DATA_CHECKSUM is an internal placeholder, it should not be used outside this class. if (needsMetadataChecksum()) { return 0; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java index d143f0365beb..72dcbc5f6034 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java @@ -49,7 +49,7 @@ public ContainerReplicaHistory(UUID id, Long firstSeenTime, this.lastSeenTime = lastSeenTime; this.bcsId = bcsId; this.state = state; - this.checksums = checksums; + setChecksums(checksums); } public long getBcsId() { @@ -93,7 +93,7 @@ public ContainerChecksums getChecksums() { } public void setChecksums(ContainerChecksums checksums) { - this.checksums = checksums; + this.checksums = checksums != null ? checksums : ContainerChecksums.unknown(); } public static ContainerReplicaHistory fromProto( @@ -107,7 +107,7 @@ public ContainerReplicaHistoryProto toProto() { return ContainerReplicaHistoryProto.newBuilder().setUuid(uuid.toString()) .setFirstSeenTime(firstSeenTime).setLastSeenTime(lastSeenTime) .setBcsId(bcsId).setState(state) - .setDataChecksum(checksums != null ? checksums.getDataChecksum() : 0L) + .setDataChecksum(checksums.getDataChecksum()) .build(); } } From df89c7fb0332afafd55706e31e6ec2bb5377084f Mon Sep 17 00:00:00 2001 From: echonesis Date: Thu, 18 Dec 2025 00:31:54 +0800 Subject: [PATCH 12/13] fix: CR update --- .../scm/container/ContainerChecksums.java | 31 ++++++------------- .../AbstractContainerReportHandler.java | 2 +- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java index 66e6de327165..ef089a87d17e 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerChecksums.java @@ -23,21 +23,18 @@ /** * Wrapper for container checksums (data, metadata, etc.). * Provides equality, hash, and hex string rendering. + * A value of 0 indicates an unknown or unset checksum. */ @Immutable public final class ContainerChecksums { // Checksum of the data within the wrapper. private final long dataChecksum; - // UNSET_DATA_CHECKSUM is an internal placeholder, it should not be used outside this class. - private static final long UNSET_DATA_CHECKSUM = -1; // Checksum of the metadata within the wrapper. private final long metadataChecksum; - // UNSET_DATA_CHECKSUM is an internal placeholder, it should not be used outside this class. - private static final long UNSET_METADATA_CHECKSUM = -1; private static final ContainerChecksums UNKNOWN = - new ContainerChecksums(UNSET_DATA_CHECKSUM, UNSET_METADATA_CHECKSUM); + new ContainerChecksums(0L, 0L); private ContainerChecksums(long dataChecksum, long metadataChecksum) { this.dataChecksum = dataChecksum; @@ -47,33 +44,23 @@ private ContainerChecksums(long dataChecksum, long metadataChecksum) { public static ContainerChecksums unknown() { return UNKNOWN; } - + + public static ContainerChecksums of(long dataChecksum) { + return new ContainerChecksums(dataChecksum, 0L); + } + public static ContainerChecksums of(long dataChecksum, long metadataChecksum) { return new ContainerChecksums(dataChecksum, metadataChecksum); } public long getDataChecksum() { - if (needsDataChecksum()) { - return 0; - } return dataChecksum; } - public boolean needsDataChecksum() { - return dataChecksum == UNSET_DATA_CHECKSUM; - } - public long getMetadataChecksum() { - if (needsMetadataChecksum()) { - return 0; - } return metadataChecksum; } - public boolean needsMetadataChecksum() { - return metadataChecksum == UNSET_METADATA_CHECKSUM; - } - @Override public boolean equals(Object obj) { if (this == obj) { @@ -83,8 +70,8 @@ public boolean equals(Object obj) { return false; } ContainerChecksums that = (ContainerChecksums) obj; - return getDataChecksum() == that.getDataChecksum() && - getMetadataChecksum() == that.getMetadataChecksum(); + return dataChecksum == that.dataChecksum && + metadataChecksum == that.metadataChecksum; } @Override diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java index b24c4ea02337..35908afff877 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java @@ -362,7 +362,7 @@ private void updateContainerReplica(final DatanodeDetails datanodeDetails, .setReplicaIndex(replicaProto.getReplicaIndex()) .setBytesUsed(replicaProto.getUsed()) .setEmpty(replicaProto.getIsEmpty()) - .setChecksums(ContainerChecksums.of(replicaProto.getDataChecksum(), 0L)) + .setChecksums(ContainerChecksums.of(replicaProto.getDataChecksum())) .build(); if (replica.getState().equals(State.DELETED)) { From bb8bb9f6a72830a1a010b641b3f6b73699acbbc5 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" <6454655+adoroszlai@users.noreply.github.com> Date: Thu, 18 Dec 2025 09:38:00 +0100 Subject: [PATCH 13/13] Use data-only factory method in ContainerReplicaHistory --- .../apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java index 72dcbc5f6034..971bc2d27258 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistory.java @@ -100,7 +100,7 @@ public static ContainerReplicaHistory fromProto( ContainerReplicaHistoryProto proto) { return new ContainerReplicaHistory(UUID.fromString(proto.getUuid()), proto.getFirstSeenTime(), proto.getLastSeenTime(), proto.getBcsId(), - proto.getState(), ContainerChecksums.of(proto.getDataChecksum(), 0L)); + proto.getState(), ContainerChecksums.of(proto.getDataChecksum())); } public ContainerReplicaHistoryProto toProto() {