HDDS-12824. Optimize container checksum read during datanode startup

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -88,6 +88,7 @@ public final class OzoneConsts {
   public static final Path ROOT_PATH = Paths.get(OZONE_ROOT);
 
   public static final String CONTAINER_EXTENSION = ".container";
+  public static final String CONTAINER_DATA_CHECKSUM_EXTENSION = ".tree";
   public static final String CONTAINER_META_PATH = "metadata";
   public static final String CONTAINER_TEMPORARY_CHUNK_PREFIX = "tmp";
   public static final String CONTAINER_CHUNK_NAME_DELIMITER = ".";
@@ -141,6 +142,7 @@ public final class OzoneConsts {
   public static final String CONTAINER_BYTES_USED = "#BYTESUSED";
   public static final String PENDING_DELETE_BLOCK_COUNT =
       "#PENDINGDELETEBLOCKCOUNT";
+  public static final String CONTAINER_DATA_CHECKSUM = "#DATACHECKSUM";
 
   /**
    * OM LevelDB prefixes.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java

@@ -19,6 +19,7 @@
 
 import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
 import static org.apache.hadoop.hdds.HddsUtils.checksumToString;
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DATA_CHECKSUM_EXTENSION;
 import static org.apache.hadoop.ozone.util.MetricUtil.captureLatencyNs;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -316,12 +317,12 @@ public static boolean hasContainerChecksumFile(ContainerData data) {
    */
   @VisibleForTesting
   public static File getContainerChecksumFile(ContainerData data) {
-    return new File(data.getMetadataPath(), data.getContainerID() + ".tree");
+    return new File(data.getMetadataPath(), data.getContainerID() + CONTAINER_DATA_CHECKSUM_EXTENSION);
   }
 
   @VisibleForTesting
   public static File getTmpContainerChecksumFile(ContainerData data) {
-    return new File(data.getMetadataPath(), data.getContainerID() + ".tree.tmp");
+    return new File(data.getMetadataPath(), data.getContainerID() + CONTAINER_DATA_CHECKSUM_EXTENSION + ".tmp");
   }
 
   private Lock getLock(long containerID) {

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java

@@ -22,6 +22,7 @@
 import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_COUNT;
 import static org.apache.hadoop.ozone.OzoneConsts.CHUNKS_PATH;
 import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_BYTES_USED;
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DATA_CHECKSUM;
 import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE;
 import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE_ROCKSDB;
 import static org.apache.hadoop.ozone.OzoneConsts.DELETE_TRANSACTION_KEY;
@@ -409,6 +410,10 @@ public String getPendingDeleteBlockCountKey() {
     return formatKey(PENDING_DELETE_BLOCK_COUNT);
   }
 
+  public String getContainerDataChecksumKey() {
+    return formatKey(CONTAINER_DATA_CHECKSUM);
+  }
+
   public String getDeletingBlockKeyPrefix() {
     return formatKey(DELETING_KEY_PREFIX);
   }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java

@@ -1409,7 +1409,7 @@ private void updateContainerChecksumFromMetadataIfNeeded(Container container) {
    * This method does not send an ICR with the updated checksum info.
    * @param container - Container for which the container merkle tree needs to be updated.
    */
-  private ContainerProtos.ContainerChecksumInfo updateAndGetContainerChecksumFromMetadata(
+  public ContainerProtos.ContainerChecksumInfo updateAndGetContainerChecksumFromMetadata(
       KeyValueContainer container) throws IOException {
     ContainerMerkleTreeWriter merkleTree = new ContainerMerkleTreeWriter();
     try (DBHandle dbHandle = BlockUtils.getDB(container.getContainerData(), conf);
@@ -1429,7 +1429,7 @@ private ContainerProtos.ContainerChecksumInfo updateAndGetContainerChecksumFromM
 
   private ContainerProtos.ContainerChecksumInfo updateAndGetContainerChecksum(Container container,
       ContainerMerkleTreeWriter treeWriter, boolean sendICR) throws IOException {
-    ContainerData containerData = container.getContainerData();
+    KeyValueContainerData containerData = (KeyValueContainerData) container.getContainerData();
 
     // Attempt to write the new data checksum to disk. If persisting this fails, keep using the original data
     // checksum to prevent divergence from what SCM sees in the ICR vs what datanode peers will see when pulling the
@@ -1441,6 +1441,14 @@ private ContainerProtos.ContainerChecksumInfo updateAndGetContainerChecksum(Cont
 
     if (updatedDataChecksum != originalDataChecksum) {
       containerData.setDataChecksum(updatedDataChecksum);
+      try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
+        dbHandle.getStore().getMetadataTable().put(containerData.getContainerDataChecksumKey(), updatedDataChecksum);
+      } catch (IOException e) {
+        LOG.error("Failed to update container data checksum in RocksDB for container {}. " +
+                "Continuing with original checksum for RocksDB {}.", containerData.getContainerID(),
+            originalDataChecksum, e);
+      }
+
       String message =
           "Container data checksum updated from " + checksumToString(originalDataChecksum) + " to " +
               checksumToString(updatedDataChecksum);

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java

@@ -24,6 +24,7 @@
 import static org.apache.hadoop.hdds.utils.Archiver.readEntry;
 import static org.apache.hadoop.hdds.utils.Archiver.tar;
 import static org.apache.hadoop.hdds.utils.Archiver.untar;
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DATA_CHECKSUM_EXTENSION;
 import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V3;
 
 import java.io.File;
@@ -42,6 +43,7 @@
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
 import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
 import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
 import org.apache.hadoop.ozone.container.common.interfaces.Container;
 import org.apache.hadoop.ozone.container.common.interfaces.ContainerPacker;
 import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerLocationUtil;
@@ -87,7 +89,10 @@ public byte[] unpackContainerData(Container<KeyValueContainerData> container,
 
     Path dbRoot = getDbPath(containerUntarDir, containerData);
     Path chunksRoot = getChunkPath(containerUntarDir, containerData);
-    byte[] descriptorFileContent = innerUnpack(input, dbRoot, chunksRoot);
+    Path containerMetadataPath = Paths.get(container.getContainerData().getMetadataPath());
+    Path tempContainerMetadataPath = Paths.get(containerUntarDir.toString(),
+        containerMetadataPath.getName(containerMetadataPath.getNameCount() - 1).toString());
+    byte[] descriptorFileContent = innerUnpack(input, dbRoot, chunksRoot, tempContainerMetadataPath);
 
     if (!Files.exists(destContainerDir)) {
       Files.createDirectories(destContainerDir);
@@ -96,9 +101,6 @@ public byte[] unpackContainerData(Container<KeyValueContainerData> container,
       // Before the atomic move, the destination dir is empty and doesn't have a metadata directory.
       // Writing the .container file will fail as the metadata dir doesn't exist.
       // So we instead save the container file to the containerUntarDir.
-      Path containerMetadataPath = Paths.get(container.getContainerData().getMetadataPath());
-      Path tempContainerMetadataPath = Paths.get(containerUntarDir.toString(),
-          containerMetadataPath.getName(containerMetadataPath.getNameCount() - 1).toString());
       persistCustomContainerState(container, descriptorFileContent, State.RECOVERING, tempContainerMetadataPath);
       Files.move(containerUntarDir, destContainerDir,
               StandardCopyOption.ATOMIC_MOVE,
@@ -131,6 +133,11 @@ public void pack(Container<KeyValueContainerData> container,
       includeFile(container.getContainerFile(), CONTAINER_FILE_NAME,
           archiveOutput);
 
+      File containerChecksumFile = ContainerChecksumTreeManager.getContainerChecksumFile(containerData);
+      if (containerChecksumFile.exists()) {
+        includeFile(containerChecksumFile, containerChecksumFile.getName(), archiveOutput);
+      }
+
       includePath(getDbPath(containerData), DB_DIR_NAME,
           archiveOutput);
 
@@ -200,7 +207,7 @@ OutputStream compress(OutputStream output) throws IOException {
     return compression.wrap(output);
   }
 
-  private byte[] innerUnpack(InputStream input, Path dbRoot, Path chunksRoot)
+  private byte[] innerUnpack(InputStream input, Path dbRoot, Path chunksRoot, Path tempContainerMetadataPath)
       throws IOException {
     byte[] descriptorFileContent = null;
     try (ArchiveInputStream<TarArchiveEntry> archiveInput = untar(decompress(input))) {
@@ -218,6 +225,10 @@ private byte[] innerUnpack(InputStream input, Path dbRoot, Path chunksRoot)
               .resolve(name.substring(CHUNKS_DIR_NAME.length() + 1));
           extractEntry(entry, archiveInput, size, chunksRoot,
               destinationPath);
+        } else if (name.endsWith(CONTAINER_DATA_CHECKSUM_EXTENSION)) {
+          Path destinationPath = tempContainerMetadataPath.resolve(name);
+          extractEntry(entry, archiveInput, size, tempContainerMetadataPath,
+              destinationPath);
         } else if (CONTAINER_FILE_NAME.equals(name)) {
           //Don't do anything. Container file should be unpacked in a
           //separated step by unpackContainerDescriptor call.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java

@@ -278,7 +278,8 @@ public static void parseKVContainerData(KeyValueContainerData kvContainerData,
     }
   }
 
-  private static void populateContainerDataChecksum(KeyValueContainerData kvContainerData) {
+  private static void populateContainerDataChecksum(KeyValueContainerData kvContainerData,
+      Table<String, Long> metadataTable) {
     if (kvContainerData.isOpen()) {
       return;
     }
@@ -289,6 +290,8 @@ private static void populateContainerDataChecksum(KeyValueContainerData kvContai
       if (optionalContainerChecksumInfo.isPresent()) {
         ContainerChecksumInfo containerChecksumInfo = optionalContainerChecksumInfo.get();
         kvContainerData.setDataChecksum(containerChecksumInfo.getContainerMerkleTree().getDataChecksum());
+        metadataTable.put(kvContainerData.getContainerDataChecksumKey(),
+            containerChecksumInfo.getContainerMerkleTree().getDataChecksum());
       }
     } catch (IOException ex) {
       LOG.warn("Failed to read checksum info for container {}", kvContainerData.getContainerID(), ex);
@@ -367,14 +370,23 @@ private static void populateContainerMetadata(
       kvContainerData.markAsEmpty();
     }
 
+    // Set container data checksum.
+    Long containerDataChecksum = metadataTable.get(
+        kvContainerData.getContainerDataChecksumKey());
+
+    if (containerDataChecksum != null) {
+      kvContainerData.setDataChecksum(containerDataChecksum);
+    } else if (ContainerChecksumTreeManager.hasContainerChecksumFile(kvContainerData)) {
+      populateContainerDataChecksum(kvContainerData, metadataTable);
+    }
+
     // Run advanced container inspection/repair operations if specified on
     // startup. If this method is called but not as a part of startup,
     // The inspectors will be unloaded and this will be a no-op.
     ContainerInspectorUtil.process(kvContainerData, store);
 
     // Load finalizeBlockLocalIds for container in memory.
     populateContainerFinalizeBlock(kvContainerData, store);
-    populateContainerDataChecksum(kvContainerData);
   }
 
   /**

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerChecksumTreeManager.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.container.checksum;
 
+import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DATA_CHECKSUM_EXTENSION;
 import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.assertContainerDiffMatch;
 import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.assertTreesSortedAndMatch;
 import static org.apache.hadoop.ozone.container.checksum.ContainerMerkleTreeTestUtils.buildTestTree;
@@ -97,7 +98,7 @@ public void init() {
     container = mock(KeyValueContainerData.class);
     when(container.getContainerID()).thenReturn(CONTAINER_ID);
     when(container.getMetadataPath()).thenReturn(testDir.getAbsolutePath());
-    checksumFile = new File(testDir, CONTAINER_ID + ".tree");
+    checksumFile = new File(testDir, CONTAINER_ID + CONTAINER_DATA_CHECKSUM_EXTENSION);
     checksumManager = new ContainerChecksumTreeManager(new OzoneConfiguration());
     metrics = checksumManager.getMetrics();
     config = new OzoneConfiguration();

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java

@@ -350,13 +350,18 @@ public ContainerProtos.GetContainerChecksumInfoResponseProto getChecksumInfo(lon
      */
     public long checkAndGetDataChecksum(long containerID) {
       KeyValueContainer container = getContainer(containerID);
+      KeyValueContainerData containerData = container.getContainerData();
       long dataChecksum = 0;
       try {
         Optional<ContainerProtos.ContainerChecksumInfo> containerChecksumInfo =
-            handler.getChecksumManager().read(container.getContainerData());
+            handler.getChecksumManager().read(containerData);
         assertTrue(containerChecksumInfo.isPresent());
         dataChecksum = containerChecksumInfo.get().getContainerMerkleTree().getDataChecksum();
-        assertEquals(container.getContainerData().getDataChecksum(), dataChecksum);
+        assertEquals(containerData.getDataChecksum(), dataChecksum);
+        try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
+          Long dbDataChecksum = dbHandle.getStore().getMetadataTable().get(containerData.getContainerDataChecksumKey());
+          assertEquals(containerData.getDataChecksum(), dbDataChecksum, "DB should have the updated data checksum.");
+        }
       } catch (IOException ex) {
         fail("Failed to read container checksum from disk", ex);
       }

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java

@@ -87,6 +87,7 @@
 import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
 import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
 import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.DBHandle;
 import org.apache.hadoop.ozone.container.common.interfaces.Handler;
 import org.apache.hadoop.ozone.container.common.report.IncrementalReportSender;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
@@ -96,6 +97,7 @@
 import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
 import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
 import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
 import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
 import org.apache.hadoop.ozone.container.ozoneimpl.ContainerScannerConfiguration;
 import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
@@ -679,6 +681,10 @@ public void testUpdateContainerChecksum(ContainerLayoutVersion layoutVersion) th
 
     // Initially, container should have no checksum information.
     assertEquals(0, containerData.getDataChecksum());
+    try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
+      Long dbDataChecksum = dbHandle.getStore().getMetadataTable().get(containerData.getContainerDataChecksumKey());
+      assertEquals(0, dbDataChecksum, "DB should have 0 checksum.");
+    }
     assertFalse(checksumManager.read(containerData).isPresent());
     assertEquals(0, icrCount.get());
 
@@ -688,6 +694,10 @@ public void testUpdateContainerChecksum(ContainerLayoutVersion layoutVersion) th
     assertEquals(1, icrCount.get());
     // Check checksum in memory.
     assertEquals(updatedDataChecksum, containerData.getDataChecksum());
+    try (DBHandle dbHandle = BlockUtils.getDB(containerData, conf)) {
+      Long dbDataChecksum = dbHandle.getStore().getMetadataTable().get(containerData.getContainerDataChecksumKey());
+      assertEquals(updatedDataChecksum, dbDataChecksum, "DB should have the updated data checksum.");
+    }
     // Check disk content.
     ContainerProtos.ContainerChecksumInfo checksumInfo = checksumManager.read(containerData).get();
     assertTreesSortedAndMatch(treeWriter.toProto(), checksumInfo.getContainerMerkleTree());