HDDS-12087. TransactionToDNCommitMap too large causes GC to pause for a long time

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java

@@ -74,6 +74,10 @@ public final class HddsConfigKeys {
   public static final String HDDS_CONTAINER_CLOSE_THRESHOLD =
       "hdds.container.close.threshold";
   public static final float HDDS_CONTAINER_CLOSE_THRESHOLD_DEFAULT = 0.9f;
+
+  public static final String HDDS_SCM_TXN_DN_COMMIT_MAP_SIZE = "hdds.scm.txn.dn.commit.map.size";
+
+  public static final int HDDS_SCM_TXN_DN_COMMIT_MAP_SIZE_DEFAULT = 5000000;
   public static final String HDDS_SCM_SAFEMODE_ENABLED =
       "hdds.scm.safemode.enabled";
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java

@@ -142,4 +142,6 @@ void addTransactions(Map<Long, List<Long>> containerBlocksMap)
    * @param deletedBlocksTXTable delete transaction table
    */
   void reinitialize(Table<Long, DeletedBlocksTransaction> deletedBlocksTXTable);
+
+  int getTransactionToDNsCommitMapSize();
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java

@@ -488,6 +488,11 @@ public void recordTransactionCreated(UUID dnId, long scmCmdId,
         .recordTransactionCreated(dnId, scmCmdId, dnTxSet);
   }
 
+  @Override
+  public int getTransactionToDNsCommitMapSize() {
+    return getSCMDeletedBlockTransactionStatusManager().getTransactionToDNsCommitMapSize();
+  }
+
   @Override
   public void onDatanodeDead(UUID dnId) {
     getSCMDeletedBlockTransactionStatusManager().onDatanodeDead(dnId);

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java

@@ -17,6 +17,8 @@
 
 package org.apache.hadoop.hdds.scm.block;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_TXN_DN_COMMIT_MAP_SIZE;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_TXN_DN_COMMIT_MAP_SIZE_DEFAULT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
 
@@ -93,6 +95,7 @@ public class SCMBlockDeletingService extends BackgroundService
   private final long safemodeExitRunDelayMillis;
   private final long deleteBlocksPendingCommandLimit;
   private final Clock clock;
+  private final int transactionToDNsCommitMapSize;
 
   @SuppressWarnings("parameternumber")
   public SCMBlockDeletingService(DeletedBlockLog deletedBlockLog,
@@ -115,6 +118,8 @@ public SCMBlockDeletingService(DeletedBlockLog deletedBlockLog,
     DatanodeConfiguration dnConf =
         conf.getObject(DatanodeConfiguration.class);
     this.deleteBlocksPendingCommandLimit = dnConf.getBlockDeleteQueueLimit();
+    this.transactionToDNsCommitMapSize =
+        conf.getInt(HDDS_SCM_TXN_DN_COMMIT_MAP_SIZE, HDDS_SCM_TXN_DN_COMMIT_MAP_SIZE_DEFAULT);
     this.clock = clock;
     this.deletedBlockLog = deletedBlockLog;
     this.nodeManager = nodeManager;
@@ -167,6 +172,12 @@ public EmptyTaskResult call() throws Exception {
           final Set<DatanodeDetails> included =
               getDatanodesWithinCommandLimit(datanodes);
           int blockDeletionLimit = getBlockDeleteTXNum();
+          int txnToDNsCommitMapSize = deletedBlockLog.getTransactionToDNsCommitMapSize();
+          if (txnToDNsCommitMapSize >= transactionToDNsCommitMapSize) {
+            LOG.warn("Skipping block deletion as transactionToDNsCommitMap size = {}, exceeds threshold {}",
+                txnToDNsCommitMapSize, transactionToDNsCommitMapSize);
+            return EmptyTaskResult.newResult();
+          }
           DatanodeDeletedBlockTransactions transactions =
               deletedBlockLog.getTransactions(blockDeletionLimit, included);
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java

@@ -572,4 +572,8 @@ private boolean isTransactionFailed(DeleteBlockTransactionResult result) {
     }
     return false;
   }
+
+  public int getTransactionToDNsCommitMapSize() {
+    return transactionToDNsCommitMap.size();
+  }
 }