HDDS-12087. TransactionToDNCommitMap too large causes GC to pause for a long time

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java

@@ -128,6 +128,20 @@ public class ScmConfig extends ReconfigurableConfig {
   )
   private Duration blockDeletionInterval = Duration.ofSeconds(60);
 
+  @Config(key = "hdds.scm.block.deletion.txn.dn.commit.map.limit",
+      defaultValue = "5000000",
+      type = ConfigType.INT,
+      tags = { ConfigTag.SCM },
+      description =
+          " This value indicates the size of the transactionToDNsCommitMap after which" +
+              " we will skip one round of scm block deleting interval."
+  )
+  private int transactionToDNsCommitMapLimit = 5000000;
+
+  public int getTransactionToDNsCommitMapLimit() {
+    return transactionToDNsCommitMapLimit;
+  }
+
   public Duration getBlockDeletionInterval() {
     return blockDeletionInterval;
   }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java

@@ -142,4 +142,6 @@ void addTransactions(Map<Long, List<Long>> containerBlocksMap)
    * @param deletedBlocksTXTable delete transaction table
    */
   void reinitialize(Table<Long, DeletedBlocksTransaction> deletedBlocksTXTable);
+
+  int getTransactionToDNsCommitMapSize();
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java

@@ -512,6 +512,11 @@ public void recordTransactionCreated(UUID dnId, long scmCmdId,
         .recordTransactionCreated(dnId, scmCmdId, dnTxSet);
   }
 
+  @Override
+  public int getTransactionToDNsCommitMapSize() {
+    return getSCMDeletedBlockTransactionStatusManager().getTransactionToDNsCommitMapSize();
+  }
+
   @Override
   public void onDatanodeDead(UUID dnId) {
     getSCMDeletedBlockTransactionStatusManager().onDatanodeDead(dnId);

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java

@@ -93,6 +93,7 @@ public class SCMBlockDeletingService extends BackgroundService
   private final long safemodeExitRunDelayMillis;
   private final long deleteBlocksPendingCommandLimit;
   private final Clock clock;
+  private final int transactionToDNsCommitMapLimit;
 
   @SuppressWarnings("parameternumber")
   public SCMBlockDeletingService(DeletedBlockLog deletedBlockLog,
@@ -115,6 +116,7 @@ public SCMBlockDeletingService(DeletedBlockLog deletedBlockLog,
     DatanodeConfiguration dnConf =
         conf.getObject(DatanodeConfiguration.class);
     this.deleteBlocksPendingCommandLimit = dnConf.getBlockDeleteQueueLimit();
+    this.transactionToDNsCommitMapLimit = scmConfig.getTransactionToDNsCommitMapLimit();
     this.clock = clock;
     this.deletedBlockLog = deletedBlockLog;
     this.nodeManager = nodeManager;
@@ -167,6 +169,12 @@ public EmptyTaskResult call() throws Exception {
           final Set<DatanodeDetails> included =
               getDatanodesWithinCommandLimit(datanodes);
           int blockDeletionLimit = getBlockDeleteTXNum();
+          int txnToDNsCommitMapSize = deletedBlockLog.getTransactionToDNsCommitMapSize();
+          if (txnToDNsCommitMapSize >= transactionToDNsCommitMapLimit) {
+            LOG.warn("Skipping block deletion as transactionToDNsCommitMap size = {}, exceeds threshold {}",
+                txnToDNsCommitMapSize, transactionToDNsCommitMapLimit);
+            return EmptyTaskResult.newResult();
+          }
           DatanodeDeletedBlockTransactions transactions =
               deletedBlockLog.getTransactions(blockDeletionLimit, included);
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java

@@ -572,4 +572,8 @@ private boolean isTransactionFailed(DeleteBlockTransactionResult result) {
     }
     return false;
   }
+
+  public int getTransactionToDNsCommitMapSize() {
+    return transactionToDNsCommitMap.size();
+  }
 }