diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java index c17d5d52d6f7..dbab95efd4bd 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java @@ -128,6 +128,20 @@ public class ScmConfig extends ReconfigurableConfig { ) private Duration blockDeletionInterval = Duration.ofSeconds(60); + @Config(key = "hdds.scm.block.deletion.txn.dn.commit.map.limit", + defaultValue = "5000000", + type = ConfigType.INT, + tags = { ConfigTag.SCM }, + description = + " This value indicates the size of the transactionToDNsCommitMap after which" + + " we will skip one round of scm block deleting interval." + ) + private int transactionToDNsCommitMapLimit = 5000000; + + public int getTransactionToDNsCommitMapLimit() { + return transactionToDNsCommitMapLimit; + } + public Duration getBlockDeletionInterval() { return blockDeletionInterval; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java index 1952bf7f2fb2..1ac97dae3bcd 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java @@ -142,4 +142,6 @@ void addTransactions(Map> containerBlocksMap) * @param deletedBlocksTXTable delete transaction table */ void reinitialize(Table deletedBlocksTXTable); + + int getTransactionToDNsCommitMapSize(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java index f6fd621393fb..0fa5a09fe2d7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java @@ -512,6 +512,11 @@ public void recordTransactionCreated(UUID dnId, long scmCmdId, .recordTransactionCreated(dnId, scmCmdId, dnTxSet); } + @Override + public int getTransactionToDNsCommitMapSize() { + return getSCMDeletedBlockTransactionStatusManager().getTransactionToDNsCommitMapSize(); + } + @Override public void onDatanodeDead(UUID dnId) { getSCMDeletedBlockTransactionStatusManager().onDatanodeDead(dnId); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java index 31c3eea14b67..6d80dbf0b52d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMBlockDeletingService.java @@ -93,6 +93,7 @@ public class SCMBlockDeletingService extends BackgroundService private final long safemodeExitRunDelayMillis; private final long deleteBlocksPendingCommandLimit; private final Clock clock; + private final int transactionToDNsCommitMapLimit; @SuppressWarnings("parameternumber") public SCMBlockDeletingService(DeletedBlockLog deletedBlockLog, @@ -115,6 +116,7 @@ public SCMBlockDeletingService(DeletedBlockLog deletedBlockLog, DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class); this.deleteBlocksPendingCommandLimit = dnConf.getBlockDeleteQueueLimit(); + this.transactionToDNsCommitMapLimit = scmConfig.getTransactionToDNsCommitMapLimit(); this.clock = clock; this.deletedBlockLog = deletedBlockLog; this.nodeManager = nodeManager; @@ -167,6 +169,12 @@ public EmptyTaskResult call() throws Exception { final Set included = getDatanodesWithinCommandLimit(datanodes); int blockDeletionLimit = getBlockDeleteTXNum(); + int txnToDNsCommitMapSize = deletedBlockLog.getTransactionToDNsCommitMapSize(); + if (txnToDNsCommitMapSize >= transactionToDNsCommitMapLimit) { + LOG.warn("Skipping block deletion as transactionToDNsCommitMap size = {}, exceeds threshold {}", + txnToDNsCommitMapSize, transactionToDNsCommitMapLimit); + return EmptyTaskResult.newResult(); + } DatanodeDeletedBlockTransactions transactions = deletedBlockLog.getTransactions(blockDeletionLimit, included); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java index 4a4aa3b607fc..4be3a1a57461 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java @@ -572,4 +572,8 @@ private boolean isTransactionFailed(DeleteBlockTransactionResult result) { } return false; } + + public int getTransactionToDNsCommitMapSize() { + return transactionToDNsCommitMap.size(); + } }