HDDS-8869. Make DN DeleteBlocksCommandHandler wait for the lock can timeout.

hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml

@@ -65,6 +65,10 @@
     <Class name="org.apache.hadoop.ozone.container.common.statemachine.TestStateContext" />
     <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
   </Match>
+  <Match>
+    <Class name="org.apache.hadoop.ozone.container.common.statemachine.commandhandler.TestDeleteBlocksCommandHandler" />
+    <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
+  </Match>
   <Match>
     <Class name="org.apache.hadoop.ozone.container.common.volume.TestVolumeSet" />
     <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java

@@ -74,6 +74,10 @@ public final class BlockDeletingServiceMetrics {
   @Metric(about = "The total number of Container chosen to be deleted.")
   private MutableGaugeLong totalContainerChosenCount;
 
+  @Metric(about = "The total number of transactions which was failed " +
+      "cause by wait Container lock timeout.")
+  private MutableGaugeLong totalLockTimeoutTransactionCount;
+
   private BlockDeletingServiceMetrics() {
   }
 
@@ -140,6 +144,10 @@ public void setTotalPendingBlockCount(long count) {
     this.totalPendingBlockCount.set(count);
   }
 
+  public void incrTotalLockTimeoutTransactionCount() {
+    totalLockTimeoutTransactionCount.incr();
+  }
+
   public long getSuccessCount() {
     return successCount.value();
   }
@@ -172,6 +180,10 @@ public long getTotalContainerChosenCount() {
     return totalContainerChosenCount.value();
   }
 
+  public long getTotalLockTimeoutTransactionCount() {
+    return totalLockTimeoutTransactionCount.value();
+  }
+
   @Override
   public String toString() {
     StringBuffer buffer = new StringBuffer();
@@ -195,7 +207,9 @@ public String toString() {
         .append("receivedBlockCount = "
             + receivedBlockCount.value()).append("\t")
         .append("markedBlockCount = "
-            + markedBlockCount.value()).append("\t");
+            + markedBlockCount.value()).append("\t")
+        .append("totalLockTimeoutTransactionCount = "
+            + totalLockTimeoutTransactionCount.value()).append("\t");
     return buffer.toString();
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java

@@ -188,6 +188,24 @@ public class DatanodeConfiguration {
   private long recoveringContainerScrubInterval =
       Duration.ofMinutes(10).toMillis();
 
+  /**
+   * Timeout for the thread used to process the delete block command
+   * to wait for the container lock.
+   * It takes about 200ms to open a RocksDB with HDD media.
+   * Set the default value to 100ms, so that after one times retry
+   * after waiting timeout, the hold time spent waiting for the lock
+   * is not greater than the time spent operating RocksDB
+   */
+  @Config(key = "block.delete.command.handle.lock.timeout",
+      defaultValue = "100ms",
+      type = ConfigType.TIME,
+      tags = { DATANODE, ConfigTag.DELETION},
+      description = "Timeout for the thread used to process the delete" +
+          " block command to wait for the container lock."
+  )
+  private long blockDeleteCommandHandleLockTimeoutMs =
+      Duration.ofMillis(100).toMillis();
+
   public Duration getBlockDeletionInterval() {
     return Duration.ofMillis(blockDeletionInterval);
   }
@@ -573,6 +591,10 @@ public int getBlockDeleteQueueLimit() {
     return blockDeleteQueueLimit;
   }
 
+  public long getBlockDeleteCommandHandleLockTimeoutMs() {
+    return blockDeleteCommandHandleLockTimeoutMs;
+  }
+
   public void setBlockDeleteQueueLimit(int queueLimit) {
     this.blockDeleteQueueLimit = queueLimit;
   }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java

@@ -225,8 +225,7 @@ public DatanodeStateMachine(DatanodeDetails datanodeDetails,
     commandDispatcher = CommandDispatcher.newBuilder()
         .addHandler(new CloseContainerCommandHandler())
         .addHandler(new DeleteBlocksCommandHandler(getContainer(),
-            conf, dnConf.getBlockDeleteThreads(),
-            dnConf.getBlockDeleteQueueLimit()))
+            conf, dnConf))
         .addHandler(new ReplicateContainerCommandHandler(conf, supervisor,
             pullReplicatorWithMetrics, pushReplicatorWithMetrics))
         .addHandler(reconstructECContainersCommandHandler)