HDDS-8869. Make DN DeleteBlocksCommandHandler wait for the lock can timeout.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java

@@ -52,6 +52,10 @@ public final class BlockDeletingServiceMetrics {
   @Metric(about = "The total number of blocks pending for processing.")
   private MutableGaugeLong totalPendingBlockCount;
 
+  @Metric(about = "The total number of transactions witch was failed " +
+      "cause by wait Container lock timeout.")
+  private MutableGaugeLong totalLockTimeoutTransactionCount;
+
   private BlockDeletingServiceMetrics() {
   }
 
@@ -90,6 +94,10 @@ public void setTotalPendingBlockCount(long count) {
     this.totalPendingBlockCount.set(count);
   }
 
+  public void incrTotalLockTimeoutTransactionCount() {
+    totalLockTimeoutTransactionCount.incr();
+  }
+
   public long getSuccessCount() {
     return successCount.value();
   }
@@ -114,6 +122,10 @@ public long getTotalPendingBlockCount() {
     return totalPendingBlockCount.value();
   }
 
+  public long getTotalLockTimeoutTransactionCount() {
+    return totalLockTimeoutTransactionCount.value();
+  }
+
   @Override
   public String toString() {
     StringBuffer buffer = new StringBuffer();
@@ -123,7 +135,9 @@ public String toString() {
         .append("outOfOrderDeleteBlockTransactionCount = "
             + outOfOrderDeleteBlockTransactionCount.value()).append("\t")
         .append("totalPendingBlockCount = "
-            + totalPendingBlockCount.value()).append("\t");
+            + totalPendingBlockCount.value()).append("\t")
+        .append("totalLockTimeoutTransactionCount = "
+            + totalLockTimeoutTransactionCount.value()).append("\t");
     return buffer.toString();
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java

@@ -192,6 +192,24 @@ public class DatanodeConfiguration {
   private long recoveringContainerScrubInterval =
       Duration.ofMinutes(10).toMillis();
 
+  /**
+   * Timeout for the thread used to process the delete block command
+   * to wait for the container lock.
+   * It takes about 200ms to open a RocksDB with HDD media.
+   * Set the default value to 100ms, so that after one times retry
+   * after waiting timeout, the hold time spent waiting for the lock
+   * is not greater than the time spent operating RocksDB
+   */
+  @Config(key = "block.delete.command.handle.lock.timeout",
+      defaultValue = "100ms",
+      type = ConfigType.TIME,
+      tags = { DATANODE, ConfigTag.DELETION},
+      description = "Timeout for the thread used to process the delete" +
+          " block command to wait for the container lock."
+  )
+  private long blockDeleteCommandHandleLockTimeoutMs =
+      Duration.ofMillis(100).toMillis();
+
   public Duration getBlockDeletionInterval() {
     return Duration.ofMillis(blockDeletionInterval);
   }
@@ -559,6 +577,10 @@ public int getBlockDeleteQueueLimit() {
     return blockDeleteQueueLimit;
   }
 
+  public long getBlockDeleteCommandHandleLockTimeoutMs() {
+    return blockDeleteCommandHandleLockTimeoutMs;
+  }
+
   public void setBlockDeleteQueueLimit(int queueLimit) {
     this.blockDeleteQueueLimit = queueLimit;
   }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java

@@ -226,7 +226,8 @@ public DatanodeStateMachine(DatanodeDetails datanodeDetails,
         .addHandler(new CloseContainerCommandHandler())
         .addHandler(new DeleteBlocksCommandHandler(getContainer(),
             conf, dnConf.getBlockDeleteThreads(),
-            dnConf.getBlockDeleteQueueLimit()))
+            dnConf.getBlockDeleteQueueLimit(),
+            dnConf.getBlockDeleteCommandHandleLockTimeoutMs()))
         .addHandler(new ReplicateContainerCommandHandler(conf, supervisor,
             pullReplicatorWithMetrics, pushReplicatorWithMetrics))
         .addHandler(reconstructECContainersCommandHandler)