HDDS-8729. Add metrics for count of pending Block on DN

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/BlockDeletingServiceMetrics.java

@@ -23,6 +23,7 @@
 import org.apache.hadoop.metrics2.annotation.Metrics;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
 import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.BlockDeletingService;
 
 /**
@@ -48,6 +49,9 @@ public final class BlockDeletingServiceMetrics {
   @Metric(about = "The number of out of order delete block transaction.")
   private MutableCounterLong outOfOrderDeleteBlockTransactionCount;
 
+  @Metric(about = "The total number of blocks pending for processing.")
+  private MutableGaugeLong totalPendingBlockCount;
+
   private BlockDeletingServiceMetrics() {
   }
 
@@ -82,6 +86,10 @@ public void incrFailureCount() {
     this.failureCount.incr();
   }
 
+  public void setTotalPendingBlockCount(long count) {
+    this.totalPendingBlockCount.set(count);
+  }
+
   public long getSuccessCount() {
     return successCount.value();
   }
@@ -102,14 +110,20 @@ public long getOutOfOrderDeleteBlockTransactionCount() {
     return outOfOrderDeleteBlockTransactionCount.value();
   }
 
+  public long getTotalPendingBlockCount() {
+    return totalPendingBlockCount.value();
+  }
+
   @Override
   public String toString() {
     StringBuffer buffer = new StringBuffer();
     buffer.append("successCount = " + successCount.value()).append("\t")
         .append("successBytes = " + successBytes.value()).append("\t")
         .append("failureCount = " + failureCount.value()).append("\t")
         .append("outOfOrderDeleteBlockTransactionCount = "
-            + outOfOrderDeleteBlockTransactionCount.value()).append("\t");
+            + outOfOrderDeleteBlockTransactionCount.value()).append("\t")
+        .append("totalPendingBlockCount = "
+            + totalPendingBlockCount.value()).append("\t");
     return buffer.toString();
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java

@@ -182,6 +182,13 @@ public List<ContainerBlockInfo> chooseContainerForBlockDeletion(
             .filter(e -> isDeletionAllowed(e.getValue().getContainerData(),
                 deletionPolicy)).collect(Collectors
             .toMap(Map.Entry::getKey, e -> e.getValue().getContainerData()));
+
+    long totalPendingBlockCount =
+        containerDataMap.values().stream().mapToLong(
+            containerData -> ((KeyValueContainerData) containerData)
+            .getNumPendingDeletionBlocks())
+        .sum();
+    metrics.setTotalPendingBlockCount(totalPendingBlockCount);
     return deletionPolicy
         .chooseContainerForBlockDeletion(blockLimit, containerDataMap);
   }

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java

@@ -613,6 +613,11 @@ public void testBlockDeletion() throws Exception {
       Assert.assertEquals(2,
           deletingServiceMetrics.getSuccessCount()
               - deleteSuccessCount);
+      // The value of the getTotalPendingBlockCount Metrics is obtained
+      // before the deletion is processing
+      // So the Pending Block count will be 3
+      Assert.assertEquals(3,
+          deletingServiceMetrics.getTotalPendingBlockCount());
 
       deleteAndWait(svc, 2);
 
@@ -634,8 +639,12 @@ public void testBlockDeletion() throws Exception {
 
       // check if blockData get deleted
       assertBlockDataTableRecordCount(0, meta, filter, data.getContainerID());
+      // The value of the getTotalPendingBlockCount Metrics is obtained
+      // before the deletion is processing
+      // So the Pending Block count will be 1
+      Assert.assertEquals(1,
+          deletingServiceMetrics.getTotalPendingBlockCount());
     }
-
     svc.shutdown();
   }