HDDS-8627. Recon - API for Count of deletePending directories

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java

@@ -57,6 +57,7 @@
 import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE;
 import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_KEY_TABLE;
 import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE;
+import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_DIR_TABLE;
 import static org.apache.hadoop.ozone.recon.ReconConstants.DEFAULT_FETCH_COUNT;
 import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_LIMIT;
 import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_PREVKEY;
@@ -651,6 +652,36 @@ public Response getDeletedDirInfo(
     return Response.ok(deletedDirInsightInfo).build();
   }
 
+  /**
+   * Retrieves the summary of deleted directories.
+   *
+   * This method calculates and returns a summary of deleted directories.
+   * @return The HTTP  response body includes a map with the following entries:
+   * - "totalDeletedDirectories": the total number of deleted directories
+   *
+   * Example response:
+   *   {
+   *    "totalDeletedDirectories": 8,
+   *   }
+   */
+  @GET
+  @Path("/deletePending/dirs/summary")
+  public Response getDeletedDirectorySummary() {
+    Map<String, Long> dirSummary = new HashMap<>();
+    // Create a keys summary for deleted directories
+    createSummaryForDeletedDirectories(dirSummary);
+    return Response.ok(dirSummary).build();
+  }
+
+  private void createSummaryForDeletedDirectories(
+      Map<String, Long> dirSummary) {
+    // Fetch the necessary metrics for deleted directories.
+    Long deletedDirCount = getValueFromId(globalStatsDao.findById(
+        OmTableInsightTask.getTableCountKeyFromTable(DELETED_DIR_TABLE)));
+    // Calculate the total number of deleted directories
+    dirSummary.put("totalDeletedDirectories", deletedDirCount);
+  }
+
   private void updateReplicatedAndUnReplicatedTotal(
       KeyInsightInfoResponse deletedKeyAndDirInsightInfo,
       RepeatedOmKeyInfo repeatedOmKeyInfo) {

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/DeletedKeysInsightHandler.java

@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.tasks;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+/**
+ * Manages records in the Deleted Table, updating counts and sizes of
+ * pending Key Deletions in the backend.
+ */
+public class DeletedKeysInsightHandler implements OmTableHandler {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(DeletedKeysInsightHandler.class);
+
+  /**
+   * Invoked by the process method to add information on those keys that have
+   * been backlogged in the backend for deletion.
+   */
+  @Override
+  public void handlePutEvent(OMDBUpdateEvent<String, Object> event,
+                             String tableName,
+                             HashMap<String, Long> objectCountMap,
+                             HashMap<String, Long> unReplicatedSizeMap,
+                             HashMap<String, Long> replicatedSizeMap) {
+
+    String countKey = getTableCountKeyFromTable(tableName);
+    String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
+    String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
+
+    if (event.getValue() != null) {
+      RepeatedOmKeyInfo repeatedOmKeyInfo =
+          (RepeatedOmKeyInfo) event.getValue();
+      objectCountMap.computeIfPresent(countKey,
+          (k, count) -> count + repeatedOmKeyInfo.getOmKeyInfoList().size());
+      Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
+      unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey,
+          (k, size) -> size + result.getLeft());
+      replicatedSizeMap.computeIfPresent(replicatedSizeKey,
+          (k, size) -> size + result.getRight());
+    } else {
+      LOG.warn("Put event does not have the Key Info for {}.",
+          event.getKey());
+    }
+
+  }
+
+  /**
+   * Invoked by the process method to remove information on those keys that have
+   * been successfully deleted from the backend.
+   */
+  @Override
+  public void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
+                                String tableName,
+                                HashMap<String, Long> objectCountMap,
+                                HashMap<String, Long> unReplicatedSizeMap,
+                                HashMap<String, Long> replicatedSizeMap) {
+
+    String countKey = getTableCountKeyFromTable(tableName);
+    String unReplicatedSizeKey = getUnReplicatedSizeKeyFromTable(tableName);
+    String replicatedSizeKey = getReplicatedSizeKeyFromTable(tableName);
+
+    if (event.getValue() != null) {
+      RepeatedOmKeyInfo repeatedOmKeyInfo =
+          (RepeatedOmKeyInfo) event.getValue();
+      objectCountMap.computeIfPresent(countKey, (k, count) ->
+          count > 0 ? count - repeatedOmKeyInfo.getOmKeyInfoList().size() : 0L);
+      Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
+      unReplicatedSizeMap.computeIfPresent(unReplicatedSizeKey,
+          (k, size) -> size > result.getLeft() ? size - result.getLeft() : 0L);
+      replicatedSizeMap.computeIfPresent(replicatedSizeKey,
+          (k, size) -> size > result.getRight() ? size - result.getRight() :
+              0L);
+    } else {
+      LOG.warn("Delete event does not have the Key Info for {}.",
+          event.getKey());
+    }
+  }
+
+  /**
+   * Invoked by the process method to update the statistics on the keys
+   * pending to be deleted.
+   */
+  @Override
+  public void handleUpdateEvent(OMDBUpdateEvent<String, Object> event,
+                                String tableName,
+                                HashMap<String, Long> objectCountMap,
+                                HashMap<String, Long> unReplicatedSizeMap,
+                                HashMap<String, Long> replicatedSizeMap) {
+    // The size of deleted keys cannot change hence no-op.
+    return;
+  }
+
+  /**
+   * Invoked by the reprocess method to calculate the records count of the
+   * deleted table and the sizes of replicated and unreplicated keys that are
+   * pending deletion in Ozone.
+   */
+  @Override
+  public Triple<Long, Long, Long> getTableSizeAndCount(
+      TableIterator<String, ? extends Table.KeyValue<String, ?>> iterator)
+      throws IOException {
+    long count = 0;
+    long unReplicatedSize = 0;
+    long replicatedSize = 0;
+
+    if (iterator != null) {
+      while (iterator.hasNext()) {
+        Table.KeyValue<String, ?> kv = iterator.next();
+        if (kv != null && kv.getValue() != null) {
+          RepeatedOmKeyInfo repeatedOmKeyInfo = (RepeatedOmKeyInfo) kv
+              .getValue();
+          Pair<Long, Long> result = repeatedOmKeyInfo.getTotalSize();
+          unReplicatedSize += result.getRight();
+          replicatedSize += result.getLeft();
+          count += repeatedOmKeyInfo.getOmKeyInfoList().size();
+        }
+      }
+    }
+    return Triple.of(count, unReplicatedSize, replicatedSize);
+  }
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmTableHandler.java

@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.tasks;
+
+import org.apache.commons.lang3.tuple.Triple;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+/**
+ * Interface for handling PUT, DELETE and UPDATE events for size-related
+ * tables for OM Insights.
+ */
+public interface OmTableHandler {
+
+  /**
+   * Handles a PUT event for size-related tables by updating both the data
+   * sizes and their corresponding record counts in the tables.
+   *
+   * @param event                    The PUT event to be processed.
+   * @param tableName                Table name associated with the event.
+   * @param objectCountMap           A map storing object counts.
+   * @param unReplicatedSizeMap A map storing unReplicated size counts.
+   * @param replicatedSizeMap   A map storing replicated size counts.
+   */
+  void handlePutEvent(OMDBUpdateEvent<String, Object> event,
+                      String tableName,
+                      HashMap<String, Long> objectCountMap,
+                      HashMap<String, Long> unReplicatedSizeMap,
+                      HashMap<String, Long> replicatedSizeMap);
+
+
+  /**
+   * Handles a DELETE event for size-related tables by updating both the data
+   * sizes and their corresponding record counts in the tables.
+   *
+   * @param event                    The DELETE event to be processed.
+   * @param tableName                Table name associated with the event.
+   * @param objectCountMap           A map storing object counts.
+   * @param unReplicatedSizeMap A map storing unReplicated size counts.
+   * @param replicatedSizeMap   A map storing replicated size counts.
+   */
+  void handleDeleteEvent(OMDBUpdateEvent<String, Object> event,
+                         String tableName,
+                         HashMap<String, Long> objectCountMap,
+                         HashMap<String, Long> unReplicatedSizeMap,
+                         HashMap<String, Long> replicatedSizeMap);
+
+
+  /**
+   * Handles an UPDATE event for size-related tables by updating both the data
+   * sizes and their corresponding record counts in the tables.
+   *
+   * @param event                    The UPDATE event to be processed.
+   * @param tableName                Table name associated with the event.
+   * @param objectCountMap           A map storing object counts.
+   * @param unReplicatedSizeMap A map storing unReplicated size counts.
+   * @param replicatedSizeMap   A map storing replicated size counts.
+   */
+  void handleUpdateEvent(OMDBUpdateEvent<String, Object> event,
+                         String tableName,
+                         HashMap<String, Long> objectCountMap,
+                         HashMap<String, Long> unReplicatedSizeMap,
+                         HashMap<String, Long> replicatedSizeMap);
+
+
+  /**
+   * Returns a triple with the total count of records (left), total unreplicated
+   * size (middle), and total replicated size (right) in the given iterator.
+   * Increments count for each record and adds the dataSize if a record's value
+   * is an instance of OmKeyInfo,RepeatedOmKeyInfo.
+   * If the iterator is null, returns (0, 0, 0).
+   *
+   * @param iterator The iterator over the table to be iterated.
+   * @return A Triple with three Long values representing the count,
+   * unReplicated size and replicated size.
+   * @throws IOException If an I/O error occurs during the iterator traversal.
+   */
+  Triple<Long, Long, Long> getTableSizeAndCount(
+      TableIterator<String, ? extends Table.KeyValue<String, ?>> iterator)
+      throws IOException;
+
+
+  /**
+   * Returns the count key for the given table.
+   *
+   * @param tableName The name of the table.
+   * @return The count key for the table.
+   */
+  default String getTableCountKeyFromTable(String tableName) {
+    return tableName + "Count";
+  }
+
+  /**
+   * Returns the replicated size key for the given table.
+   *
+   * @param tableName The name of the table.
+   * @return The replicated size key for the table.
+   */
+  default String getReplicatedSizeKeyFromTable(String tableName) {
+    return tableName + "ReplicatedDataSize";
+  }
+
+  /**
+   * Returns the unreplicated size key for the given table.
+   *
+   * @param tableName The name of the table.
+   * @return The unreplicated size key for the table.
+   */
+  default String getUnReplicatedSizeKeyFromTable(String tableName) {
+    return tableName + "UnReplicatedDataSize";
+  }
+}