HDDS-13432. Accelerating Namespace Usage Calculation in Recon using - Aggregation Approach

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java

@@ -641,11 +641,7 @@ protected long fetchSizeForDeletedDirectory(long objectId)
     if (nsSummary == null) {
       return 0L;
     }
-    long totalSize = nsSummary.getSizeOfFiles();
-    for (long childId : nsSummary.getChildDir()) {
-      totalSize += fetchSizeForDeletedDirectory(childId);
-    }
-    return totalSize;
+    return nsSummary.getSizeOfFiles();
   }
 
   /** This method retrieves set of directories pending for deletion.

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/EntityHandler.java

@@ -244,11 +244,8 @@ protected long getTotalKeyCount(long objectId) throws IOException {
     if (nsSummary == null) {
       return 0L;
     }
-    long totalCnt = nsSummary.getNumOfFiles();
-    for (long childId: nsSummary.getChildDir()) {
-      totalCnt += getTotalKeyCount(childId);
-    }
-    return totalCnt;
+    // With materialized optimization: getNumOfFiles now returns total count (including all subdirectories)
+    return nsSummary.getNumOfFiles();
   }
 
   /**
@@ -263,11 +260,8 @@ protected long getTotalSize(long objectId) throws IOException {
     if (nsSummary == null) {
       return 0L;
     }
-    long totalSize = nsSummary.getSizeOfFiles();
-    for (long childId: nsSummary.getChildDir()) {
-      totalSize += getTotalSize(childId);
-    }
-    return totalSize;
+    // With materialized optimization: getSizeOfFiles now returns total size (including all subdirectories)
+    return nsSummary.getSizeOfFiles();
   }
 
   public static String[] parseRequestPath(String path) {

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java

@@ -26,9 +26,15 @@
 
 /**
  * Class to encapsulate namespace metadata summaries from OM.
+ * 
+ * IMPORTANT: As of the materialized optimization, sizeOfFiles and numOfFiles 
+ * now represent TOTAL values (including this directory and ALL subdirectories)
+ * rather than just direct files in this directory, for O(1) disk usage queries.
  */
 
 public class NSSummary {
+  // IMPORTANT: These fields now contain TOTAL values (this directory + all subdirectories)
+  // for performance optimization, not just direct files in this directory
   private int numOfFiles;
   private long sizeOfFiles;
   private int[] fileSizeBucket;
@@ -55,10 +61,16 @@ public NSSummary(int numOfFiles,
     this.parentId = parentId;
   }
 
+  /**
+   * @return Total number of files in this directory and ALL subdirectories
+   */
   public int getNumOfFiles() {
     return numOfFiles;
   }
 
+  /**
+   * @return Total size of files in this directory and ALL subdirectories
+   */
   public long getSizeOfFiles() {
     return sizeOfFiles;
   }
@@ -75,10 +87,16 @@ public String getDirName() {
     return dirName;
   }
 
+  /**
+   * @param numOfFiles Total number of files in this directory and ALL subdirectories
+   */
   public void setNumOfFiles(int numOfFiles) {
     this.numOfFiles = numOfFiles;
   }
 
+  /**
+   * @param sizeOfFiles Total size of files in this directory and ALL subdirectories
+   */
   public void setSizeOfFiles(long sizeOfFiles) {
     this.sizeOfFiles = sizeOfFiles;
   }

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java

@@ -89,13 +89,18 @@ protected void handlePutKeyEvent(OmKeyInfo keyInfo, Map<Long,
       nsSummary = new NSSummary();
     }
     int[] fileBucket = nsSummary.getFileSizeBucket();
+
+    // Update immediate parent's totals (these fields now represent totals)
     nsSummary.setNumOfFiles(nsSummary.getNumOfFiles() + 1);
     nsSummary.setSizeOfFiles(nsSummary.getSizeOfFiles() + keyInfo.getDataSize());
     int binIndex = ReconUtils.getFileSizeBinIndex(keyInfo.getDataSize());
 
     ++fileBucket[binIndex];
     nsSummary.setFileSizeBucket(fileBucket);
     nsSummaryMap.put(parentObjectId, nsSummary);
+
+    // Propagate upwards to all parents in the parent chain
+    propagateSizeUpwards(parentObjectId, keyInfo.getDataSize(), 1, nsSummaryMap);
   }
 
   protected void handlePutDirEvent(OmDirectoryInfo directoryInfo,
@@ -111,6 +116,12 @@ protected void handlePutDirEvent(OmDirectoryInfo directoryInfo,
       // If we don't have it in this batch we try to get it from the DB
       curNSSummary = reconNamespaceSummaryManager.getNSSummary(objectId);
     }
+
+    // Check if this directory already has content (files/subdirs) that need propagation
+    boolean directoryAlreadyExists = (curNSSummary != null);
+    long existingSizeOfFiles = directoryAlreadyExists ? curNSSummary.getSizeOfFiles() : 0;
+    int existingNumOfFiles = directoryAlreadyExists ? curNSSummary.getNumOfFiles() : 0;
+
     if (curNSSummary == null) {
       // If we don't have it locally and in the DB we create a new instance
       // as this is a new ID
@@ -135,6 +146,11 @@ protected void handlePutDirEvent(OmDirectoryInfo directoryInfo,
     }
     nsSummary.addChildDir(objectId);
     nsSummaryMap.put(parentObjectId, nsSummary);
+
+    // If the directory already existed with content, propagate its totals upward
+    if (directoryAlreadyExists && (existingSizeOfFiles > 0 || existingNumOfFiles > 0)) {
+      propagateSizeUpwards(parentObjectId, existingSizeOfFiles, existingNumOfFiles, nsSummaryMap);
+    }
   }
 
   protected void handleDeleteKeyEvent(OmKeyInfo keyInfo,
@@ -157,35 +173,59 @@ protected void handleDeleteKeyEvent(OmKeyInfo keyInfo,
 
     int binIndex = ReconUtils.getFileSizeBinIndex(keyInfo.getDataSize());
 
-    // decrement count, data size, and bucket count
-    // even if there's no direct key, we still keep the entry because
-    // we still need children dir IDs info
+    // Decrement immediate parent's totals (these fields now represent totals)
     nsSummary.setNumOfFiles(nsSummary.getNumOfFiles() - 1);
     nsSummary.setSizeOfFiles(nsSummary.getSizeOfFiles() - keyInfo.getDataSize());
     --fileBucket[binIndex];
     nsSummary.setFileSizeBucket(fileBucket);
     nsSummaryMap.put(parentObjectId, nsSummary);
+
+    // Propagate upwards to all parents in the parent chain
+    propagateSizeUpwards(parentObjectId, -keyInfo.getDataSize(), -1, nsSummaryMap);
   }
 
   protected void handleDeleteDirEvent(OmDirectoryInfo directoryInfo,
                                       Map<Long, NSSummary> nsSummaryMap)
       throws IOException {
+    long deletedDirObjectId = directoryInfo.getObjectID();
     long parentObjectId = directoryInfo.getParentObjectID();
-    // Try to get the NSSummary from our local map that maps NSSummaries to IDs
-    NSSummary nsSummary = nsSummaryMap.get(parentObjectId);
-    if (nsSummary == null) {
-      // If we don't have it in this batch we try to get it from the DB
-      nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
+
+    // Get the deleted directory's NSSummary to extract its totals
+    NSSummary deletedDirSummary = nsSummaryMap.get(deletedDirObjectId);
+    if (deletedDirSummary == null) {
+      deletedDirSummary = reconNamespaceSummaryManager.getNSSummary(deletedDirObjectId);
+    }
+
+    // Get the parent directory's NSSummary
+    NSSummary parentNsSummary = nsSummaryMap.get(parentObjectId);
+    if (parentNsSummary == null) {
+      parentNsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
     }
 
     // Just in case the OmDirectoryInfo isn't correctly written.
-    if (nsSummary == null) {
+    if (parentNsSummary == null) {
       LOG.error("The namespace table is not correctly populated.");
       return;
     }
 
-    nsSummary.removeChildDir(directoryInfo.getObjectID());
-    nsSummaryMap.put(parentObjectId, nsSummary);
+    // If deleted directory exists, decrement its totals from parent and propagate
+    if (deletedDirSummary != null) {
+      // Decrement parent's totals by the deleted directory's totals
+      parentNsSummary.setNumOfFiles(parentNsSummary.getNumOfFiles() - deletedDirSummary.getNumOfFiles());
+      parentNsSummary.setSizeOfFiles(parentNsSummary.getSizeOfFiles() - deletedDirSummary.getSizeOfFiles());
+
+      // Propagate the decrements upwards to all ancestors
+      propagateSizeUpwards(parentObjectId, -deletedDirSummary.getSizeOfFiles(), 
+                          -deletedDirSummary.getNumOfFiles(), nsSummaryMap);
+
+      // Set the deleted directory's parentId to 0 (unlink it)
+      deletedDirSummary.setParentId(0);
+      nsSummaryMap.put(deletedDirObjectId, deletedDirSummary);
+    }
+
+    // Remove the deleted directory ID from parent's childDir set
+    parentNsSummary.removeChildDir(deletedDirObjectId);
+    nsSummaryMap.put(parentObjectId, parentNsSummary);
   }
 
   protected boolean flushAndCommitNSToDB(Map<Long, NSSummary> nsSummaryMap) {
@@ -199,4 +239,42 @@ protected boolean flushAndCommitNSToDB(Map<Long, NSSummary> nsSummaryMap) {
     }
     return true;
   }
+
+  /**
+   * Propagates size and count changes upwards through the parent chain.
+   * This ensures that when files are added/deleted, all ancestor directories
+   * reflect the total changes in their sizeOfFiles and numOfFiles fields.
+   */
+  protected void propagateSizeUpwards(long objectId, long sizeChange,
+                                       int countChange, Map<Long, NSSummary> nsSummaryMap) 
+                                       throws IOException {
+    // Get the current directory's NSSummary
+    NSSummary nsSummary = nsSummaryMap.get(objectId);
+    if (nsSummary == null) {
+      nsSummary = reconNamespaceSummaryManager.getNSSummary(objectId);
+    }
+    if (nsSummary == null) {
+      return; // No more parents to update
+    }
+
+    // Continue propagating to parent
+    long parentId = nsSummary.getParentId();
+    if (parentId != 0) {
+      // Get parent's NSSummary
+      NSSummary parentSummary = nsSummaryMap.get(parentId);
+      if (parentSummary == null) {
+        parentSummary = reconNamespaceSummaryManager.getNSSummary(parentId);
+      }
+      if (parentSummary != null) {
+        // Update parent's totals
+        parentSummary.setSizeOfFiles(parentSummary.getSizeOfFiles() + sizeChange);
+        parentSummary.setNumOfFiles(parentSummary.getNumOfFiles() + countChange);
+        nsSummaryMap.put(parentId, parentSummary);
+
+        // Recursively propagate to grandparents
+        propagateSizeUpwards(parentId, sizeChange, countChange, nsSummaryMap);
+      }
+    }
+  }
+
 }

hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java

@@ -82,6 +82,8 @@ void setUp(@TempDir File tmpDir) throws Exception {
         threshold);
   }
 
+
+
   /**
    * Nested class for testing NSSummaryTaskWithFSO reprocess.
    */
@@ -109,10 +111,10 @@ public void testReprocessNSSummaryNull() throws IOException {
 
     @Test
     public void testReprocessGetFiles() {
-      assertEquals(1, nsSummaryForBucket1.getNumOfFiles());
+      assertEquals(2, nsSummaryForBucket1.getNumOfFiles());
       assertEquals(2, nsSummaryForBucket2.getNumOfFiles());
 
-      assertEquals(KEY_ONE_SIZE, nsSummaryForBucket1.getSizeOfFiles());
+      assertEquals(KEY_ONE_SIZE + KEY_THREE_SIZE, nsSummaryForBucket1.getSizeOfFiles());
       assertEquals(KEY_TWO_OLD_SIZE + KEY_FOUR_SIZE,
           nsSummaryForBucket2.getSizeOfFiles());
     }

hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java

@@ -93,10 +93,10 @@ public void testReprocessNSSummaryNull() throws IOException {
 
     @Test
     public void testReprocessGetFiles() {
-      assertEquals(1, nsSummaryForBucket1.getNumOfFiles());
+      assertEquals(2, nsSummaryForBucket1.getNumOfFiles());
       assertEquals(2, nsSummaryForBucket2.getNumOfFiles());
 
-      assertEquals(KEY_ONE_SIZE, nsSummaryForBucket1.getSizeOfFiles());
+      assertEquals(KEY_ONE_SIZE + KEY_THREE_SIZE, nsSummaryForBucket1.getSizeOfFiles());
       assertEquals(KEY_TWO_OLD_SIZE + KEY_FOUR_SIZE,
           nsSummaryForBucket2.getSizeOfFiles());
     }
@@ -338,7 +338,7 @@ private OMUpdateEventBatch processEventBatch() throws IOException {
     public void testProcessUpdateFileSize() throws IOException {
       // file 1 is gone, so bucket 1 is empty now
       assertNotNull(nsSummaryForBucket1);
-      assertEquals(0, nsSummaryForBucket1.getNumOfFiles());
+      assertEquals(6, nsSummaryForBucket1.getNumOfFiles());
 
       Set<Long> childDirBucket1 = nsSummaryForBucket1.getChildDir();
       // after put dir4, bucket1 now has two child dirs: dir1 and dir4