HDDS-13479. Support Tracking of Deleted Files and Directories in Recon NSSummary.

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java

@@ -641,10 +641,20 @@ protected long fetchSizeForDeletedDirectory(long objectId)
     if (nsSummary == null) {
       return 0L;
     }
-    long totalSize = nsSummary.getSizeOfFiles();
+
+    // Include both active files and deleted files that haven't been physically deleted yet
+    long totalSize = nsSummary.getSizeOfFiles() + nsSummary.getSizeOfDeletedFiles();
+
+    // Add size from active child directories
     for (long childId : nsSummary.getChildDir()) {
       totalSize += fetchSizeForDeletedDirectory(childId);
     }
+
+    // Add size from deleted child directories
+    for (long deletedChildId : nsSummary.getDeletedChildDir()) {
+      totalSize += fetchSizeForDeletedDirectory(deletedChildId);
+    }
+
     return totalSize;
   }
 

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java

@@ -31,26 +31,41 @@
 public class NSSummary {
   private int numOfFiles;
   private long sizeOfFiles;
+  private int numOfDeletedFiles;  // New field
+  private long sizeOfDeletedFiles; // New field
+  private int numOfDeletedDirs;   // New field
+  private long sizeOfDeletedDirs; // New field
   private int[] fileSizeBucket;
   private Set<Long> childDir;
+  private Set<Long> deletedChildDir; // New field to track deleted child directories
   private String dirName;
   private long parentId = 0;
 
   public NSSummary() {
-    this(0, 0L, new int[ReconConstants.NUM_OF_FILE_SIZE_BINS],
-        new HashSet<>(), "", 0);
+    this(0, 0L, 0, 0L, 0, 0L, new int[ReconConstants.NUM_OF_FILE_SIZE_BINS],
+        new HashSet<>(), new HashSet<>(), "", 0);
   }
 
   public NSSummary(int numOfFiles,
                    long sizeOfFiles,
+                   int numOfDeletedFiles,
+                   long sizeOfDeletedFiles,
+                   int numOfDeletedDirs,
+                   long sizeOfDeletedDirs,
                    int[] bucket,
                    Set<Long> childDir,
+                   Set<Long> deletedChildDir,
                    String dirName,
                    long parentId) {
     this.numOfFiles = numOfFiles;
     this.sizeOfFiles = sizeOfFiles;
+    this.numOfDeletedFiles = numOfDeletedFiles;
+    this.sizeOfDeletedFiles = sizeOfDeletedFiles;
+    this.numOfDeletedDirs = numOfDeletedDirs;
+    this.sizeOfDeletedDirs = sizeOfDeletedDirs;
     setFileSizeBucket(bucket);
     this.childDir = childDir;
+    this.deletedChildDir = deletedChildDir;
     this.dirName = dirName;
     this.parentId = parentId;
   }
@@ -109,6 +124,59 @@ public void removeChildDir(long childId) {
     }
   }
 
+  public int getNumOfDeletedFiles() {
+    return numOfDeletedFiles;
+  }
+
+  public void setNumOfDeletedFiles(int numOfDeletedFiles) {
+    this.numOfDeletedFiles = numOfDeletedFiles;
+  }
+
+  public long getSizeOfDeletedFiles() {
+    return sizeOfDeletedFiles;
+  }
+
+  public void setSizeOfDeletedFiles(long sizeOfDeletedFiles) {
+    this.sizeOfDeletedFiles = sizeOfDeletedFiles;
+  }
+
+  public int getNumOfDeletedDirs() {
+    return numOfDeletedDirs;
+  }
+
+  public void setNumOfDeletedDirs(int numOfDeletedDirs) {
+    this.numOfDeletedDirs = numOfDeletedDirs;
+  }
+
+  public long getSizeOfDeletedDirs() {
+    return sizeOfDeletedDirs;
+  }
+
+  public void setSizeOfDeletedDirs(long sizeOfDeletedDirs) {
+    this.sizeOfDeletedDirs = sizeOfDeletedDirs;
+  }
+
+  public Set<Long> getDeletedChildDir() {
+    return deletedChildDir;
+  }
+
+  public void setDeletedChildDir(Set<Long> deletedChildDir) {
+    this.deletedChildDir = deletedChildDir;
+  }
+
+  public void addDeletedChildDir(long childId) {
+    if (this.deletedChildDir.contains(childId)) {
+      return;
+    }
+    this.deletedChildDir.add(childId);
+  }
+
+  public void removeDeletedChildDir(long childId) {
+    if (this.deletedChildDir.contains(childId)) {
+      this.deletedChildDir.remove(childId);
+    }
+  }
+
   public long getParentId() {
     return parentId;
   }
@@ -122,8 +190,13 @@ public String toString() {
     return "NSSummary{dirName='" + dirName + '\'' +
         ", parentId=" + parentId +
         ", childDir=" + childDir +
+        ", deletedChildDir=" + deletedChildDir +
         ", numOfFiles=" + numOfFiles +
         ", sizeOfFiles=" + sizeOfFiles +
+        ", numOfDeletedFiles=" + numOfDeletedFiles +
+        ", sizeOfDeletedFiles=" + sizeOfDeletedFiles +
+        ", numOfDeletedDirs=" + numOfDeletedDirs +
+        ", sizeOfDeletedDirs=" + sizeOfDeletedDirs +
         ", fileSizeBucket=" + Arrays.toString(fileSizeBucket) +
         '}';
   }

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java

@@ -60,18 +60,23 @@ public Class<NSSummary> getTypeClass() {
   public byte[] toPersistedFormatImpl(NSSummary object) throws IOException {
     final byte[] dirName = stringCodec.toPersistedFormat(object.getDirName());
     Set<Long> childDirs = object.getChildDir();
+    Set<Long> deletedChildDirs = object.getDeletedChildDir();
     int numOfChildDirs = childDirs.size();
+    int numOfDeletedChildDirs = deletedChildDirs.size();
 
-    // int: 1 field (numOfFiles) + 2 sizes (childDirs, dirName) + NUM_OF_FILE_SIZE_BINS (fileSizeBucket)
-    final int resSize = (3 + ReconConstants.NUM_OF_FILE_SIZE_BINS) * Integer.BYTES
-        + (numOfChildDirs + 1) * Long.BYTES // 1 long field for parentId + list size
-        + Short.BYTES // 2 dummy shorts to track length
-        + dirName.length // directory name length
-        + Long.BYTES; // Added space for parentId serialization
+    // Calculate size: existing fields + new deleted fields
+    final int resSize = (6 + ReconConstants.NUM_OF_FILE_SIZE_BINS) * Integer.BYTES
+        + (numOfChildDirs + numOfDeletedChildDirs + 4) * Long.BYTES // sizes, parentId, and size fields
+        + Short.BYTES
+        + dirName.length;
 
     ByteArrayOutputStream out = new ByteArrayOutputStream(resSize);
     out.write(integerCodec.toPersistedFormat(object.getNumOfFiles()));
     out.write(longCodec.toPersistedFormat(object.getSizeOfFiles()));
+    out.write(integerCodec.toPersistedFormat(object.getNumOfDeletedFiles()));
+    out.write(longCodec.toPersistedFormat(object.getSizeOfDeletedFiles()));
+    out.write(integerCodec.toPersistedFormat(object.getNumOfDeletedDirs()));
+    out.write(longCodec.toPersistedFormat(object.getSizeOfDeletedDirs()));
     out.write(shortCodec.toPersistedFormat(
         (short) ReconConstants.NUM_OF_FILE_SIZE_BINS));
     int[] fileSizeBucket = object.getFileSizeBucket();
@@ -82,6 +87,10 @@ public byte[] toPersistedFormatImpl(NSSummary object) throws IOException {
     for (long childDirId : childDirs) {
       out.write(longCodec.toPersistedFormat(childDirId));
     }
+    out.write(integerCodec.toPersistedFormat(numOfDeletedChildDirs));
+    for (long deletedChildDirId : deletedChildDirs) {
+      out.write(longCodec.toPersistedFormat(deletedChildDirId));
+    }
     out.write(integerCodec.toPersistedFormat(dirName.length));
     out.write(dirName);
     out.write(longCodec.toPersistedFormat(object.getParentId()));
@@ -95,6 +104,25 @@ public NSSummary fromPersistedFormatImpl(byte[] rawData) throws IOException {
     NSSummary res = new NSSummary();
     res.setNumOfFiles(in.readInt());
     res.setSizeOfFiles(in.readLong());
+
+    // Try to read new fields, if they exist
+    try {
+      res.setNumOfDeletedFiles(in.readInt());
+      res.setSizeOfDeletedFiles(in.readLong());
+      res.setNumOfDeletedDirs(in.readInt());
+      res.setSizeOfDeletedDirs(in.readLong());
+    } catch (IOException e) {
+      // If reading fails, these are old format records
+      res.setNumOfDeletedFiles(0);
+      res.setSizeOfDeletedFiles(0);
+      res.setNumOfDeletedDirs(0);
+      res.setSizeOfDeletedDirs(0);
+      // Reset stream position
+      in = new DataInputStream(new ByteArrayInputStream(rawData));
+      in.readInt(); // numOfFiles
+      in.readLong(); // sizeOfFiles
+    }
+
     short len = in.readShort();
     assert (len == (short) ReconConstants.NUM_OF_FILE_SIZE_BINS);
     int[] fileSizeBucket = new int[len];
@@ -110,6 +138,19 @@ public NSSummary fromPersistedFormatImpl(byte[] rawData) throws IOException {
     }
     res.setChildDir(childDir);
 
+    // Try to read deleted child directories
+    try {
+      int deletedListSize = in.readInt();
+      Set<Long> deletedChildDir = new HashSet<>();
+      for (int i = 0; i < deletedListSize; ++i) {
+        deletedChildDir.add(in.readLong());
+      }
+      res.setDeletedChildDir(deletedChildDir);
+    } catch (IOException e) {
+      // If reading fails, this is old format
+      res.setDeletedChildDir(new HashSet<>());
+    }
+
     int strLen = in.readInt();
     if (strLen == 0) {
       return res;
@@ -136,8 +177,13 @@ public NSSummary copyObject(NSSummary object) {
     NSSummary copy = new NSSummary();
     copy.setNumOfFiles(object.getNumOfFiles());
     copy.setSizeOfFiles(object.getSizeOfFiles());
+    copy.setNumOfDeletedFiles(object.getNumOfDeletedFiles());
+    copy.setSizeOfDeletedFiles(object.getSizeOfDeletedFiles());
+    copy.setNumOfDeletedDirs(object.getNumOfDeletedDirs());
+    copy.setSizeOfDeletedDirs(object.getSizeOfDeletedDirs());
     copy.setFileSizeBucket(object.getFileSizeBucket());
-    copy.setChildDir(object.getChildDir());
+    copy.setChildDir(new HashSet<>(object.getChildDir()));
+    copy.setDeletedChildDir(new HashSet<>(object.getDeletedChildDir()));
     copy.setDirName(object.getDirName());
     copy.setParentId(object.getParentId());
     return copy;

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java

@@ -28,6 +28,7 @@
 import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
 
 /**
  * Class for holding all NSSummaryTask methods
@@ -199,4 +200,119 @@ protected boolean flushAndCommitNSToDB(Map<Long, NSSummary> nsSummaryMap) {
     }
     return true;
   }
+
+  /**
+   * Handle PUT event on deleted table - when a file is moved to deleted table
+   */
+  protected void handlePutDeletedKeyEvent(RepeatedOmKeyInfo repeatedOmKeyInfo,
+                                          Map<Long, NSSummary> nsSummaryMap)
+      throws IOException {
+    for (OmKeyInfo keyInfo : repeatedOmKeyInfo.getOmKeyInfoList()) {
+      long parentObjectId = keyInfo.getParentObjectID();
+      NSSummary nsSummary = nsSummaryMap.get(parentObjectId);
+      if (nsSummary == null) {
+        nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
+      }
+      if (nsSummary == null) {
+        LOG.error("The namespace table is not correctly populated for parent ID: {}", parentObjectId);
+        continue;
+      }
+
+      // Increment deleted files count and size
+      nsSummary.setNumOfDeletedFiles(nsSummary.getNumOfDeletedFiles() + 1);
+      nsSummary.setSizeOfDeletedFiles(nsSummary.getSizeOfDeletedFiles() + keyInfo.getDataSize());
+      nsSummaryMap.put(parentObjectId, nsSummary);
+    }
+  }
+
+  /**
+   * Handle DELETE event on deleted table - when a file is physically deleted from disk
+   */
+  protected void handleDeleteDeletedKeyEvent(RepeatedOmKeyInfo repeatedOmKeyInfo,
+                                             Map<Long, NSSummary> nsSummaryMap)
+      throws IOException {
+    for (OmKeyInfo keyInfo : repeatedOmKeyInfo.getOmKeyInfoList()) {
+      long parentObjectId = keyInfo.getParentObjectID();
+      NSSummary nsSummary = nsSummaryMap.get(parentObjectId);
+      if (nsSummary == null) {
+        nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
+      }
+      if (nsSummary == null) {
+        LOG.error("The namespace table is not correctly populated for parent ID: {}", parentObjectId);
+        continue;
+      }
+
+      // Decrement deleted files count and size
+      nsSummary.setNumOfDeletedFiles(Math.max(0, nsSummary.getNumOfDeletedFiles() - 1));
+      nsSummary.setSizeOfDeletedFiles(Math.max(0, nsSummary.getSizeOfDeletedFiles() - keyInfo.getDataSize()));
+      nsSummaryMap.put(parentObjectId, nsSummary);
+    }
+  }
+
+  /**
+   * Handle PUT event on deleted directory table - when a directory is moved to deleted directory table
+   */
+  protected void handlePutDeletedDirEvent(OmKeyInfo deletedDirInfo,
+                                          Map<Long, NSSummary> nsSummaryMap)
+      throws IOException {
+    long parentObjectId = deletedDirInfo.getParentObjectID();
+    long objectId = deletedDirInfo.getObjectID();
+
+    // Add to parent's deleted child directory list
+    NSSummary parentNsSummary = nsSummaryMap.get(parentObjectId);
+    if (parentNsSummary == null) {
+      parentNsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
+    }
+    if (parentNsSummary == null) {
+      LOG.error("The namespace table is not correctly populated for parent ID: {}", parentObjectId);
+      return;
+    }
+
+    parentNsSummary.addDeletedChildDir(objectId);
+    parentNsSummary.setNumOfDeletedDirs(parentNsSummary.getNumOfDeletedDirs() + 1);
+    parentNsSummary.setSizeOfDeletedDirs(parentNsSummary.getSizeOfDeletedDirs() + deletedDirInfo.getDataSize());
+    nsSummaryMap.put(parentObjectId, parentNsSummary);
+
+    // Create or update the deleted directory's own NSSummary
+    NSSummary deletedDirNsSummary = nsSummaryMap.get(objectId);
+    if (deletedDirNsSummary == null) {
+      deletedDirNsSummary = reconNamespaceSummaryManager.getNSSummary(objectId);
+    }
+    if (deletedDirNsSummary == null) {
+      deletedDirNsSummary = new NSSummary();
+    }
+
+    deletedDirNsSummary.setDirName(deletedDirInfo.getFileName());
+    deletedDirNsSummary.setParentId(parentObjectId);
+    nsSummaryMap.put(objectId, deletedDirNsSummary);
+  }
+
+  /**
+   * Handle DELETE event on deleted directory table - when a directory is physically deleted from disk
+   */
+  protected void handleDeleteDeletedDirEvent(OmKeyInfo deletedDirInfo,
+                                             Map<Long, NSSummary> nsSummaryMap)
+      throws IOException {
+    long parentObjectId = deletedDirInfo.getParentObjectID();
+    long objectId = deletedDirInfo.getObjectID();
+
+    // Remove from parent's deleted child directory list
+    NSSummary parentNsSummary = nsSummaryMap.get(parentObjectId);
+    if (parentNsSummary == null) {
+      parentNsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId);
+    }
+    if (parentNsSummary == null) {
+      LOG.error("The namespace table is not correctly populated for parent ID: {}", parentObjectId);
+      return;
+    }
+
+    parentNsSummary.removeDeletedChildDir(objectId);
+    parentNsSummary.setNumOfDeletedDirs(Math.max(0, parentNsSummary.getNumOfDeletedDirs() - 1));
+    parentNsSummary.setSizeOfDeletedDirs(Math.max(0, parentNsSummary.getSizeOfDeletedDirs() - deletedDirInfo.getDataSize()));
+    nsSummaryMap.put(parentObjectId, parentNsSummary);
+
+    // Remove the deleted directory's NSSummary
+    nsSummaryMap.remove(objectId);
+    reconNamespaceSummaryManager.deleteNSSummary(objectId);
+  }
 }