diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java index faf2008ec6b5..e83fc64ad4cc 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java @@ -127,7 +127,8 @@ protected void handlePutDirEvent(OmDirectoryInfo directoryInfo, long objectId = directoryInfo.getObjectID(); // write the dir name to the current directory String dirName = directoryInfo.getName(); - // Try to get the NSSummary from our local map that maps NSSummaries to IDs + + // Get or create the directory's NSSummary NSSummary curNSSummary = nsSummaryMap.get(objectId); if (curNSSummary == null) { // If we don't have it in this batch we try to get it from the DB @@ -140,34 +141,32 @@ protected void handlePutDirEvent(OmDirectoryInfo directoryInfo, int existingNumOfFiles = directoryAlreadyExists ? curNSSummary.getNumOfFiles() : 0; long existingReplicatedSizeOfFiles = directoryAlreadyExists ? curNSSummary.getReplicatedSizeOfFiles() : 0; - if (curNSSummary == null) { - // If we don't have it locally and in the DB we create a new instance - // as this is a new ID + if (!directoryAlreadyExists) { curNSSummary = new NSSummary(); } curNSSummary.setDirName(dirName); - // Set the parent directory ID curNSSummary.setParentId(parentObjectId); nsSummaryMap.put(objectId, curNSSummary); - // Write the child dir list to the parent directory - // Try to get the NSSummary from our local map that maps NSSummaries to IDs - NSSummary nsSummary = nsSummaryMap.get(parentObjectId); - if (nsSummary == null) { - // If we don't have it in this batch we try to get it from the DB - nsSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); + // Get or create the parent's NSSummary + NSSummary parentNSSummary = nsSummaryMap.get(parentObjectId); + if (parentNSSummary == null) { + parentNSSummary = reconNamespaceSummaryManager.getNSSummary(parentObjectId); } - if (nsSummary == null) { + if (parentNSSummary == null) { // If we don't have it locally and in the DB we create a new instance // as this is a new ID - nsSummary = new NSSummary(); + parentNSSummary = new NSSummary(); } - nsSummary.addChildDir(objectId); - nsSummaryMap.put(parentObjectId, nsSummary); + + // Add child directory to parent + parentNSSummary.addChildDir(objectId); + nsSummaryMap.put(parentObjectId, parentNSSummary); // If the directory already existed with content, propagate its totals upward + // propagateSizeUpwards will update parent, grandparent, etc. if (directoryAlreadyExists && (existingSizeOfFiles > 0 || existingNumOfFiles > 0)) { - propagateSizeUpwards(parentObjectId, existingSizeOfFiles, + propagateSizeUpwards(objectId, existingSizeOfFiles, existingReplicatedSizeOfFiles, existingNumOfFiles, nsSummaryMap); } } @@ -233,32 +232,25 @@ protected void handleDeleteDirEvent(OmDirectoryInfo directoryInfo, return; } - // If deleted directory exists, decrement its totals from parent and propagate + // Remove the deleted directory ID from parent's childDir set + parentNsSummary.removeChildDir(deletedDirObjectId); + nsSummaryMap.put(parentObjectId, parentNsSummary); + + // If deleted directory exists, propagate its totals upward (as negative deltas) + // propagateSizeUpwards will update parent, grandparent, etc. if (deletedDirSummary != null) { - // Decrement parent's totals by the deleted directory's totals - parentNsSummary.setNumOfFiles(parentNsSummary.getNumOfFiles() - deletedDirSummary.getNumOfFiles()); - parentNsSummary.setSizeOfFiles(parentNsSummary.getSizeOfFiles() - deletedDirSummary.getSizeOfFiles()); - long parentReplSize = parentNsSummary.getReplicatedSizeOfFiles(); long deletedReplSize = deletedDirSummary.getReplicatedSizeOfFiles(); - if (parentReplSize >= 0 && deletedReplSize >= 0) { - parentNsSummary.setReplicatedSizeOfFiles(parentReplSize - deletedReplSize); - } - - // Propagate the decrements upwards to all ancestors if (deletedReplSize < 0) { deletedReplSize = 0; } - propagateSizeUpwards(parentObjectId, -deletedDirSummary.getSizeOfFiles(), + + propagateSizeUpwards(deletedDirObjectId, -deletedDirSummary.getSizeOfFiles(), -deletedReplSize, -deletedDirSummary.getNumOfFiles(), nsSummaryMap); // Set the deleted directory's parentId to 0 (unlink it) deletedDirSummary.setParentId(0); nsSummaryMap.put(deletedDirObjectId, deletedDirSummary); } - - // Remove the deleted directory ID from parent's childDir set - parentNsSummary.removeChildDir(deletedDirObjectId); - nsSummaryMap.put(parentObjectId, parentNsSummary); } protected boolean flushAndCommitNSToDB(Map nsSummaryMap) { diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java index 5978e11ba658..e93f9176c142 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java @@ -234,6 +234,7 @@ public class TestProcess { @BeforeEach public void setUp() throws IOException { + getReconNamespaceSummaryManager().clearNSSummaryTable(); nSSummaryTaskWithFso.reprocessWithFSO(getReconOMMetadataManager()); result = nSSummaryTaskWithFso.processWithFSO(processEventBatch(), 0); } @@ -347,9 +348,9 @@ private OMUpdateEventBatch processEventBatch() throws IOException { public void testProcessUpdateFileSize() throws IOException { NSSummary nsSummaryForBucket1 = getReconNamespaceSummaryManager().getNSSummary(BUCKET_ONE_OBJECT_ID); - // file 1 is gone, so bucket 1 is empty now + assertNotNull(nsSummaryForBucket1); - assertEquals(0, nsSummaryForBucket1.getNumOfFiles()); + assertEquals(1, nsSummaryForBucket1.getNumOfFiles()); Set childDirBucket1 = nsSummaryForBucket1.getChildDir(); // after put dir4, bucket1 now has two child dirs: dir1 and dir4 diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java index fc2425cdc4d2..ecd886f276b6 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithLegacy.java @@ -185,6 +185,7 @@ public class TestProcess { @BeforeEach public void setUp() throws IOException { + getReconNamespaceSummaryManager().clearNSSummaryTable(); nSSummaryTaskWithLegacy.reprocessWithLegacy(getReconOMMetadataManager()); nSSummaryTaskWithLegacy.processWithLegacy(processEventBatch(), 0); @@ -330,7 +331,7 @@ private OMUpdateEventBatch processEventBatch() throws IOException { public void testProcessUpdateFileSize() throws IOException { // file 1 is gone, so bucket 1 is empty now assertNotNull(nsSummaryForBucket1); - assertEquals(6, nsSummaryForBucket1.getNumOfFiles()); + assertEquals(2, nsSummaryForBucket1.getNumOfFiles()); Set childDirBucket1 = nsSummaryForBucket1.getChildDir(); // after put dir4, bucket1 now has two child dirs: dir1 and dir4