diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index 2c2d16bf14c7..197f0f863e4b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -408,6 +408,8 @@ public long optimizeDirDeletesAndSubmitRequest(long remainNum, } consumedSize += request.getSerializedSize(); purgePathRequestList.add(request); + // reduce remain count for self, sub-files, and sub-directories + remainNum = remainNum - 1; remainNum = remainNum - request.getDeletedSubFilesCount(); remainNum = remainNum - request.getMarkDeletedSubDirsCount(); // Count up the purgeDeletedDir, subDirs and subFiles diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java index ad16c49d5e6b..914042462550 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java @@ -201,6 +201,8 @@ public BackgroundTaskResult call() { } consumedSize += request.getSerializedSize(); purgePathRequestList.add(request); + // reduce remain count for self, sub-files, and sub-directories + remainNum = remainNum - 1; remainNum = remainNum - request.getDeletedSubFilesCount(); remainNum = remainNum - request.getMarkDeletedSubDirsCount(); // Count up the purgeDeletedDir, subDirs and subFiles diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java index 8dcb030d637a..04e8efa7b794 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java @@ -50,6 +50,7 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -161,4 +162,55 @@ public void testDeleteDirectoryCrossingSizeLimit() throws Exception { 500, 60000); assertThat(dirDeletingService.getRunCount().get()).isGreaterThanOrEqualTo(1); } + + @Test + public void testDeleteDirectoryFlatDirsHavingNoChilds() throws Exception { + OzoneConfiguration conf = createConfAndInitValues(); + OmTestManagers omTestManagers + = new OmTestManagers(conf); + KeyManager keyManager = omTestManagers.getKeyManager(); + writeClient = omTestManagers.getWriteClient(); + om = omTestManagers.getOzoneManager(); + + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + om.getMetadataManager(), BucketLayout.FILE_SYSTEM_OPTIMIZED); + String bucketKey = om.getMetadataManager().getBucketKey(volumeName, bucketName); + OmBucketInfo bucketInfo = om.getMetadataManager().getBucketTable().get(bucketKey); + + int dirCreatesCount = OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT * 2 + 100; + long parentId = 1; + OmDirectoryInfo baseDir = new OmDirectoryInfo.Builder().setName("dir_base") + .setCreationTime(Time.now()).setModificationTime(Time.now()) + .setObjectID(parentId).setParentObjectID(bucketInfo.getObjectID()) + .setUpdateID(0).build(); + OMRequestTestUtils.addDirKeyToDirTable(true, baseDir, volumeName, bucketName, + 1L, om.getMetadataManager()); + for (int i = 0; i < dirCreatesCount; ++i) { + OmDirectoryInfo dir1 = new OmDirectoryInfo.Builder().setName("dir" + i) + .setCreationTime(Time.now()).setModificationTime(Time.now()).setParentObjectID(parentId) + .setObjectID(i + 100).setUpdateID(i).build(); + OMRequestTestUtils.addDirKeyToDirTable(true, dir1, volumeName, bucketName, + 1L, om.getMetadataManager()); + } + + DirectoryDeletingService dirDeletingService = keyManager.getDirDeletingService(); + long[] delDirCnt = new long[2]; + delDirCnt[0] = dirDeletingService.getDeletedDirsCount(); + + OmKeyArgs delArgs = new OmKeyArgs.Builder() + .setVolumeName(volumeName).setBucketName(bucketName).setKeyName("dir_base") + .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)) + .setDataSize(0).setRecursive(true).build(); + writeClient.deleteKey(delArgs); + + // check if difference between each run should not cross the directory deletion limit + // and wait till all dir is removed + GenericTestUtils.waitFor(() -> { + delDirCnt[1] = dirDeletingService.getDeletedDirsCount(); + assertTrue(delDirCnt[1] - delDirCnt[0] <= OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT, + "base: " + delDirCnt[0] + ", new: " + delDirCnt[1]); + delDirCnt[0] = delDirCnt[1]; + return dirDeletingService.getDeletedDirsCount() >= dirCreatesCount; + }, 500, 300000); + } }