HDDS-11492. Directory deletion get stuck having millions of directory

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java

@@ -408,6 +408,8 @@ public long optimizeDirDeletesAndSubmitRequest(long remainNum,
         }
         consumedSize += request.getSerializedSize();
         purgePathRequestList.add(request);
+        // reduce remain count for self, sub-files, and sub-directories
+        remainNum = remainNum - 1;
         remainNum = remainNum - request.getDeletedSubFilesCount();
         remainNum = remainNum - request.getMarkDeletedSubDirsCount();
         // Count up the purgeDeletedDir, subDirs and subFiles

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java

@@ -201,6 +201,8 @@ public BackgroundTaskResult call() {
             }
             consumedSize += request.getSerializedSize();
             purgePathRequestList.add(request);
+            // reduce remain count for self, sub-files, and sub-directories
+            remainNum = remainNum - 1;
             remainNum = remainNum - request.getDeletedSubFilesCount();
             remainNum = remainNum - request.getMarkDeletedSubDirsCount();
             // Count up the purgeDeletedDir, subDirs and subFiles

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestDirectoryDeletingService.java

@@ -50,6 +50,7 @@
 
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -161,4 +162,55 @@ public void testDeleteDirectoryCrossingSizeLimit() throws Exception {
         500, 60000);
     assertThat(dirDeletingService.getRunCount().get()).isGreaterThanOrEqualTo(1);
   }
+
+  @Test
+  public void testDeleteDirectoryFlatDirsHavingNoChilds() throws Exception {
+    OzoneConfiguration conf = createConfAndInitValues();
+    OmTestManagers omTestManagers
+        = new OmTestManagers(conf);
+    KeyManager keyManager = omTestManagers.getKeyManager();
+    writeClient = omTestManagers.getWriteClient();
+    om = omTestManagers.getOzoneManager();
+
+    OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+        om.getMetadataManager(), BucketLayout.FILE_SYSTEM_OPTIMIZED);
+    String bucketKey = om.getMetadataManager().getBucketKey(volumeName, bucketName);
+    OmBucketInfo bucketInfo = om.getMetadataManager().getBucketTable().get(bucketKey);
+
+    int dirCreatesCount = OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT * 2 + 100;
+    long parentId = 1;
+    OmDirectoryInfo baseDir = new OmDirectoryInfo.Builder().setName("dir_base")
+        .setCreationTime(Time.now()).setModificationTime(Time.now())
+        .setObjectID(parentId).setParentObjectID(bucketInfo.getObjectID())
+        .setUpdateID(0).build();
+    OMRequestTestUtils.addDirKeyToDirTable(true, baseDir, volumeName, bucketName,
+        1L, om.getMetadataManager());
+    for (int i = 0; i < dirCreatesCount; ++i) {
+      OmDirectoryInfo dir1 = new OmDirectoryInfo.Builder().setName("dir" + i)
+          .setCreationTime(Time.now()).setModificationTime(Time.now()).setParentObjectID(parentId)
+          .setObjectID(i + 100).setUpdateID(i).build();
+      OMRequestTestUtils.addDirKeyToDirTable(true, dir1, volumeName, bucketName,
+          1L, om.getMetadataManager());
+    }
+
+    DirectoryDeletingService dirDeletingService = keyManager.getDirDeletingService();
+    long[] delDirCnt = new long[2];
+    delDirCnt[0] = dirDeletingService.getDeletedDirsCount();
+
+    OmKeyArgs delArgs = new OmKeyArgs.Builder()
+        .setVolumeName(volumeName).setBucketName(bucketName).setKeyName("dir_base")
+        .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE))
+        .setDataSize(0).setRecursive(true).build();
+    writeClient.deleteKey(delArgs);
+
+    // check if difference between each run should not cross the directory deletion limit
+    // and wait till all dir is removed
+    GenericTestUtils.waitFor(() -> {
+      delDirCnt[1] = dirDeletingService.getDeletedDirsCount();
+      assertTrue(delDirCnt[1] - delDirCnt[0] <= OZONE_PATH_DELETING_LIMIT_PER_TASK_DEFAULT,
+          "base: " + delDirCnt[0] + ", new: " + delDirCnt[1]);
+      delDirCnt[0] =  delDirCnt[1];
+      return dirDeletingService.getDeletedDirsCount() >= dirCreatesCount;
+    }, 500, 300000);
+  }
 }