HDDS-11244. Draft PR to address multiple issues in garbage collection. Will split it into multiple PRs

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java

@@ -112,6 +112,9 @@ public enum ResultCodes {
     FAILED_TO_FIND_CONTAINER,
     FAILED_TO_FIND_CONTAINER_WITH_SPACE,
     BLOCK_EXISTS,
+    //OM can send a deleteBlockRequest for key delete more than once. Thus, it shouldn't
+    // fail the request since SCM already deleted the block in the previous run. We should never use this enum.
+    @Deprecated
     FAILED_TO_FIND_BLOCK,
     IO_EXCEPTION,
     UNEXPECTED_CONTAINER_STATE,

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckedExceptionOperation.java

@@ -0,0 +1,17 @@
+package org.apache.hadoop.ozone.util;
+
+/**
+ *
+ * Represents a function that accepts one argument and produces a result.
+ * This is a functional interface whose functional method is apply(Object).
+ * Type parameters:
+ * <T> – the type of the input to the function <R> – the type of the result of the function
+ * <E> - the type of exception thrown.
+ */
+public interface CheckedExceptionOperation<T, R, E extends Exception> {
+  R apply(T t) throws E;
+
+  default <V> CheckedExceptionOperation<T, V, E> andThen(CheckedExceptionOperation<R, V, E> operation) throws E {
+    return (T t) -> operation.apply(this.apply(t));
+  }
+}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java

@@ -282,18 +282,11 @@ public List<DeleteBlockGroupResult> deleteKeyBlocks(
       e = ioe;
       perfMetrics.updateDeleteKeyFailureStats(startNanos);
       LOG.warn("Fail to delete {} keys", keyBlocksInfoList.size(), ioe);
-      switch (ioe instanceof SCMException ? ((SCMException) ioe).getResult() :
-          IO_EXCEPTION) {
-      case SAFE_MODE_EXCEPTION:
+      if ((ioe instanceof SCMException ? ((SCMException) ioe).getResult() :
+          IO_EXCEPTION) == SCMException.ResultCodes.SAFE_MODE_EXCEPTION) {
         resultCode =
             ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result.safeMode;
-        break;
-      case FAILED_TO_FIND_BLOCK:
-        resultCode =
-            ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result.
-                errorNotFound;
-        break;
-      default:
+      } else {
         resultCode =
             ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result.
                 unknownFailure;

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java

@@ -404,7 +404,7 @@ private OMConfigKeys() {
    * Configuration properties for Snapshot Directory Service.
    */
   public static final String OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED = "ozone.snapshot.deep.cleaning.enabled";
-  public static final boolean OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED_DEFAULT = false;
+  public static final boolean OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED_DEFAULT = true;
   public static final String OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL =
       "ozone.snapshot.directory.service.interval";
   public static final String OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL_DEFAULT

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLock.java

@@ -457,7 +457,8 @@ public enum Resource {
     S3_SECRET_LOCK((byte) 4, "S3_SECRET_LOCK"), // 31
     KEY_PATH_LOCK((byte) 5, "KEY_PATH_LOCK"), //63
     PREFIX_LOCK((byte) 6, "PREFIX_LOCK"), //127
-    SNAPSHOT_LOCK((byte) 7, "SNAPSHOT_LOCK"); // = 255
+    SNAPSHOT_LOCK((byte) 7, "SNAPSHOT_LOCK"), // = 255
+    SNAPSHOT_GC_LOCK((byte) 8, "SNAPSHOT_GC_LOCK");
 
     // level of the resource
     private byte lockLevel;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java

@@ -140,7 +140,7 @@ public void testKeysPurgingByKeyDeletingService() throws Exception {
     GenericTestUtils.waitFor(
         () -> {
           try {
-            return keyManager.getPendingDeletionKeys(Integer.MAX_VALUE)
+            return keyManager.getPendingDeletionKeys(null, Integer.MAX_VALUE)
                 .getKeyBlocksList().size() == 0;
           } catch (IOException e) {
             return false;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java

@@ -632,9 +632,9 @@ public void testParallelExcecutionOfKeyDeletionAndSnapshotDeletion() throws Exce
     try (ReferenceCounted<OmSnapshot> snapshot = om.getOmSnapshotManager().getSnapshot(testBucket.getVolumeName(),
         testBucket.getName(), testBucket.getName() + "snap2")) {
       renamesKeyEntries = snapshot.get().getKeyManager().getRenamesKeyEntries(testBucket.getVolumeName(),
-          testBucket.getName(), "", 1000);
+          testBucket.getName(), "", (kv) -> true, 1000);
       deletedKeyEntries = snapshot.get().getKeyManager().getDeletedKeyEntries(testBucket.getVolumeName(),
-          testBucket.getName(), "", 1000);
+          testBucket.getName(), "", (kv) -> true, 1000);
       deletedDirEntries = snapshot.get().getKeyManager().getDeletedDirEntries(testBucket.getVolumeName(),
           testBucket.getName(), 1000);
     }
@@ -669,20 +669,20 @@ public void testParallelExcecutionOfKeyDeletionAndSnapshotDeletion() throws Exce
         testBucket.getName(), testBucket.getName() + "snap2")) {
       Assertions.assertEquals(Collections.emptyList(),
           snapshot.get().getKeyManager().getRenamesKeyEntries(testBucket.getVolumeName(),
-          testBucket.getName(), "", 1000));
+          testBucket.getName(), "", (kv) -> true, 1000));
       Assertions.assertEquals(Collections.emptyList(),
           snapshot.get().getKeyManager().getDeletedKeyEntries(testBucket.getVolumeName(),
-          testBucket.getName(), "", 1000));
+          testBucket.getName(), "", (kv) -> true, 1000));
       Assertions.assertEquals(Collections.emptyList(),
           snapshot.get().getKeyManager().getDeletedDirEntries(testBucket.getVolumeName(),
           testBucket.getName(), 1000));
     }
     List<Table.KeyValue<String, String>> aosRenamesKeyEntries =
         om.getKeyManager().getRenamesKeyEntries(testBucket.getVolumeName(),
-            testBucket.getName(), "", 1000);
+            testBucket.getName(), "", (kv) -> true, 1000);
     List<Table.KeyValue<String, List<OmKeyInfo>>> aosDeletedKeyEntries =
         om.getKeyManager().getDeletedKeyEntries(testBucket.getVolumeName(),
-            testBucket.getName(), "", 1000);
+            testBucket.getName(), "", (kv) -> true, 1000);
     List<Table.KeyValue<String, OmKeyInfo>> aosDeletedDirEntries =
         om.getKeyManager().getDeletedDirEntries(testBucket.getVolumeName(),
             testBucket.getName(), 1000);