HDDS-11603. Reclaimable Filter for Snapshots garbage reclaimation

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/CheckedFunction.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.util;
+
+/**
+ *
+ * Represents a function that accepts one argument and produces a result.
+ * This is a functional interface whose functional method is apply(Object).
+ * Type parameters:
+ * <T> – the type of the input to the function
+ * <R> – the type of the result of the function
+ * <E> - the type of exception thrown.
+ */
+public interface CheckedFunction<T, R, E extends Exception> {
+  R apply(T t) throws E;
+}

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java

@@ -41,6 +41,7 @@
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assumptions.assumeFalse;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.anyInt;
 import static org.mockito.Mockito.anyLong;
@@ -53,6 +54,7 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Sets;
 import jakarta.annotation.Nonnull;
 import java.io.File;
@@ -69,6 +71,7 @@
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.UUID;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.tuple.Pair;
@@ -104,6 +107,7 @@
 import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
 import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
 import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
 import org.apache.hadoop.ozone.OzoneAcl;
@@ -112,6 +116,7 @@
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -140,6 +145,7 @@
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.EnumSource;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
 
@@ -1585,6 +1591,144 @@ void testGetNotExistedPart() throws IOException {
     assertEquals(0, locationList.size());
   }
 
+  private <T> Table<String, T> getMockedTable(Map<String, T> map) throws IOException {
+    Table<String, T> table = mock(Table.class);
+    when(table.get(anyString())).thenAnswer(i -> map.get(i.getArgument(0)));
+    when(table.getIfExist(anyString())).thenAnswer(i -> map.get(i.getArgument(0)));
+    return table;
+  }
+
+  private OmKeyInfo getMockedOmKeyInfo(OmBucketInfo bucketInfo, long parentId, String key, long objectId) {
+    OmKeyInfo omKeyInfo = mock(OmKeyInfo.class);
+    if (bucketInfo.getBucketLayout().isFileSystemOptimized()) {
+      when(omKeyInfo.getFileName()).thenReturn(key);
+    } else {
+      when(omKeyInfo.getParentObjectID()).thenReturn(parentId);
+      when(omKeyInfo.getKeyName()).thenReturn(key);
+    }
+    when(omKeyInfo.getObjectID()).thenReturn(objectId);
+    return omKeyInfo;
+  }
+
+  private OmDirectoryInfo getMockedOmDirInfo(long parentId, String key, long objectId) {
+    OmDirectoryInfo omKeyInfo = mock(OmDirectoryInfo.class);
+    when(omKeyInfo.getName()).thenReturn(key);
+    when(omKeyInfo.getParentObjectID()).thenReturn(parentId);
+    when(omKeyInfo.getParentObjectID()).thenReturn(0L);
+    when(omKeyInfo.getObjectID()).thenReturn(objectId);
+    return omKeyInfo;
+  }
+
+  private String getPath(long volumeId, OmBucketInfo bucketInfo, OmKeyInfo omKeyInfo) {
+    if (bucketInfo.getBucketLayout().isFileSystemOptimized()) {
+      return volumeId + "/" + bucketInfo.getObjectID() + "/" + omKeyInfo.getParentObjectID() + "/" +
+          omKeyInfo.getFileName();
+    } else {
+      return bucketInfo.getVolumeName() + "/" + bucketInfo.getBucketName() + "/" + omKeyInfo.getKeyName();
+    }
+  }
+
+  private String getPath(long volumeId, OmBucketInfo bucketInfo, OmDirectoryInfo omDirInfo) {
+    return volumeId + "/" + bucketInfo.getObjectID() + "/" + omDirInfo.getParentObjectID() + "/" +
+        omDirInfo.getName();
+  }
+
+  private String getRenameKey(String volume, String bucket, long objectId) {
+    return volume + "/" + bucket + "/" + objectId;
+  }
+
+  @ParameterizedTest
+  @EnumSource(value = BucketLayout.class)
+  public void testPreviousSnapshotOzoneKeyInfo(BucketLayout bucketLayout) throws IOException {
+    OMMetadataManager omMetadataManager = mock(OMMetadataManager.class);
+    if (bucketLayout.isFileSystemOptimized()) {
+      when(omMetadataManager.getOzonePathKey(anyLong(), anyLong(), anyLong(), anyString()))
+          .thenAnswer(i -> Arrays.stream(i.getArguments()).map(Object::toString)
+              .collect(Collectors.joining("/")));
+    } else {
+      when(omMetadataManager.getOzoneKey(anyString(), anyString(), anyString()))
+          .thenAnswer(i -> Arrays.stream(i.getArguments()).map(Object::toString)
+              .collect(Collectors.joining("/")));
+    }
+    when(omMetadataManager.getRenameKey(anyString(), anyString(), anyLong())).thenAnswer(
+        i -> getRenameKey(i.getArgument(0), i.getArgument(1), i.getArgument(2)));
+
+    OMMetadataManager previousMetadataManager = mock(OMMetadataManager.class);
+    OzoneConfiguration configuration = new OzoneConfiguration();
+    KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null);
+    KeyManagerImpl prevKM = new KeyManagerImpl(null, null, previousMetadataManager, configuration, null, null, null);
+    long volumeId = 1L;
+    OmBucketInfo bucketInfo = OmBucketInfo.newBuilder().setBucketName(BUCKET_NAME).setVolumeName(VOLUME_NAME)
+        .setObjectID(2L).setBucketLayout(bucketLayout).build();
+    OmKeyInfo prevKey = getMockedOmKeyInfo(bucketInfo, 5, "key", 1);
+    OmKeyInfo prevKey2 = getMockedOmKeyInfo(bucketInfo, 7, "key2", 2);
+    OmKeyInfo currentKey = getMockedOmKeyInfo(bucketInfo, 6, "renamedKey", 1);
+    OmKeyInfo currentKey2 = getMockedOmKeyInfo(bucketInfo, 7, "key2", 2);
+    OmKeyInfo currentKey3 = getMockedOmKeyInfo(bucketInfo, 8, "key3", 3);
+    OmKeyInfo currentKey4 = getMockedOmKeyInfo(bucketInfo, 8, "key4", 4);
+    Table<String, OmKeyInfo> prevKeyTable =
+        getMockedTable(ImmutableMap.of(
+            getPath(volumeId, bucketInfo, prevKey), prevKey,
+            getPath(volumeId, bucketInfo, prevKey2), prevKey2));
+    Table<String, String> renameTable = getMockedTable(
+        ImmutableMap.of(getRenameKey(VOLUME_NAME, BUCKET_NAME, 1), getPath(volumeId, bucketInfo, prevKey),
+            getRenameKey(VOLUME_NAME, BUCKET_NAME, 3), getPath(volumeId, bucketInfo,
+                getMockedOmKeyInfo(bucketInfo, 6, "unknownKey", 9))));
+    when(previousMetadataManager.getKeyTable(eq(bucketLayout))).thenReturn(prevKeyTable);
+    when(omMetadataManager.getSnapshotRenamedTable()).thenReturn(renameTable);
+    assertEquals(prevKey, km.getPreviousSnapshotOzoneKeyInfo(volumeId, bucketInfo, currentKey).apply(prevKM));
+    assertEquals(prevKey2, km.getPreviousSnapshotOzoneKeyInfo(volumeId, bucketInfo, currentKey2).apply(prevKM));
+    assertNull(km.getPreviousSnapshotOzoneKeyInfo(volumeId, bucketInfo, currentKey3).apply(prevKM));
+    assertNull(km.getPreviousSnapshotOzoneKeyInfo(volumeId, bucketInfo, currentKey4).apply(prevKM));
+  }
+
+  @Test
+  public void testPreviousSnapshotOzoneDirInfo() throws IOException {
+    OMMetadataManager omMetadataManager = mock(OMMetadataManager.class);
+    when(omMetadataManager.getOzonePathKey(anyLong(), anyLong(), anyLong(), anyString()))
+        .thenAnswer(i -> Arrays.stream(i.getArguments()).map(Object::toString)
+            .collect(Collectors.joining("/")));
+    when(omMetadataManager.getRenameKey(anyString(), anyString(), anyLong())).thenAnswer(
+        i -> getRenameKey(i.getArgument(0), i.getArgument(1), i.getArgument(2)));
+
+    OMMetadataManager previousMetadataManager = mock(OMMetadataManager.class);
+    OzoneConfiguration configuration = new OzoneConfiguration();
+    KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null);
+    KeyManagerImpl prevKM = new KeyManagerImpl(null, null, previousMetadataManager, configuration, null, null, null);
+    long volumeId = 1L;
+    OmBucketInfo bucketInfo = OmBucketInfo.newBuilder().setBucketName(BUCKET_NAME).setVolumeName(VOLUME_NAME)
+        .setObjectID(2L).setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED).build();
+    OmDirectoryInfo prevKey = getMockedOmDirInfo(5, "key", 1);
+    OmDirectoryInfo prevKey2 = getMockedOmDirInfo(7, "key2", 2);
+    OmKeyInfo currentKey = getMockedOmKeyInfo(bucketInfo, 6, "renamedKey", 1);
+    OmDirectoryInfo currentKeyDir = getMockedOmDirInfo(6, "renamedKey", 1);
+    OmKeyInfo currentKey2 = getMockedOmKeyInfo(bucketInfo, 7, "key2", 2);
+    OmDirectoryInfo currentKeyDir2 = getMockedOmDirInfo(7, "key2", 2);
+    OmKeyInfo currentKey3 = getMockedOmKeyInfo(bucketInfo, 8, "key3", 3);
+    OmDirectoryInfo currentKeyDir3 = getMockedOmDirInfo(8, "key3", 3);
+    OmKeyInfo currentKey4 = getMockedOmKeyInfo(bucketInfo, 8, "key4", 4);
+    OmDirectoryInfo currentKeyDir4 = getMockedOmDirInfo(8, "key4", 4);
+    Table<String, OmDirectoryInfo> prevDirTable =
+        getMockedTable(ImmutableMap.of(
+            getPath(volumeId, bucketInfo, prevKey), prevKey,
+            getPath(volumeId, bucketInfo, prevKey2), prevKey2));
+    Table<String, String> renameTable = getMockedTable(
+        ImmutableMap.of(getRenameKey(VOLUME_NAME, BUCKET_NAME, 1), getPath(volumeId, bucketInfo, prevKey),
+            getRenameKey(VOLUME_NAME, BUCKET_NAME, 3), getPath(volumeId, bucketInfo,
+                getMockedOmKeyInfo(bucketInfo, 6, "unknownKey", 9))));
+    when(previousMetadataManager.getDirectoryTable()).thenReturn(prevDirTable);
+    when(omMetadataManager.getSnapshotRenamedTable()).thenReturn(renameTable);
+    assertEquals(prevKey, km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKey).apply(prevKM));
+    assertEquals(prevKey2, km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKey2).apply(prevKM));
+    assertNull(km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKey3).apply(prevKM));
+    assertNull(km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKey4).apply(prevKM));
+
+    assertEquals(prevKey, km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKeyDir).apply(prevKM));
+    assertEquals(prevKey2, km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKeyDir2).apply(prevKM));
+    assertNull(km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKeyDir3).apply(prevKM));
+    assertNull(km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKeyDir4).apply(prevKM));
+  }
+
   private void initKeyTableForMultipartTest(String keyName, String volume) throws IOException {
     List<OmKeyLocationInfoGroup> locationInfoGroups = new ArrayList<>();
     List<OmKeyLocationInfo> locationInfoList = new ArrayList<>();

hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java

@@ -613,6 +613,11 @@ default String getOpenFileName(long volumeId, long bucketId, long parentObjectId
    */
   String getRenameKey(String volume, String bucket, long objectID);
 
+  /**
+   * Given renameKey, return the volume, bucket and objectID from the key.
+   */
+  String[] splitRenameKey(String renameKey);
+
   /**
    * Returns the DB key name of a multipart upload key in OM metadata store
    * for FSO-enabled buckets.

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java

@@ -31,6 +31,8 @@
 import org.apache.hadoop.ozone.om.fs.OzoneManagerFS;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
@@ -41,6 +43,7 @@
 import org.apache.hadoop.ozone.om.service.SnapshotDeletingService;
 import org.apache.hadoop.ozone.om.service.SnapshotDirectoryCleaningService;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket;
+import org.apache.hadoop.ozone.util.CheckedFunction;
 
 /**
  * Handles key level commands.
@@ -84,7 +87,6 @@ OmKeyInfo lookupKey(OmKeyArgs args, ResolvedBucket bucketLayout,
   OmKeyInfo getKeyInfo(OmKeyArgs args, ResolvedBucket buctket,
       String clientAddress) throws IOException;
 
-
   /**
    * Returns a list of keys represented by {@link OmKeyInfo}
    * in the given bucket.
@@ -135,6 +137,24 @@ List<Table.KeyValue<String, String>> getRenamesKeyEntries(
       String volume, String bucket, String startKey, int size) throws IOException;
 
 
+  /**
+   * Returns the previous snapshot's ozone keyInfo corresponding for the object.
+   */
+  CheckedFunction<KeyManager, OmDirectoryInfo, IOException> getPreviousSnapshotOzoneDirInfo(
+      long volumeId, OmBucketInfo bucketInfo, OmDirectoryInfo directoryInfo) throws IOException;
+
+  /**
+   * Returns the previous snapshot's ozone keyInfo corresponding for the object.
+   */
+  CheckedFunction<KeyManager, OmDirectoryInfo, IOException> getPreviousSnapshotOzoneDirInfo(
+      long volumeId, OmBucketInfo bucketInfo, OmKeyInfo directoryInfo) throws IOException;
+
+  /**
+   * Returns the previous snapshot's ozone keyInfo corresponding for the object.
+   */
+  CheckedFunction<KeyManager, OmKeyInfo, IOException> getPreviousSnapshotOzoneKeyInfo(
+      long volumeId, OmBucketInfo bucketInfo, OmKeyInfo keyInfo) throws IOException;
+
   /**
    * Returns a list deleted entries from the deletedTable.
    *

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java

@@ -171,6 +171,7 @@
 import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
 import org.apache.hadoop.ozone.security.acl.OzoneObj;
 import org.apache.hadoop.ozone.security.acl.RequestContext;
+import org.apache.hadoop.ozone.util.CheckedFunction;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Time;
@@ -774,6 +775,45 @@ public List<Table.KeyValue<String, String>> getRenamesKeyEntries(
     }
   }
 
+  @Override
+  public CheckedFunction<KeyManager, OmDirectoryInfo, IOException> getPreviousSnapshotOzoneDirInfo(
+      long volumeId, OmBucketInfo bucketInfo, OmDirectoryInfo keyInfo) throws IOException {
+    String currentKeyPath = metadataManager.getOzonePathKey(volumeId, bucketInfo.getObjectID(),
+        keyInfo.getParentObjectID(), keyInfo.getName());
+    return getPreviousSnapshotOzonePathInfo(bucketInfo, keyInfo.getObjectID(), currentKeyPath,
+        (km) -> km.getMetadataManager().getDirectoryTable());
+  }
+
+  @Override
+  public CheckedFunction<KeyManager, OmDirectoryInfo, IOException> getPreviousSnapshotOzoneDirInfo(
+      long volumeId, OmBucketInfo bucketInfo, OmKeyInfo keyInfo) throws IOException {
+    String currentKeyPath = metadataManager.getOzonePathKey(volumeId, bucketInfo.getObjectID(),
+        keyInfo.getParentObjectID(), keyInfo.getFileName());
+    return getPreviousSnapshotOzonePathInfo(bucketInfo, keyInfo.getObjectID(), currentKeyPath,
+        (previousSnapshotKM) -> previousSnapshotKM.getMetadataManager().getDirectoryTable());
+  }
+
+  @Override
+  public CheckedFunction<KeyManager, OmKeyInfo, IOException> getPreviousSnapshotOzoneKeyInfo(
+      long volumeId, OmBucketInfo bucketInfo, OmKeyInfo keyInfo) throws IOException {
+    String currentKeyPath = bucketInfo.getBucketLayout().isFileSystemOptimized()
+        ? metadataManager.getOzonePathKey(volumeId, bucketInfo.getObjectID(), keyInfo.getParentObjectID(),
+        keyInfo.getFileName()) : metadataManager.getOzoneKey(bucketInfo.getVolumeName(), bucketInfo.getBucketName(),
+        keyInfo.getKeyName());
+    return getPreviousSnapshotOzonePathInfo(bucketInfo, keyInfo.getObjectID(), currentKeyPath,
+        (previousSnapshotKM) -> previousSnapshotKM.getMetadataManager().getKeyTable(bucketInfo.getBucketLayout()));
+  }
+
+
+  private <T> CheckedFunction<KeyManager, T, IOException> getPreviousSnapshotOzonePathInfo(
+      OmBucketInfo bucketInfo, long objectId, String currentKeyPath,
+      Function<KeyManager, Table<String, T>> table) throws IOException {
+    String renameKey = metadataManager.getRenameKey(bucketInfo.getVolumeName(), bucketInfo.getBucketName(), objectId);
+    String renamedKey = metadataManager.getSnapshotRenamedTable().getIfExist(renameKey);
+    return (previousSnapshotKM) -> table.apply(previousSnapshotKM).get(
+        renamedKey != null ? renamedKey : currentKeyPath);
+  }
+
   @Override
   public List<Table.KeyValue<String, List<OmKeyInfo>>> getDeletedKeyEntries(
       String volume, String bucket, String startKey, int size) throws IOException {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java

@@ -33,8 +33,8 @@
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
-import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame;
 import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotDirExist;
+import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.isBlockLocationInfoSame;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
@@ -1911,6 +1911,12 @@ public String getRenameKey(String volumeName, String bucketName,
     return renameKey.toString();
   }
 
+  @Override
+  public String[] splitRenameKey(String renameKey) {
+    String[] splitVals = renameKey.split(OM_KEY_PREFIX);
+    return new String[]{splitVals[1], splitVals[2], splitVals[3]};
+  }
+
   @Override
   public String getMultipartKey(long volumeId, long bucketId,
                                 long parentID, String fileName,

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java

@@ -19,7 +19,7 @@
 
 import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS;
 import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
-import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame;
+import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.isBlockLocationInfoSame;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.ServiceException;