HDDS-12562. Reclaimable Directory entry filter for reclaiming deleted directory entries

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/InMemoryTestTable.java

@@ -19,6 +19,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -28,7 +29,16 @@
  * InMemory Table implementation for tests.
  */
 public final class InMemoryTestTable<KEY, VALUE> implements Table<KEY, VALUE> {
-  private final Map<KEY, VALUE> map = new ConcurrentHashMap<>();
+  private final Map<KEY, VALUE> map;
+
+  public InMemoryTestTable() {
+    this(Collections.emptyMap());
+  }
+
+  public InMemoryTestTable(Map<KEY, VALUE> map) {
+    this.map = new ConcurrentHashMap<>();
+    this.map.putAll(map);
+  }
 
   @Override
   public void close() {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java

@@ -53,6 +53,7 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Sets;
 import jakarta.annotation.Nonnull;
 import java.io.File;
@@ -69,6 +70,7 @@
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.UUID;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.tuple.Pair;
@@ -104,6 +106,8 @@
 import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
 import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.utils.db.InMemoryTestTable;
+import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
 import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
 import org.apache.hadoop.ozone.OzoneAcl;
@@ -112,6 +116,7 @@
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -1585,6 +1590,91 @@ void testGetNotExistedPart() throws IOException {
     assertEquals(0, locationList.size());
   }
 
+  private OmKeyInfo getMockedOmKeyInfo(OmBucketInfo bucketInfo, long parentId, String key, long objectId) {
+    OmKeyInfo omKeyInfo = mock(OmKeyInfo.class);
+    if (bucketInfo.getBucketLayout().isFileSystemOptimized()) {
+      when(omKeyInfo.getFileName()).thenReturn(key);
+      when(omKeyInfo.getParentObjectID()).thenReturn(parentId);
+    } else {
+      when(omKeyInfo.getKeyName()).thenReturn(key);
+    }
+    when(omKeyInfo.getObjectID()).thenReturn(objectId);
+    return omKeyInfo;
+  }
+
+  private OmDirectoryInfo getMockedOmDirInfo(long parentId, String key, long objectId) {
+    OmDirectoryInfo omKeyInfo = mock(OmDirectoryInfo.class);
+    when(omKeyInfo.getName()).thenReturn(key);
+    when(omKeyInfo.getParentObjectID()).thenReturn(parentId);
+    when(omKeyInfo.getObjectID()).thenReturn(objectId);
+    return omKeyInfo;
+  }
+
+  private String getDirectoryKey(long volumeId, OmBucketInfo bucketInfo, OmKeyInfo omKeyInfo) {
+    if (bucketInfo.getBucketLayout().isFileSystemOptimized()) {
+      return volumeId + "/" + bucketInfo.getObjectID() + "/" + omKeyInfo.getParentObjectID() + "/" +
+          omKeyInfo.getFileName();
+    } else {
+      return bucketInfo.getVolumeName() + "/" + bucketInfo.getBucketName() + "/" + omKeyInfo.getKeyName();
+    }
+  }
+
+  private String getDirectoryKey(long volumeId, OmBucketInfo bucketInfo, OmDirectoryInfo omDirInfo) {
+    return volumeId + "/" + bucketInfo.getObjectID() + "/" + omDirInfo.getParentObjectID() + "/" +
+        omDirInfo.getName();
+  }
+
+  private String getRenameKey(String volume, String bucket, long objectId) {
+    return volume + "/" + bucket + "/" + objectId;
+  }
+
+  @Test
+  public void testPreviousSnapshotOzoneDirInfo() throws IOException {
+    OMMetadataManager omMetadataManager = mock(OMMetadataManager.class);
+    when(omMetadataManager.getOzonePathKey(anyLong(), anyLong(), anyLong(), anyString()))
+        .thenAnswer(i -> Arrays.stream(i.getArguments()).map(Object::toString)
+            .collect(Collectors.joining("/")));
+    when(omMetadataManager.getRenameKey(anyString(), anyString(), anyLong())).thenAnswer(
+        i -> getRenameKey(i.getArgument(0), i.getArgument(1), i.getArgument(2)));
+
+    OMMetadataManager previousMetadataManager = mock(OMMetadataManager.class);
+    OzoneConfiguration configuration = new OzoneConfiguration();
+    KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null);
+    KeyManagerImpl prevKM = new KeyManagerImpl(null, null, previousMetadataManager, configuration, null, null, null);
+    long volumeId = 1L;
+    OmBucketInfo bucketInfo = OmBucketInfo.newBuilder().setBucketName(BUCKET_NAME).setVolumeName(VOLUME_NAME)
+        .setObjectID(2L).setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED).build();
+    OmDirectoryInfo prevKey = getMockedOmDirInfo(5, "key", 1);
+    OmDirectoryInfo prevKey2 = getMockedOmDirInfo(7, "key2", 2);
+    OmKeyInfo currentKey =  getMockedOmKeyInfo(bucketInfo, 6, "renamedKey", 1);
+    OmDirectoryInfo currentKeyDir = getMockedOmDirInfo(6, "renamedKey", 1);
+    OmKeyInfo currentKey2 = getMockedOmKeyInfo(bucketInfo, 7, "key2", 2);
+    OmDirectoryInfo currentKeyDir2 = getMockedOmDirInfo(7, "key2", 2);
+    OmKeyInfo currentKey3 = getMockedOmKeyInfo(bucketInfo, 8, "key3", 3);
+    OmDirectoryInfo currentKeyDir3 = getMockedOmDirInfo(8, "key3", 3);
+    OmKeyInfo currentKey4 = getMockedOmKeyInfo(bucketInfo, 8, "key4", 4);
+    OmDirectoryInfo currentKeyDir4 = getMockedOmDirInfo(8, "key4", 4);
+    Table<String, OmDirectoryInfo> prevDirTable = new InMemoryTestTable<>(
+        ImmutableMap.of(getDirectoryKey(volumeId, bucketInfo, prevKey), prevKey,
+            getDirectoryKey(volumeId, bucketInfo, prevKey2), prevKey2));
+    Table<String, String> renameTable = new InMemoryTestTable<>(
+        ImmutableMap.of(getRenameKey(VOLUME_NAME, BUCKET_NAME, 1),
+            getDirectoryKey(volumeId, bucketInfo, prevKey),
+        getRenameKey(VOLUME_NAME, BUCKET_NAME, 3), getDirectoryKey(volumeId, bucketInfo,
+            getMockedOmKeyInfo(bucketInfo, 6, "unknownKey", 9))));
+    when(previousMetadataManager.getDirectoryTable()).thenReturn(prevDirTable);
+    when(omMetadataManager.getSnapshotRenamedTable()).thenReturn(renameTable);
+    assertEquals(prevKey, km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKey).apply(prevKM));
+    assertEquals(prevKey2, km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKey2).apply(prevKM));
+    assertNull(km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKey3).apply(prevKM));
+    assertNull(km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKey4).apply(prevKM));
+
+    assertEquals(prevKey, km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKeyDir).apply(prevKM));
+    assertEquals(prevKey2, km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKeyDir2).apply(prevKM));
+    assertNull(km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKeyDir3).apply(prevKM));
+    assertNull(km.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, currentKeyDir4).apply(prevKM));
+  }
+
   private void initKeyTableForMultipartTest(String keyName, String volume) throws IOException {
     List<OmKeyLocationInfoGroup> locationInfoGroups = new ArrayList<>();
     List<OmKeyLocationInfo> locationInfoList = new ArrayList<>();

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java

@@ -31,6 +31,8 @@
 import org.apache.hadoop.ozone.om.fs.OzoneManagerFS;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
@@ -41,6 +43,7 @@
 import org.apache.hadoop.ozone.om.service.SnapshotDeletingService;
 import org.apache.hadoop.ozone.om.service.SnapshotDirectoryCleaningService;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket;
+import org.apache.ratis.util.function.CheckedFunction;
 
 /**
  * Handles key level commands.
@@ -84,7 +87,6 @@ OmKeyInfo lookupKey(OmKeyArgs args, ResolvedBucket bucketLayout,
   OmKeyInfo getKeyInfo(OmKeyArgs args, ResolvedBucket buctket,
       String clientAddress) throws IOException;
 
-
   /**
    * Returns a list of keys represented by {@link OmKeyInfo}
    * in the given bucket.
@@ -135,6 +137,18 @@ List<Table.KeyValue<String, String>> getRenamesKeyEntries(
       String volume, String bucket, String startKey, int size) throws IOException;
 
 
+  /**
+   * Returns the previous snapshot's ozone directorInfo corresponding for the object.
+   */
+  CheckedFunction<KeyManager, OmDirectoryInfo, IOException> getPreviousSnapshotOzoneDirInfo(
+      long volumeId, OmBucketInfo bucketInfo, OmDirectoryInfo directoryInfo) throws IOException;
+
+  /**
+   * Returns the previous snapshot's ozone directoryInfo corresponding for the object.
+   */
+  CheckedFunction<KeyManager, OmDirectoryInfo, IOException> getPreviousSnapshotOzoneDirInfo(
+      long volumeId, OmBucketInfo bucketInfo, OmKeyInfo directoryInfo) throws IOException;
+
   /**
    * Returns a list deleted entries from the deletedTable.
    *

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java

@@ -174,6 +174,7 @@
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Time;
+import org.apache.ratis.util.function.CheckedFunction;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -774,6 +775,33 @@ public List<Table.KeyValue<String, String>> getRenamesKeyEntries(
     }
   }
 
+  @Override
+  public CheckedFunction<KeyManager, OmDirectoryInfo, IOException> getPreviousSnapshotOzoneDirInfo(
+      long volumeId, OmBucketInfo bucketInfo, OmDirectoryInfo keyInfo) throws IOException {
+    String currentKeyPath = metadataManager.getOzonePathKey(volumeId, bucketInfo.getObjectID(),
+        keyInfo.getParentObjectID(), keyInfo.getName());
+    return getPreviousSnapshotOzonePathInfo(bucketInfo, keyInfo.getObjectID(), currentKeyPath,
+        (km) -> km.getMetadataManager().getDirectoryTable());
+  }
+
+  @Override
+  public CheckedFunction<KeyManager, OmDirectoryInfo, IOException> getPreviousSnapshotOzoneDirInfo(
+      long volumeId, OmBucketInfo bucketInfo, OmKeyInfo keyInfo) throws IOException {
+    String currentKeyPath = metadataManager.getOzonePathKey(volumeId, bucketInfo.getObjectID(),
+        keyInfo.getParentObjectID(), keyInfo.getFileName());
+    return getPreviousSnapshotOzonePathInfo(bucketInfo, keyInfo.getObjectID(), currentKeyPath,
+        (previousSnapshotKM) -> previousSnapshotKM.getMetadataManager().getDirectoryTable());
+  }
+
+  private <T> CheckedFunction<KeyManager, T, IOException> getPreviousSnapshotOzonePathInfo(
+      OmBucketInfo bucketInfo, long objectId, String currentKeyPath,
+      Function<KeyManager, Table<String, T>> table) throws IOException {
+    String renameKey = metadataManager.getRenameKey(bucketInfo.getVolumeName(), bucketInfo.getBucketName(), objectId);
+    String renamedKey = metadataManager.getSnapshotRenamedTable().getIfExist(renameKey);
+    return (previousSnapshotKM) -> table.apply(previousSnapshotKM).get(
+        renamedKey != null ? renamedKey : currentKeyPath);
+  }
+
   @Override
   public List<Table.KeyValue<String, List<OmKeyInfo>>> getDeletedKeyEntries(
       String volume, String bucket, String startKey, int size) throws IOException {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableDirFilter.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.filter;
+
+import java.io.IOException;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.om.KeyManager;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.SnapshotChainManager;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
+import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted;
+
+/**
+ * Class to filter out deleted directories which are reclaimable based on their presence in previous snapshot in
+ * the snapshot chain.
+ */
+public class ReclaimableDirFilter extends ReclaimableFilter<OmKeyInfo> {
+
+  public ReclaimableDirFilter(OzoneManager ozoneManager,
+                              OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager,
+                              SnapshotInfo currentSnapshotInfo, KeyManager keyManager,
+                              IOzoneManagerLock lock) {
+    super(ozoneManager, omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager, lock, 1);
+  }
+
+  @Override
+  protected String getVolumeName(Table.KeyValue<String, OmKeyInfo> keyValue) throws IOException {
+    return keyValue.getValue().getVolumeName();
+  }
+
+  @Override
+  protected String getBucketName(Table.KeyValue<String, OmKeyInfo> keyValue) throws IOException {
+    return keyValue.getValue().getBucketName();
+  }
+
+  @Override
+  protected Boolean isReclaimable(Table.KeyValue<String, OmKeyInfo> deletedDirInfo) throws IOException {
+    ReferenceCounted<OmSnapshot> previousSnapshot = getPreviousOmSnapshot(0);
+    KeyManager prevKeyManager = previousSnapshot == null ? null : previousSnapshot.get().getKeyManager();
+    return isDirReclaimable(getVolumeId(), getBucketInfo(), deletedDirInfo.getValue(), getKeyManager(), prevKeyManager);
+  }
+
+  private boolean isDirReclaimable(long volumeId, OmBucketInfo bucketInfo, OmKeyInfo dirInfo,
+                                   KeyManager keyManager, KeyManager previousKeyManager) throws IOException {
+    if (previousKeyManager == null) {
+      return true;
+    }
+    OmDirectoryInfo prevDirectoryInfo =
+        keyManager.getPreviousSnapshotOzoneDirInfo(volumeId, bucketInfo, dirInfo).apply(previousKeyManager);
+    return prevDirectoryInfo == null || prevDirectoryInfo.getObjectID() != dirInfo.getObjectID();
+  }
+}

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableRenameEntryFilter.java

@@ -32,7 +32,7 @@
 import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted;
 
 /**
- * Filter to return rename table entries which are reclaimable based on the key presence in previous snapshot's
+ * Class to filter out rename table entries which are reclaimable based on the key presence in previous snapshot's
  * keyTable/DirectoryTable in the snapshot chain.
  */
 public class ReclaimableRenameEntryFilter extends ReclaimableFilter<String> {