diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index 046ed87abdd9..0a8b4fb2c83b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -41,6 +41,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.anyLong; @@ -145,6 +146,7 @@ import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; @@ -1628,6 +1630,51 @@ private String getRenameKey(String volume, String bucket, long objectId) { return volume + "/" + bucket + "/" + objectId; } + @ParameterizedTest + @EnumSource(value = BucketLayout.class) + public void testPreviousSnapshotOzoneKeyInfo(BucketLayout bucketLayout) throws IOException { + OMMetadataManager omMetadataManager = mock(OMMetadataManager.class); + if (bucketLayout.isFileSystemOptimized()) { + when(omMetadataManager.getOzonePathKey(anyLong(), anyLong(), anyLong(), anyString())) + .thenAnswer(i -> Arrays.stream(i.getArguments()).map(Object::toString) + .collect(Collectors.joining("/"))); + } else { + when(omMetadataManager.getOzoneKey(anyString(), anyString(), anyString())) + .thenAnswer(i -> Arrays.stream(i.getArguments()).map(Object::toString) + .collect(Collectors.joining("/"))); + } + when(omMetadataManager.getRenameKey(anyString(), anyString(), anyLong())).thenAnswer( + i -> getRenameKey(i.getArgument(0), i.getArgument(1), i.getArgument(2))); + + OMMetadataManager previousMetadataManager = mock(OMMetadataManager.class); + OzoneConfiguration configuration = new OzoneConfiguration(); + KeyManagerImpl km = new KeyManagerImpl(null, null, omMetadataManager, configuration, null, null, null); + KeyManagerImpl prevKM = new KeyManagerImpl(null, null, previousMetadataManager, configuration, null, null, null); + long volumeId = 1L; + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder().setBucketName(BUCKET_NAME).setVolumeName(VOLUME_NAME) + .setObjectID(2L).setBucketLayout(bucketLayout).build(); + OmKeyInfo prevKey = getMockedOmKeyInfo(bucketInfo, 5, "key", 1); + OmKeyInfo prevKey2 = getMockedOmKeyInfo(bucketInfo, 7, "key2", 2); + OmKeyInfo currentKey = getMockedOmKeyInfo(bucketInfo, 6, "renamedKey", 1); + OmKeyInfo currentKey2 = getMockedOmKeyInfo(bucketInfo, 7, "key2", 2); + OmKeyInfo currentKey3 = getMockedOmKeyInfo(bucketInfo, 8, "key3", 3); + OmKeyInfo currentKey4 = getMockedOmKeyInfo(bucketInfo, 8, "key4", 4); + Table prevKeyTable = + new InMemoryTestTable<>(ImmutableMap.of( + getDirectoryKey(volumeId, bucketInfo, prevKey), prevKey, + getDirectoryKey(volumeId, bucketInfo, prevKey2), prevKey2)); + Table renameTable = new InMemoryTestTable<>( + ImmutableMap.of(getRenameKey(VOLUME_NAME, BUCKET_NAME, 1), getDirectoryKey(volumeId, bucketInfo, prevKey), + getRenameKey(VOLUME_NAME, BUCKET_NAME, 3), getDirectoryKey(volumeId, bucketInfo, + getMockedOmKeyInfo(bucketInfo, 6, "unknownKey", 9)))); + when(previousMetadataManager.getKeyTable(eq(bucketLayout))).thenReturn(prevKeyTable); + when(omMetadataManager.getSnapshotRenamedTable()).thenReturn(renameTable); + assertEquals(prevKey, km.getPreviousSnapshotOzoneKeyInfo(volumeId, bucketInfo, currentKey).apply(prevKM)); + assertEquals(prevKey2, km.getPreviousSnapshotOzoneKeyInfo(volumeId, bucketInfo, currentKey2).apply(prevKM)); + assertNull(km.getPreviousSnapshotOzoneKeyInfo(volumeId, bucketInfo, currentKey3).apply(prevKM)); + assertNull(km.getPreviousSnapshotOzoneKeyInfo(volumeId, bucketInfo, currentKey4).apply(prevKM)); + } + @Test public void testPreviousSnapshotOzoneDirInfo() throws IOException { OMMetadataManager omMetadataManager = mock(OMMetadataManager.class); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 268db5b75188..fa3e622313df 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -149,6 +149,12 @@ CheckedFunction getPreviousSnapshotOzo CheckedFunction getPreviousSnapshotOzoneDirInfo( long volumeId, OmBucketInfo bucketInfo, OmKeyInfo directoryInfo) throws IOException; + /** + * Returns the previous snapshot's ozone keyInfo corresponding for the object. + */ + CheckedFunction getPreviousSnapshotOzoneKeyInfo( + long volumeId, OmBucketInfo bucketInfo, OmKeyInfo keyInfo) throws IOException; + /** * Returns a list deleted entries from the deletedTable. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 9ec658db1a01..96673113771b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -793,6 +793,17 @@ public CheckedFunction getPreviousSnap (previousSnapshotKM) -> previousSnapshotKM.getMetadataManager().getDirectoryTable()); } + @Override + public CheckedFunction getPreviousSnapshotOzoneKeyInfo( + long volumeId, OmBucketInfo bucketInfo, OmKeyInfo keyInfo) throws IOException { + String currentKeyPath = bucketInfo.getBucketLayout().isFileSystemOptimized() + ? metadataManager.getOzonePathKey(volumeId, bucketInfo.getObjectID(), keyInfo.getParentObjectID(), + keyInfo.getFileName()) : metadataManager.getOzoneKey(bucketInfo.getVolumeName(), bucketInfo.getBucketName(), + keyInfo.getKeyName()); + return getPreviousSnapshotOzonePathInfo(bucketInfo, keyInfo.getObjectID(), currentKeyPath, + (previousSnapshotKM) -> previousSnapshotKM.getMetadataManager().getKeyTable(bucketInfo.getBucketLayout())); + } + private CheckedFunction getPreviousSnapshotOzonePathInfo( OmBucketInfo bucketInfo, long objectId, String currentKeyPath, Function> table) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index c59e0629b31a..7a4b6ecdb2e3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -33,8 +33,8 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; -import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotDirExist; +import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.isBlockLocationInfoSame; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index ccf77eb44ee9..0d36da711703 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -19,7 +19,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; -import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; +import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.isBlockLocationInfoSame; import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ServiceException; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 8b28416f4e8b..5c2b16a604b3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -52,8 +52,6 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; @@ -330,60 +328,6 @@ boolean shouldIgnoreSnapshot(SnapshotInfo snapInfo) throws IOException { !OmSnapshotManager.areSnapshotChangesFlushedToDB(getOzoneManager().getMetadataManager(), snapInfo); } - // TODO: Move this util class. - public static boolean isBlockLocationInfoSame(OmKeyInfo prevKeyInfo, - OmKeyInfo deletedKeyInfo) { - - if (prevKeyInfo == null && deletedKeyInfo == null) { - LOG.debug("Both prevKeyInfo and deletedKeyInfo are null."); - return true; - } - if (prevKeyInfo == null || deletedKeyInfo == null) { - LOG.debug("prevKeyInfo: '{}' or deletedKeyInfo: '{}' is null.", - prevKeyInfo, deletedKeyInfo); - return false; - } - // For hsync, Though the blockLocationInfo of a key may not be same - // at the time of snapshot and key deletion as blocks can be appended. - // If the objectId is same then the key is same. - if (prevKeyInfo.isHsync() && deletedKeyInfo.isHsync()) { - return true; - } - - if (prevKeyInfo.getKeyLocationVersions().size() != - deletedKeyInfo.getKeyLocationVersions().size()) { - return false; - } - - OmKeyLocationInfoGroup deletedOmKeyLocation = - deletedKeyInfo.getLatestVersionLocations(); - OmKeyLocationInfoGroup prevOmKeyLocation = - prevKeyInfo.getLatestVersionLocations(); - - if (deletedOmKeyLocation == null || prevOmKeyLocation == null) { - return false; - } - - List deletedLocationList = - deletedOmKeyLocation.getLocationList(); - List prevLocationList = - prevOmKeyLocation.getLocationList(); - - if (deletedLocationList.size() != prevLocationList.size()) { - return false; - } - - for (int idx = 0; idx < deletedLocationList.size(); idx++) { - OmKeyLocationInfo deletedLocationInfo = deletedLocationList.get(idx); - OmKeyLocationInfo prevLocationInfo = prevLocationList.get(idx); - if (!deletedLocationInfo.hasSameBlockAs(prevLocationInfo)) { - return false; - } - } - - return true; - } - @Override public BackgroundTaskQueue getTasks() { BackgroundTaskQueue queue = new BackgroundTaskQueue(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 99626360e573..511426885d85 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -117,7 +117,6 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithObjectID; import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; -import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; @@ -1460,8 +1459,7 @@ long generateDiffReport( private boolean isKeyModified(OmKeyInfo fromKey, OmKeyInfo toKey) { return !fromKey.isKeyInfoSame(toKey, false, false, false, false, true) - || !SnapshotDeletingService.isBlockLocationInfoSame( - fromKey, toKey); + || !SnapshotUtils.isBlockLocationInfoSame(fromKey, toKey); } private boolean isObjectModified(String fromObjectName, String toObjectName, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index bfe27f3b6f98..781ab1e6f6b1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -40,6 +40,8 @@ import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; @@ -53,8 +55,7 @@ * Util class for snapshot diff APIs. */ public final class SnapshotUtils { - private static final Logger LOG = - LoggerFactory.getLogger(SnapshotUtils.class); + private static final Logger LOG = LoggerFactory.getLogger(SnapshotUtils.class); private SnapshotUtils() { throw new IllegalStateException("SnapshotUtils should not be initialized."); @@ -189,7 +190,7 @@ public static SnapshotInfo getPreviousSnapshot(OzoneManager ozoneManager, /** * Get the previous snapshot in the snapshot chain. */ - private static UUID getPreviousSnapshotId(SnapshotInfo snapInfo, SnapshotChainManager chainManager) + public static UUID getPreviousSnapshotId(SnapshotInfo snapInfo, SnapshotChainManager chainManager) throws IOException { // If the snapshot is deleted in the previous run, then the in-memory // SnapshotChainManager might throw NoSuchElementException as the snapshot @@ -299,4 +300,59 @@ public static void validatePreviousSnapshotId(SnapshotInfo snapshotInfo, OMException.ResultCodes.INVALID_REQUEST); } } + + /** + * Compares the block location info of 2 key info. + * @return true if block locations are same else false. + */ + public static boolean isBlockLocationInfoSame(OmKeyInfo prevKeyInfo, + OmKeyInfo deletedKeyInfo) { + if (prevKeyInfo == null && deletedKeyInfo == null) { + LOG.debug("Both prevKeyInfo and deletedKeyInfo are null."); + return true; + } + if (prevKeyInfo == null || deletedKeyInfo == null) { + LOG.debug("prevKeyInfo: '{}' or deletedKeyInfo: '{}' is null.", + prevKeyInfo, deletedKeyInfo); + return false; + } + // For hsync, Though the blockLocationInfo of a key may not be same + // at the time of snapshot and key deletion as blocks can be appended. + // If the objectId is same then the key is same. + if (prevKeyInfo.isHsync() && deletedKeyInfo.isHsync()) { + return true; + } + + if (prevKeyInfo.getKeyLocationVersions().size() != + deletedKeyInfo.getKeyLocationVersions().size()) { + return false; + } + + OmKeyLocationInfoGroup deletedOmKeyLocation = + deletedKeyInfo.getLatestVersionLocations(); + OmKeyLocationInfoGroup prevOmKeyLocation = + prevKeyInfo.getLatestVersionLocations(); + + if (deletedOmKeyLocation == null || prevOmKeyLocation == null) { + return false; + } + + List deletedLocationList = + deletedOmKeyLocation.getLocationList(); + List prevLocationList = + prevOmKeyLocation.getLocationList(); + + if (deletedLocationList.size() != prevLocationList.size()) { + return false; + } + + for (int idx = 0; idx < deletedLocationList.size(); idx++) { + OmKeyLocationInfo deletedLocationInfo = deletedLocationList.get(idx); + OmKeyLocationInfo prevLocationInfo = prevLocationList.get(idx); + if (!deletedLocationInfo.hasSameBlockAs(prevLocationInfo)) { + return false; + } + } + return true; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableKeyFilter.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableKeyFilter.java new file mode 100644 index 000000000000..6b558b58549a --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/filter/ReclaimableKeyFilter.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.filter; + +import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.isBlockLocationInfoSame; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.KeyManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.ratis.util.MemoizedCheckedSupplier; +import org.apache.ratis.util.function.CheckedSupplier; + +/** + * Filter to return deleted keys which are reclaimable based on their presence in previous snapshot in + * the snapshot chain. + */ +public class ReclaimableKeyFilter extends ReclaimableFilter { + private final Map exclusiveSizeMap; + private final Map exclusiveReplicatedSizeMap; + + /** + * @param currentSnapshotInfo : If null the deleted keys in AOS needs to be processed, hence the latest snapshot + * in the snapshot chain corresponding to bucket key needs to be processed. + * @param keyManager : keyManager corresponding to snapshot or AOS. + * @param lock : Lock for Active OM. + */ + public ReclaimableKeyFilter(OzoneManager ozoneManager, + OmSnapshotManager omSnapshotManager, SnapshotChainManager snapshotChainManager, + SnapshotInfo currentSnapshotInfo, KeyManager keyManager, + IOzoneManagerLock lock) { + super(ozoneManager, omSnapshotManager, snapshotChainManager, currentSnapshotInfo, keyManager, lock, 2); + this.exclusiveSizeMap = new HashMap<>(); + this.exclusiveReplicatedSizeMap = new HashMap<>(); + } + + @Override + protected String getVolumeName(Table.KeyValue keyValue) throws IOException { + return keyValue.getValue().getVolumeName(); + } + + @Override + protected String getBucketName(Table.KeyValue keyValue) throws IOException { + return keyValue.getValue().getBucketName(); + } + + @Override + /** + * Determines whether a deleted key entry is reclaimable by checking its presence in prior snapshots. + * + * This method validates the existence of the deleted key in the previous snapshot's key table or file table. + * If the key is not found in the previous snapshot, it is marked as reclaimable. Otherwise, additional checks + * are performed using the previous-to-previous snapshot to confirm if the key is exclusively present in the + * previous snapshot and accounted in the previous snapshot's exclusive size. + * + * @param deletedKeyInfo The key-value pair representing the deleted key information. + * @return {@code true} if the key is reclaimable (not present in prior snapshots), {@code false} otherwise. + * @throws IOException If an error occurs while accessing snapshot data or key information. + */ + protected Boolean isReclaimable(Table.KeyValue deletedKeyInfo) throws IOException { + ReferenceCounted previousSnapshot = getPreviousOmSnapshot(1); + + + KeyManager previousKeyManager = Optional.ofNullable(previousSnapshot) + .map(i -> i.get().getKeyManager()).orElse(null); + + + // Getting keyInfo from prev snapshot's keyTable/fileTable + CheckedSupplier, IOException> previousKeyInfo = + MemoizedCheckedSupplier.valueOf(() -> getPreviousSnapshotKeyInfo(getVolumeId(), getBucketInfo(), + deletedKeyInfo.getValue(), getKeyManager(), previousKeyManager)); + // If file not present in previous snapshot then it won't be present in previous to previous snapshot either. + if (!previousKeyInfo.get().isPresent()) { + return true; + } + + ReferenceCounted previousToPreviousSnapshot = getPreviousOmSnapshot(0); + KeyManager previousToPreviousKeyManager = Optional.ofNullable(previousToPreviousSnapshot) + .map(i -> i.get().getKeyManager()).orElse(null); + + // Getting keyInfo from prev to prev snapshot's keyTable/fileTable based on keyInfo of prev keyTable + CheckedSupplier, IOException> previousPrevKeyInfo = + MemoizedCheckedSupplier.valueOf(() -> getPreviousSnapshotKeyInfo( + getVolumeId(), getBucketInfo(), previousKeyInfo.get().orElse(null), previousKeyManager, + previousToPreviousKeyManager)); + SnapshotInfo previousSnapshotInfo = getPreviousSnapshotInfo(1); + calculateExclusiveSize(previousSnapshotInfo, previousKeyInfo, previousPrevKeyInfo, + exclusiveSizeMap, exclusiveReplicatedSizeMap); + return false; + } + + public Map getExclusiveSizeMap() { + return exclusiveSizeMap; + } + + public Map getExclusiveReplicatedSizeMap() { + return exclusiveReplicatedSizeMap; + } + + /** + * To calculate Exclusive Size for current snapshot, Check + * the next snapshot deletedTable if the deleted key is + * referenced in current snapshot and not referenced in the + * previous snapshot then that key is exclusive to the current + * snapshot. Here since we are only iterating through + * deletedTable we can check the previous and previous to + * previous snapshot to achieve the same. + * previousSnapshot - Snapshot for which exclusive size is + * getting calculating. + * currSnapshot - Snapshot's deletedTable is used to calculate + * previousSnapshot snapshot's exclusive size. + * previousToPrevSnapshot - Snapshot which is used to check + * if key is exclusive to previousSnapshot. + */ + private void calculateExclusiveSize(SnapshotInfo previousSnapshotInfo, + CheckedSupplier, IOException> keyInfoPrevSnapshot, + CheckedSupplier, IOException> keyInfoPrevToPrevSnapshot, + Map exclusiveSizes, Map exclusiveReplicatedSizes) + throws IOException { + if (keyInfoPrevSnapshot.get().isPresent() && !keyInfoPrevToPrevSnapshot.get().isPresent()) { + OmKeyInfo keyInfo = keyInfoPrevSnapshot.get().get(); + exclusiveSizes.compute(previousSnapshotInfo.getSnapshotId(), + (k, v) -> (v == null ? 0 : v) + keyInfo.getDataSize()); + exclusiveReplicatedSizes.compute(previousSnapshotInfo.getSnapshotId(), + (k, v) -> (v == null ? 0 : v) + keyInfo.getReplicatedSize()); + } + } + + private Optional getPreviousSnapshotKeyInfo(long volumeId, OmBucketInfo bucketInfo, + OmKeyInfo keyInfo, KeyManager keyManager, + KeyManager previousKeyManager) throws IOException { + if (keyInfo == null || previousKeyManager == null) { + return Optional.empty(); + } + OmKeyInfo prevKeyInfo = keyManager.getPreviousSnapshotOzoneKeyInfo(volumeId, bucketInfo, keyInfo) + .apply(previousKeyManager); + + // Check if objectIds are matching then the keys are the same. + if (prevKeyInfo == null || prevKeyInfo.getObjectID() != keyInfo.getObjectID()) { + return Optional.empty(); + } + return isBlockLocationInfoSame(prevKeyInfo, keyInfo) ? Optional.of(prevKeyInfo) : Optional.empty(); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index de08a13701c4..6cfb21b2feec 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -135,7 +135,6 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; -import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.om.snapshot.SnapshotTestUtils.StubbedPersistentMap; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage; @@ -820,11 +819,10 @@ public void testGenerateDiffReport() throws IOException { String bucketName = "buck"; String fromSnapName = "fs"; String toSnapName = "ts"; - try (MockedStatic - mockedSnapshotDeletingService = mockStatic( - SnapshotDeletingService.class)) { - mockedSnapshotDeletingService.when(() -> - SnapshotDeletingService.isBlockLocationInfoSame(any(OmKeyInfo.class), + try (MockedStatic + mockedSnapshotUtils = mockStatic(SnapshotUtils.class)) { + mockedSnapshotUtils.when(() -> + SnapshotUtils.isBlockLocationInfoSame(any(OmKeyInfo.class), any(OmKeyInfo.class))) .thenAnswer(i -> { int keyVal = Integer.parseInt(((OmKeyInfo)i.getArgument(0)) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/AbstractReclaimableFilterTest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/AbstractReclaimableFilterTest.java index 27158c01349c..1aeafc5c5a6f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/AbstractReclaimableFilterTest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/AbstractReclaimableFilterTest.java @@ -80,7 +80,8 @@ import org.rocksdb.RocksIterator; /** - * Test class for ReclaimableFilter. + *

Class for having a common setup containing util functions to test out functionalities of various + * implementations of {@link ReclaimableFilter} class.

*/ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class AbstractReclaimableFilterTest { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/TestReclaimableKeyFilter.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/TestReclaimableKeyFilter.java new file mode 100644 index 000000000000..021a44eafd92 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/TestReclaimableKeyFilter.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot.filter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.KeyManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.rocksdb.RocksDBException; + +/** + * Test class for verifying the behavior of the ReclaimableKeyFilter. + * + *

The ReclaimableKeyFilter is responsible for determining if a deleted key entry + * can be reclaimed based on its presence in previous snapshots. In essence, if a key + * does not appear in a previous snapshot—or if the snapshot chain's state indicates that + * the key is no longer valid—the key is considered reclaimable. These tests simulate a + * variety of snapshot chain scenarios to validate that: + *

    + *
  • Keys are not reclaimed if they exist in prior snapshots.
    • + *
  • Keys are reclaimed if they are removed from the previous snapshot.
    • + *
  • Exclusive size metrics, including data and replicated sizes, are correctly updated + * for snapshots where keys are not reclaimed.
    • + *
+ * + *

This class extends {@link AbstractReclaimableFilterTest} to leverage common setup, snapshot + * chain preparation, and utility methods. + */ +public class TestReclaimableKeyFilter extends AbstractReclaimableFilterTest { + @Override + protected ReclaimableFilter initializeFilter(OzoneManager om, OmSnapshotManager snapshotManager, + SnapshotChainManager chainManager, SnapshotInfo currentSnapshotInfo, + KeyManager km, IOzoneManagerLock lock, + int numberOfPreviousSnapshotsFromChain) { + return new ReclaimableKeyFilter(om, snapshotManager, chainManager, currentSnapshotInfo, km, lock); + } + + List testReclaimableFilterArguments() { + List arguments = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 5; j++) { + arguments.add(Arguments.of(i, j)); + } + } + return arguments; + } + + private KeyManager mockOmSnapshot(ReferenceCounted snapshot) { + if (snapshot != null) { + OmSnapshot omSnapshot = snapshot.get(); + KeyManager keyManager = mock(KeyManager.class); + when(omSnapshot.getKeyManager()).thenReturn(keyManager); + return keyManager; + } + return null; + } + + @SuppressWarnings("checkstyle:ParameterNumber") + private void testReclaimableKeyFilter(String volume, String bucket, int index, + OmKeyInfo keyInfo, OmKeyInfo prevKeyInfo, OmKeyInfo prevPrevKeyInfo, + Boolean expectedValue, + Optional size, Optional replicatedSize) + throws IOException { + List snapshotInfos = getLastSnapshotInfos(volume, bucket, 2, index); + SnapshotInfo previousToPreviousSapshotInfo = snapshotInfos.get(0); + SnapshotInfo prevSnapshotInfo = snapshotInfos.get(1); + OmBucketInfo bucketInfo = getOzoneManager().getBucketInfo(volume, bucket); + long volumeId = getOzoneManager().getMetadataManager().getVolumeId(volume); + + ReferenceCounted prevSnap = Optional.ofNullable(prevSnapshotInfo) + .map(info -> { + try { + return getOmSnapshotManager().getActiveSnapshot(volume, bucket, info.getName()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).orElse(null); + ReferenceCounted prevToPrevSnap = Optional.ofNullable(previousToPreviousSapshotInfo) + .map(info -> { + try { + return getOmSnapshotManager().getActiveSnapshot(volume, bucket, info.getName()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).orElse(null); + + KeyManager keyManager = getKeyManager(); + KeyManager prevKeyManager = mockOmSnapshot(prevSnap); + KeyManager prevToPrevKeyManager = mockOmSnapshot(prevToPrevSnap); + if (prevKeyManager != null) { + when(keyManager.getPreviousSnapshotOzoneKeyInfo(eq(volumeId), + eq(bucketInfo), eq(keyInfo))) + .thenReturn((km) -> prevKeyInfo); + } + if (prevKeyInfo != null && prevKeyManager != null && prevToPrevKeyManager != null) { + when(prevKeyManager.getPreviousSnapshotOzoneKeyInfo(eq(volumeId), + eq(bucketInfo), eq(prevKeyInfo))).thenReturn((km) -> prevPrevKeyInfo); + } + when(keyInfo.getVolumeName()).thenReturn(volume); + when(keyInfo.getBucketName()).thenReturn(bucket); + assertEquals(expectedValue, getReclaimableFilter().apply(Table.newKeyValue("key", keyInfo))); + ReclaimableKeyFilter keyFilter = (ReclaimableKeyFilter) getReclaimableFilter(); + if (prevSnap != null) { + assertEquals(size.map(AtomicLong::get).orElse(null), + keyFilter.getExclusiveSizeMap().get(prevSnap.get().getSnapshotID())); + assertEquals(replicatedSize.map(AtomicLong::get).orElse(null), + keyFilter.getExclusiveReplicatedSizeMap().get(prevSnap.get().getSnapshotID())); + } else { + assertTrue(keyFilter.getExclusiveReplicatedSizeMap().isEmpty()); + assertTrue(keyFilter.getExclusiveSizeMap().isEmpty()); + } + + } + + private OmKeyInfo getMockedOmKeyInfo(long objectId, long size, long replicatedSize) { + OmKeyInfo keyInfo = mock(OmKeyInfo.class); + when(keyInfo.getObjectID()).thenReturn(objectId); + when(keyInfo.getDataSize()).thenReturn(size); + when(keyInfo.getReplicatedSize()).thenReturn(replicatedSize); + return keyInfo; + } + + private OmKeyInfo getMockedOmKeyInfo(long objectId) { + return getMockedOmKeyInfo(objectId, 0, 0); + } + + /** + * Tests that a key present in prior snapshots is not reclaimable. + * + * @param actualNumberOfSnapshots the total number of snapshots in the chain. + * @param index the snapshot chain index used for testing. + * @throws IOException if an I/O error occurs during the test. + * @throws RocksDBException if RocksDB encounters an error. + */ + @ParameterizedTest + @MethodSource("testReclaimableFilterArguments") + public void testNonReclaimableKey(int actualNumberOfSnapshots, int index) throws IOException, RocksDBException { + setup(2, actualNumberOfSnapshots, index, 4, 2); + String volume = getVolumes().get(3); + String bucket = getBuckets().get(1); + index = Math.min(index, actualNumberOfSnapshots); + OmKeyInfo keyInfo = getMockedOmKeyInfo(1); + OmKeyInfo prevKeyInfo = index - 1 >= 0 ? getMockedOmKeyInfo(1) : null; + OmKeyInfo prevPrevKeyInfo = index - 2 >= 0 ? getMockedOmKeyInfo(3) : null; + if (prevKeyInfo != null) { + getMockedSnapshotUtils().when(() -> SnapshotUtils.isBlockLocationInfoSame(eq(prevKeyInfo), eq(keyInfo))) + .thenReturn(true); + } + if (prevPrevKeyInfo != null) { + getMockedSnapshotUtils().when(() -> SnapshotUtils.isBlockLocationInfoSame(eq(prevPrevKeyInfo), eq(prevKeyInfo))) + .thenReturn(true); + } + Optional size = Optional.ofNullable(prevKeyInfo).map(i -> new AtomicLong()); + testReclaimableKeyFilter(volume, bucket, index, keyInfo, prevKeyInfo, prevPrevKeyInfo, + prevKeyInfo == null, size, size); + } + + /** + * Tests the filter when the object IDs differ between current and previous snapshot key entries. + * + * @param actualNumberOfSnapshots the total number of snapshots in the chain. + * @param index the snapshot chain index used for testing. + * @throws IOException if an I/O error occurs during the test. + * @throws RocksDBException if RocksDB encounters an error. + */ + @ParameterizedTest + @MethodSource("testReclaimableFilterArguments") + public void testReclaimableKeyWithDifferentObjId(int actualNumberOfSnapshots, int index) + throws IOException, RocksDBException { + setup(2, actualNumberOfSnapshots, index, 4, 2); + String volume = getVolumes().get(3); + String bucket = getBuckets().get(1); + index = Math.min(index, actualNumberOfSnapshots); + OmKeyInfo keyInfo = getMockedOmKeyInfo(1); + OmKeyInfo prevKeyInfo = index - 1 >= 0 ? getMockedOmKeyInfo(2) : null; + OmKeyInfo prevPrevKeyInfo = index - 2 >= 0 ? getMockedOmKeyInfo(3) : null; + if (prevKeyInfo != null) { + getMockedSnapshotUtils().when(() -> SnapshotUtils.isBlockLocationInfoSame(eq(prevKeyInfo), eq(keyInfo))) + .thenReturn(true); + } + testReclaimableKeyFilter(volume, bucket, index, keyInfo, prevKeyInfo, prevPrevKeyInfo, + true, Optional.empty(), Optional.empty()); + } + + /** + * Tests the filter behavior when block location information differs between snapshots. + * + * @param actualNumberOfSnapshots the total number of snapshots in the chain. + * @param index the snapshot chain index used for testing. + * @throws IOException if an I/O error occurs during the test. + * @throws RocksDBException if RocksDB encounters an error. + */ + @ParameterizedTest + @MethodSource("testReclaimableFilterArguments") + public void testReclaimableKeyWithDifferentBlockIds(int actualNumberOfSnapshots, int index) + throws IOException, RocksDBException { + setup(2, actualNumberOfSnapshots, index, 4, 2); + String volume = getVolumes().get(3); + String bucket = getBuckets().get(1); + index = Math.min(index, actualNumberOfSnapshots); + OmKeyInfo keyInfo = getMockedOmKeyInfo(1); + OmKeyInfo prevKeyInfo = index - 1 >= 0 ? getMockedOmKeyInfo(1) : null; + OmKeyInfo prevPrevKeyInfo = index - 2 >= 0 ? getMockedOmKeyInfo(3) : null; + if (prevKeyInfo != null) { + getMockedSnapshotUtils().when(() -> SnapshotUtils.isBlockLocationInfoSame(eq(prevKeyInfo), eq(keyInfo))) + .thenReturn(false); + } + testReclaimableKeyFilter(volume, bucket, index, keyInfo, prevKeyInfo, prevPrevKeyInfo, + true, Optional.empty(), Optional.empty()); + } + + /** + * Tests the exclusive size calculation for a non-reclaimable key. + * + * This test verifies that for a key which is not reclaimable, + * the filter correctly updates the exclusive size and exclusive replicated size + * stored in the filter's maps. + * + * @param actualNumberOfSnapshots the total number of snapshots in the chain. + * @param index the snapshot chain index used for testing size calculations. + * @throws IOException if an I/O error occurs during the test. + * @throws RocksDBException if RocksDB encounters an error. + */ + @ParameterizedTest + @MethodSource("testReclaimableFilterArguments") + public void testExclusiveSizeCalculationWithNonReclaimableKey(int actualNumberOfSnapshots, int index) + throws IOException, RocksDBException { + setup(2, actualNumberOfSnapshots, index, 4, 2); + String volume = getVolumes().get(3); + String bucket = getBuckets().get(1); + index = Math.min(index, actualNumberOfSnapshots); + OmKeyInfo keyInfo = getMockedOmKeyInfo(1, 1, 4); + OmKeyInfo prevKeyInfo = index - 1 >= 0 ? getMockedOmKeyInfo(1, 2, 5) : null; + OmKeyInfo prevPrevKeyInfo = index - 2 >= 0 ? getMockedOmKeyInfo(1, 3, 6) : null; + if (prevKeyInfo != null) { + getMockedSnapshotUtils().when(() -> SnapshotUtils.isBlockLocationInfoSame(eq(prevKeyInfo), eq(keyInfo))) + .thenReturn(true); + } + if (prevPrevKeyInfo != null) { + getMockedSnapshotUtils().when(() -> SnapshotUtils.isBlockLocationInfoSame(eq(prevPrevKeyInfo), eq(prevKeyInfo))) + .thenReturn(true); + } + + Optional size = Optional.ofNullable(prevKeyInfo) + .map(i -> prevPrevKeyInfo == null ? new AtomicLong(2) : null); + Optional replicatedSize = Optional.ofNullable(prevKeyInfo) + .map(i -> prevPrevKeyInfo == null ? new AtomicLong(5) : null); + + testReclaimableKeyFilter(volume, bucket, index, keyInfo, prevKeyInfo, prevPrevKeyInfo, + prevKeyInfo == null, size, replicatedSize); + if (prevPrevKeyInfo != null) { + getMockedSnapshotUtils().when(() -> SnapshotUtils.isBlockLocationInfoSame(eq(prevPrevKeyInfo), eq(prevKeyInfo))) + .thenReturn(false); + } + if (prevKeyInfo != null) { + size = Optional.of(size.orElse(new AtomicLong())); + replicatedSize = Optional.of(replicatedSize.orElse(new AtomicLong())); + size.get().addAndGet(2L); + replicatedSize.get().addAndGet(5L); + } + testReclaimableKeyFilter(volume, bucket, index, keyInfo, prevKeyInfo, prevPrevKeyInfo, + prevKeyInfo == null, size, replicatedSize); + OmKeyInfo prevPrevKeyInfo1; + if (prevPrevKeyInfo != null) { + prevPrevKeyInfo1 = getMockedOmKeyInfo(2, 3, 4); + getMockedSnapshotUtils().when(() -> SnapshotUtils.isBlockLocationInfoSame(eq(prevPrevKeyInfo1), eq(prevKeyInfo))) + .thenReturn(true); + } else { + prevPrevKeyInfo1 = null; + } + + if (prevKeyInfo != null) { + size = Optional.of(size.orElse(new AtomicLong())); + replicatedSize = Optional.of(replicatedSize.orElse(new AtomicLong())); + size.get().addAndGet(2L); + replicatedSize.get().addAndGet(5L); + } + testReclaimableKeyFilter(volume, bucket, index, keyInfo, prevKeyInfo, prevPrevKeyInfo1, + prevKeyInfo == null, size, replicatedSize); + } +}