From d10ba4b327653fcdcca4b25b50b85b4a0feccf5b Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Thu, 2 Feb 2023 23:59:13 +0530 Subject: [PATCH 01/20] HDDS-7740. [Snapshot] Implement SnapshotDeletingService --- .../apache/hadoop/ozone/OzoneConfigKeys.java | 10 + .../src/main/resources/ozone-default.xml | 27 ++ .../java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../apache/hadoop/ozone/audit/OMAction.java | 3 +- .../apache/hadoop/ozone/om/OMConfigKeys.java | 4 + .../src/main/proto/OmClientProtocol.proto | 19 + .../apache/hadoop/ozone/om/KeyManager.java | 6 + .../hadoop/ozone/om/KeyManagerImpl.java | 32 ++ .../hadoop/ozone/om/OmSnapshotManager.java | 2 +- .../apache/hadoop/ozone/om/OzoneManager.java | 11 + .../hadoop/ozone/om/SnapshotChainManager.java | 16 +- .../ratis/utils/OzoneManagerRatisUtils.java | 3 + .../snapshot/OMSnapshotCreateRequest.java | 25 +- .../OMSnapshotMoveDeletedKeysRequest.java | 111 +++++ .../OMSnapshotMoveDeletedKeysResponse.java | 103 +++++ .../om/service/SnapshotDeletingService.java | 378 ++++++++++++++++++ .../service/TestSnapshotDeletingService.java | 233 +++++++++++ 17 files changed, 977 insertions(+), 7 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index e447dd7ba280..22bd2c209a14 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -234,6 +234,16 @@ public final class OzoneConfigKeys { OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT_DEFAULT = "300s"; // 300s for default + public static final String OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL = + "ozone.snapshot.deletion.service.interval"; + public static final String + OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL_DEFAULT = "30s"; + + public static final String OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT = + "ozone.snapshot.deletion.service.timeout"; + public static final String + OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT_DEFAULT = "300s"; + public static final String OZONE_BLOCK_DELETING_SERVICE_WORKERS = "ozone.block.deleting.service.workers"; public static final int OZONE_BLOCK_DELETING_SERVICE_WORKERS_DEFAULT diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index dd11e42ce238..f733079191b2 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3165,6 +3165,14 @@ sst filtering service per time interval. + + ozone.snapshot.deleting.limit.per.task + 10 + OZONE, PERFORMANCE, OM + The number of snapshots to be reclaimed by the + Snapshot Deleting Service per run. + + ozone.snapshot.filtering.service.interval 1m @@ -3180,6 +3188,25 @@ + + ozone.snapshot.deletion.service.timeout + 300s + OZONE, PERFORMANCE, OM + + Timeout value for SnapshotDeletingService. + + + + + ozone.snapshot.deletion.service.interval + 30s + OZONE, PERFORMANCE, OM + + The time interval between successive SnapshotDeletingService + thread run. + + + ozone.scm.event.ContainerReport.thread.pool.size 10 diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 10f3cbc439e4..94f27ddaf89c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -325,6 +325,7 @@ public static boolean isReadOnly( case TenantRevokeAdmin: case SetRangerServiceVersion: case CreateSnapshot: + case SnapshotMoveDeletedKeys: return false; default: LOG.error("CmdType {} is not categorized as readOnly or not.", cmdType); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java index c691b18ac0c9..ccd8a86801c0 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java @@ -90,7 +90,8 @@ public enum OMAction implements AuditAction { TENANT_REVOKE_ADMIN, TENANT_LIST_USER, CREATE_SNAPSHOT, - LIST_SNAPSHOT; + LIST_SNAPSHOT, + SNAPSHOT_MOVE_DELETED_KEYS; @Override public String getAction() { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 105eb67fc6cd..4fb20e8a6259 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -324,6 +324,10 @@ private OMConfigKeys() { "ozone.snapshot.filtering.limit.per.task"; public static final int SNAPSHOT_SST_DELETING_LIMIT_PER_TASK_DEFAULT = 2; + public static final String SNAPSHOT_DELETING_LIMIT_PER_TASK = + "ozone.snapshot.deleting.limit.per.task"; + public static final int SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT = 10; + public static final String OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL = "ozone.snapshot.filtering.service.interval"; public static final String diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index b810a41bb0fa..afa961b83812 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -129,6 +129,7 @@ enum Type { CreateSnapshot = 112; ListSnapshot = 113; SnapshotDiff = 114; + SnapshotMoveDeletedKeys = 115; } message OMRequest { @@ -240,6 +241,7 @@ message OMRequest { optional CreateSnapshotRequest CreateSnapshotRequest = 112; optional ListSnapshotRequest ListSnapshotRequest = 113; optional SnapshotDiffRequest snapshotDiffRequest = 114; + optional SnapshotMoveDeletedKeysRequest SnapshotMoveDeletedKeysRequest = 115; } @@ -345,6 +347,7 @@ message OMResponse { optional CreateSnapshotResponse CreateSnapshotResponse = 112; optional ListSnapshotResponse ListSnapshotResponse = 113; optional SnapshotDiffResponse snapshotDiffResponse = 114; + optional SnapshotMoveDeletedKeysResponse SnapshotMoveDeletedKeysResponse = 115; } enum Status { @@ -1680,6 +1683,18 @@ message SnapshotDiffRequest { required string toSnapshot = 4; } +message SnapshotMoveDeletedKeysRequest { + required SnapshotInfo fromSnapshot = 1; + optional SnapshotInfo nextSnapshot = 2; + repeated KeyValuePair activeDBKeys = 3; + repeated KeyValuePair nextDBKeys = 4; +} + +message KeyValuePair { + required string key = 1; + repeated KeyInfo keyInfos = 2; +} + message DeleteTenantRequest { optional string tenantId = 1; } @@ -1730,6 +1745,10 @@ message SnapshotDiffResponse { required SnapshotDiffReportProto snapshotDiffReport = 1; } +message SnapshotMoveDeletedKeysResponse { + +} + message SnapshotDiffReportProto { required string volumeName = 1; required string bucketName = 2; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index c14e97ea2b58..4785128070f0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -248,4 +248,10 @@ List getPendingDeletionSubFiles(long volumeId, * @return Background service. */ BackgroundService getSnapshotSstFilteringService(); + + /** + * Returns the instance of Snapshot Deleting service. + * @return Background service. + */ + BackgroundService getSnapshotDeletingService(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index ad86a9ed76ae..65b15fc01738 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -84,6 +84,7 @@ import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.OpenKeyCleanupService; +import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OpenKeyBucket; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager; @@ -111,6 +112,10 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; @@ -160,6 +165,7 @@ public class KeyManagerImpl implements KeyManager { private BackgroundService keyDeletingService; private BackgroundService snapshotSstFilteringService; + private BackgroundService snapshotDeletingService; private final KeyProviderCryptoExtension kmsProvider; private final boolean enableFileSystemPaths; @@ -262,6 +268,24 @@ public void start(OzoneConfiguration configuration) { snapshotSstFilteringService.start(); } } + + if (snapshotDeletingService == null) { + long snapshotServiceInterval = configuration.getTimeDuration( + OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL, + OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); + long snapshotServiceTimeout = configuration.getTimeDuration( + OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT, + OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT_DEFAULT, + TimeUnit.MILLISECONDS); + try { + snapshotDeletingService = new SnapshotDeletingService( + snapshotServiceInterval, snapshotServiceTimeout, ozoneManager); + snapshotDeletingService.start(); + } catch (IOException e) { + LOG.error("Error starting Snapshot Deleting Service", e); + } + } } KeyProviderCryptoExtension getKMSProvider() { @@ -286,6 +310,10 @@ public void stop() throws IOException { snapshotSstFilteringService.shutdown(); snapshotSstFilteringService = null; } + if (snapshotDeletingService != null) { + snapshotDeletingService.shutdown(); + snapshotDeletingService = null; + } } private OmBucketInfo getBucketInfo(String volumeName, String bucketName) @@ -622,6 +650,10 @@ public BackgroundService getSnapshotSstFilteringService() { return snapshotSstFilteringService; } + public BackgroundService getSnapshotDeletingService() { + return snapshotDeletingService; + } + @Override public OmMultipartUploadList listMultipartUploads(String volumeName, String bucketName, String prefix) throws OMException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 0b0051e49811..682250309350 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -206,7 +206,7 @@ public SnapshotInfo getSnapshotInfo(String volumeName, bucketName, snapshotName)); } - private SnapshotInfo getSnapshotInfo(String key) throws IOException { + public SnapshotInfo getSnapshotInfo(String key) throws IOException { SnapshotInfo snapshotInfo; try { snapshotInfo = ozoneManager.getMetadataManager() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 8eef32452302..0e0293108e99 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -431,6 +431,7 @@ private enum State { // This metadata reader points to the active filesystem private OmMetadataReader omMetadataReader; private OmSnapshotManager omSnapshotManager; + private SnapshotChainManager snapshotChainManager; @SuppressWarnings("methodlength") private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) @@ -726,6 +727,7 @@ private void instantiateServices(boolean withNewSnapshot) throws IOException { omMetadataReader = new OmMetadataReader(keyManager, prefixManager, this, LOG, AUDIT, metrics); omSnapshotManager = new OmSnapshotManager(this); + snapshotChainManager = new SnapshotChainManager(metadataManager); // Snapshot metrics updateActiveSnapshotMetrics(); @@ -1430,6 +1432,15 @@ public OmSnapshotManager getOmSnapshotManager() { return omSnapshotManager; } + /** + * Get Snapshot Chain Manager. + * + * @return SnapshotChainManager. + */ + public SnapshotChainManager getSnapshotChainManager() { + return snapshotChainManager; + } + /** * Get metadata manager. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java index a8fb867401ac..3a690c0e2d09 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java @@ -49,6 +49,7 @@ public class SnapshotChainManager { snapshotChainPath; private Map latestPathSnapshotID; private String latestGlobalSnapshotID; + private Map snapshotPathToKey; private static final Logger LOG = LoggerFactory.getLogger(SnapshotChainManager.class); @@ -57,6 +58,7 @@ public SnapshotChainManager(OMMetadataManager metadataManager) snapshotChainGlobal = new LinkedHashMap<>(); snapshotChainPath = new HashMap<>(); latestPathSnapshotID = new HashMap<>(); + snapshotPathToKey = new HashMap<>(); latestGlobalSnapshotID = null; loadFromSnapshotInfoTable(metadataManager); } @@ -91,8 +93,8 @@ private void addSnapshotGlobal(String snapshotID, }; private void addSnapshotPath(String snapshotPath, - String snapshotID, - String prevPathID) throws IOException { + String snapshotID, String prevPathID, String snapTableKey) + throws IOException { // set previous snapshotID to null if it is "" for // internal in-mem structure if (prevPathID != null && prevPathID.isEmpty()) { @@ -131,6 +133,7 @@ private void addSnapshotPath(String snapshotPath, new SnapshotChainInfo(snapshotID, prevPathID, null)); // set state variable latestPath snapshot entry to this snapshotID + snapshotPathToKey.put(snapshotID, snapTableKey); latestPathSnapshotID.put(snapshotPath, snapshotID); }; @@ -246,7 +249,7 @@ private boolean deleteSnapshotPath(String snapshotPath, return status; } - private void loadFromSnapshotInfoTable(OMMetadataManager metadataManager) + public void loadFromSnapshotInfoTable(OMMetadataManager metadataManager) throws IOException { // read from snapshotInfo table to populate // snapshot chains - both global and local path @@ -275,7 +278,8 @@ public void addSnapshot(SnapshotInfo sinfo) throws IOException { sinfo.getGlobalPreviousSnapshotID()); addSnapshotPath(sinfo.getSnapshotPath(), sinfo.getSnapshotID(), - sinfo.getPathPreviousSnapshotID()); + sinfo.getPathPreviousSnapshotID(), + sinfo.getTableKey()); } /** @@ -503,6 +507,10 @@ public String previousPathSnapshot(String snapshotPath, String snapshotID) .getPreviousSnapshotID(); } + public String getTableKey(String snapshotPath) { + return snapshotPathToKey.get(snapshotPath); + } + @VisibleForTesting public void loadSnapshotInfo(OMMetadataManager metadataManager) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index ad730cd5bcb9..b984467e9ab4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -71,6 +71,7 @@ import org.apache.hadoop.ozone.om.request.security.OMGetDelegationTokenRequest; import org.apache.hadoop.ozone.om.request.security.OMRenewDelegationTokenRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; +import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotMoveDeletedKeysRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMCancelPrepareRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMFinalizeUpgradeRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMPrepareRequest; @@ -212,6 +213,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMSetRangerServiceVersionRequest(omRequest); case CreateSnapshot: return new OMSnapshotCreateRequest(omRequest); + case SnapshotMoveDeletedKeys: + return new OMSnapshotMoveDeletedKeysRequest(omRequest); case DeleteOpenKeys: BucketLayout bktLayout = BucketLayout.DEFAULT; if (omRequest.getDeleteOpenKeysRequest().hasBucketLayout()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index 3f1af514c1e3..f00192cedec6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; @@ -114,7 +115,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, boolean acquiredBucketLock = false, acquiredSnapshotLock = false; IOException exception = null; OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); - + SnapshotChainManager snapshotChainManager = + ozoneManager.getSnapshotChainManager(); + OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); OMClientResponse omClientResponse = null; @@ -146,6 +149,26 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, .getLatestSequenceNumber(); snapshotInfo.setDbTxSequenceNumber(dbLatestSequenceNumber); + // Set previous path and global snapshot + String latestPathSnapshot = + snapshotChainManager.getLatestPathSnapshot(snapshotPath); + String latestGlobalSnapshot = + snapshotChainManager.getLatestGlobalSnapshot(); + + if (latestPathSnapshot == null || latestPathSnapshot.isEmpty()) { + snapshotInfo.setPathPreviousSnapshotID(""); + } else { + snapshotInfo.setPathPreviousSnapshotID(latestPathSnapshot); + } + + if (latestGlobalSnapshot == null || latestGlobalSnapshot.isEmpty()) { + snapshotInfo.setGlobalPreviousSnapshotID(""); + } else { + snapshotInfo.setGlobalPreviousSnapshotID(latestGlobalSnapshot); + } + + snapshotChainManager.addSnapshot(snapshotInfo); + omMetadataManager.getSnapshotInfoTable() .addCacheEntry(new CacheKey<>(key), new CacheValue<>(Optional.of(snapshotInfo), transactionLogIndex)); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java new file mode 100644 index 000000000000..64e63f54a209 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.request.snapshot; + +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveDeletedKeysResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyValuePair; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; + +/** + * Handles OMSnapshotMoveDeletedKeys Request. + */ +public class OMSnapshotMoveDeletedKeysRequest extends OMClientRequest { + + private static final Logger LOG = + LoggerFactory.getLogger(OMSnapshotMoveDeletedKeysRequest.class); + + public OMSnapshotMoveDeletedKeysRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, + long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { + OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager(); + + SnapshotMoveDeletedKeysRequest moveDeletedKeysRequest = + getOmRequest().getSnapshotMoveDeletedKeysRequest(); + + SnapshotInfo fromSnapshot = SnapshotInfo + .getFromProtobuf(moveDeletedKeysRequest.getFromSnapshot()); + + // If there is no Non-Deleted Snapshot move the + // keys to Active Object Store. + SnapshotInfo nextSnapshot = null; + if (moveDeletedKeysRequest.hasNextSnapshot()) { + nextSnapshot = SnapshotInfo + .getFromProtobuf(moveDeletedKeysRequest.getNextSnapshot()); + } + + List activeDBKeysList = + moveDeletedKeysRequest.getActiveDBKeysList(); + List nextDBKeysList = + moveDeletedKeysRequest.getNextDBKeysList(); + + OmSnapshot omFromSnapshot = null; + OmSnapshot omNextSnapshot = null; + + try { + omFromSnapshot = (OmSnapshot) omSnapshotManager + .checkForSnapshot(fromSnapshot.getVolumeName(), + fromSnapshot.getBucketName(), + getSnapshotPrefix(fromSnapshot.getName())); + + if (nextSnapshot != null) { + omNextSnapshot = (OmSnapshot) omSnapshotManager + .checkForSnapshot(nextSnapshot.getVolumeName(), + nextSnapshot.getBucketName(), + getSnapshotPrefix(nextSnapshot.getName())); + } + } catch (IOException ex) { + LOG.error("Error occurred when moving keys between snapshots", ex); + } + + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = + OmResponseUtil.getOMResponseBuilder( + getOmRequest()); + + OMClientResponse omClientResponse = + new OMSnapshotMoveDeletedKeysResponse(omResponse.build(), + omFromSnapshot, omNextSnapshot, activeDBKeysList, nextDBKeysList); + + addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, + omDoubleBufferHelper); + + return omClientResponse; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java new file mode 100644 index 000000000000..4050b38f564b --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.om.response.snapshot; + +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyValuePair; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; + +import java.io.IOException; +import java.util.List; + +/** + * Response for OMSnapshotMoveDeletedKeysRequest. + */ +public class OMSnapshotMoveDeletedKeysResponse extends OMClientResponse { + + private OmSnapshot fromSnapshot; + private OmSnapshot nextSnapshot; + private List activeDBKeysList; + private List nextDBKeysList; + + public OMSnapshotMoveDeletedKeysResponse(OMResponse omResponse, + OmSnapshot omFromSnapshot, OmSnapshot omNextSnapshot, + List activeDBKeysList, List nextDBKeysList) { + super(omResponse); + this.fromSnapshot = omFromSnapshot; + this.nextSnapshot = omNextSnapshot; + this.activeDBKeysList = activeDBKeysList; + this.nextDBKeysList = nextDBKeysList; + } + + @Override + protected void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) throws IOException { + + for (KeyValuePair activeDBKey : activeDBKeysList) { + RepeatedOmKeyInfo activeDBOmKeyInfo = + createRepeatedOmKeyInfo(activeDBKey.getKeyInfosList()); + + if (activeDBOmKeyInfo == null) { + continue; + } + + omMetadataManager.getDeletedTable().putWithBatch( + batchOperation, activeDBKey.getKey(), activeDBOmKeyInfo); + } + + for (KeyValuePair nextDBKey : nextDBKeysList) { + RepeatedOmKeyInfo nextDBOmKeyInfo = + createRepeatedOmKeyInfo(nextDBKey.getKeyInfosList()); + + if (nextDBOmKeyInfo == null) { + continue; + } + + if (nextSnapshot != null) { + nextSnapshot.getMetadataManager() + .getDeletedTable().putWithBatch(batchOperation, + nextDBKey.getKey(), nextDBOmKeyInfo); + } else { + omMetadataManager.getDeletedTable() + .put(nextDBKey.getKey(), nextDBOmKeyInfo); + } + } + } + + private RepeatedOmKeyInfo createRepeatedOmKeyInfo(List keyInfosList) + throws IOException { + RepeatedOmKeyInfo result = null; + + for (KeyInfo keyInfo: keyInfosList) { + if (result == null) { + result = new RepeatedOmKeyInfo(OmKeyInfo.getFromProtobuf(keyInfo)); + } else { + result.addOmKeyInfo(OmKeyInfo.getFromProtobuf(keyInfo)); + } + } + + return result; + } +} \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java new file mode 100644 index 000000000000..e8e051886d94 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -0,0 +1,378 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.om.service; + +import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.ServiceException; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.BackgroundService; +import org.apache.hadoop.hdds.utils.BackgroundTask; +import org.apache.hadoop.hdds.utils.BackgroundTaskQueue; +import org.apache.hadoop.hdds.utils.BackgroundTaskResult; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyValuePair; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.ratis.protocol.ClientId; +import org.apache.ratis.protocol.Message; +import org.apache.ratis.protocol.RaftClientRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK; +import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT; +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; + +/** + * Background Service to clean-up deleted snapshot and reclaim space. + */ +public class SnapshotDeletingService extends BackgroundService { + private static final Logger LOG = + LoggerFactory.getLogger(SnapshotDeletingService.class); + + // Use only a single thread for Snapshot Deletion. Multiple threads would read + // from the same table and can send deletion requests for same snapshot + // multiple times. + private static final int SNAPSHOT_DELETING_CORE_POOL_SIZE = 1; + private final ClientId clientId = ClientId.randomId(); + private final AtomicLong runCount; + + private final OzoneManager ozoneManager; + private final OmSnapshotManager omSnapshotManager; + private final SnapshotChainManager chainManager; + private final AtomicBoolean suspended; + private final OzoneConfiguration conf; + private final AtomicLong successRunCount; + private final long snapshotDeletionPerTask; + + public SnapshotDeletingService(long interval, long serviceTimeout, + OzoneManager ozoneManager) throws IOException { + super("SnapshotDeletingService", interval, TimeUnit.MILLISECONDS, + SNAPSHOT_DELETING_CORE_POOL_SIZE, serviceTimeout); + this.ozoneManager = ozoneManager; + this.omSnapshotManager = ozoneManager.getOmSnapshotManager(); + this.chainManager = ozoneManager.getSnapshotChainManager(); + this.runCount = new AtomicLong(0); + this.successRunCount = new AtomicLong(0); + this.suspended = new AtomicBoolean(false); + this.conf = ozoneManager.getConfiguration(); + this.snapshotDeletionPerTask = conf + .getLong(SNAPSHOT_DELETING_LIMIT_PER_TASK, + SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT); + } + + private class SnapshotDeletingTask implements BackgroundTask { + + @Override + public BackgroundTaskResult call() throws Exception { + if (!shouldRun()) { + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + + runCount.incrementAndGet(); + + Table snapshotInfoTable = + ozoneManager.getMetadataManager().getSnapshotInfoTable(); + try (TableIterator> iterator = snapshotInfoTable.iterator()) { + iterator.seekToFirst(); + + long snapshotLimit = snapshotDeletionPerTask; + + while (iterator.hasNext() && snapshotLimit > 0) { + SnapshotInfo snapInfo = iterator.next().getValue(); + SnapshotInfo.SnapshotStatus snapshotStatus = + snapInfo.getSnapshotStatus(); + + // Only Iterate in deleted snapshot + if (!snapshotStatus.equals( + SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED)) { + continue; + } + + OmSnapshot omSnapshot = (OmSnapshot) omSnapshotManager + .checkForSnapshot(snapInfo.getVolumeName(), + snapInfo.getBucketName(), + getSnapshotPrefix(snapInfo.getName())); + + Table snapshotDeletedTable = + omSnapshot.getMetadataManager().getDeletedTable(); + + if (snapshotDeletedTable.getEstimatedKeyCount() == 0) { + continue; + } + + // Get bucketInfo for the snapshot bucket to get bucket layout. + String dbBucketKey = ozoneManager.getMetadataManager().getBucketKey( + snapInfo.getVolumeName(), snapInfo.getBucketName()); + OmBucketInfo bucketInfo = ozoneManager.getMetadataManager() + .getBucketTable().get(dbBucketKey); + + + chainManager.loadFromSnapshotInfoTable( + ozoneManager.getMetadataManager()); + SnapshotInfo nextSnapshot = getNextNonDeletedSnapshot(snapInfo); + + SnapshotInfo previousSnapshot = getPreviousSnapshot(snapInfo); + Table previousKeyTable = null; + OmSnapshot omPreviousSnapshot = null; + + // Handle case when the deleted snapshot is the first snapshot. + // Move deleted keys to activeDB's deletedKeyTable + if (previousSnapshot != null) { + omPreviousSnapshot = (OmSnapshot) omSnapshotManager + .checkForSnapshot(previousSnapshot.getVolumeName(), + previousSnapshot.getBucketName(), + getSnapshotPrefix(previousSnapshot.getName())); + + previousKeyTable = omPreviousSnapshot + .getMetadataManager().getKeyTable(bucketInfo.getBucketLayout()); + } + + // Move key to either next non deleted snapshot's snapshotDeletedTable + // or move to active object store deleted table + + List toActiveDBList = new ArrayList<>(); + List toNextDBList = new ArrayList<>(); + + try (TableIterator> deletedIterator = snapshotDeletedTable + .iterator()) { + + iterator.seekToFirst(); + + while (deletedIterator.hasNext()) { + Table.KeyValue + deletedKeyValue = deletedIterator.next(); + RepeatedOmKeyInfo repeatedOmKeyInfo = deletedKeyValue.getValue(); + + KeyValuePair.Builder toActiveDb = KeyValuePair.newBuilder() + .setKey(deletedKeyValue.getKey()); + KeyValuePair.Builder toNextDb = KeyValuePair.newBuilder() + .setKey(deletedKeyValue.getKey()); + + for (OmKeyInfo keyInfo: repeatedOmKeyInfo.getOmKeyInfoList()) { + splitRepeatedOmKeyInfo(toActiveDb, toNextDb, + keyInfo, previousKeyTable); + } + + toActiveDBList.add(toActiveDb.build()); + toNextDBList.add(toNextDb.build()); + + } + // Submit Move request to OM. + submitSnapshotMoveDeletedKeys(snapInfo, nextSnapshot, + toActiveDBList, toNextDBList); + snapshotLimit--; + } catch (IOException ex) { + LOG.error("Error while running Snapshot Deleting Service", ex); + } + } + + successRunCount.incrementAndGet(); + } catch (IOException e) { + LOG.error("Error while running Snapshot Deleting Service", e); + } + + return BackgroundTaskResult.EmptyTaskResult.newResult(); + } + + private void splitRepeatedOmKeyInfo(KeyValuePair.Builder toActiveDb, + KeyValuePair.Builder toNextDb, OmKeyInfo keyInfo, + Table previousKeyTable) throws IOException { + if (checkKeyExistInPreviousTable(previousKeyTable, keyInfo)) { + // Move to next non deleted snapshot's deleted table + toNextDb.addKeyInfos(keyInfo.getProtobuf( + ClientVersion.CURRENT_VERSION)); + } else { + // Move to active DB Deleted Table. + toActiveDb.addKeyInfos(keyInfo + .getProtobuf(ClientVersion.CURRENT_VERSION)); + } + } + + private void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, + SnapshotInfo nextSnapshot, List toActiveDBList, + List toNextDBList) { + + SnapshotMoveDeletedKeysRequest.Builder moveDeletedKeysBuilder = + SnapshotMoveDeletedKeysRequest.newBuilder() + .setFromSnapshot(snapInfo.getProtobuf()); + + if (nextSnapshot != null) { + moveDeletedKeysBuilder.setNextSnapshot(nextSnapshot.getProtobuf()); + } + + SnapshotMoveDeletedKeysRequest moveDeletedKeys = + moveDeletedKeysBuilder.addAllActiveDBKeys(toActiveDBList) + .addAllNextDBKeys(toNextDBList).build(); + + + OMRequest omRequest = OMRequest.newBuilder() + .setCmdType(Type.SnapshotMoveDeletedKeys) + .setSnapshotMoveDeletedKeysRequest(moveDeletedKeys) + .setClientId(clientId.toString()) + .build(); + + submitRequest(omRequest); + } + + private boolean checkKeyExistInPreviousTable( + Table previousKeyTable, OmKeyInfo deletedKeyInfo) + throws IOException { + + if (previousKeyTable == null) { + return false; + } + + //TODO: Handle Renamed Keys + String dbKey = ozoneManager.getMetadataManager() + .getOzoneKey(deletedKeyInfo.getVolumeName(), + deletedKeyInfo.getBucketName(), deletedKeyInfo.getKeyName()); + + OmKeyInfo prevKeyInfo = previousKeyTable.get(dbKey); + if (prevKeyInfo != null && + prevKeyInfo.getObjectID() == deletedKeyInfo.getObjectID()) { + return true; + } + return false; + } + + private SnapshotInfo getPreviousSnapshot(SnapshotInfo snapInfo) + throws IOException { + if (chainManager.hasPreviousPathSnapshot(snapInfo.getSnapshotPath(), + snapInfo.getSnapshotID())) { + String previousPathSnapshot = chainManager.previousPathSnapshot( + snapInfo.getSnapshotPath(), snapInfo.getSnapshotID()); + String tableKey = chainManager.getTableKey(previousPathSnapshot); + return omSnapshotManager.getSnapshotInfo(tableKey); + } + return null; + } + + /** + * Get the next non deleted snapshot in the snapshot chain. + */ + private SnapshotInfo getNextNonDeletedSnapshot(SnapshotInfo snapInfo) + throws IOException { + while (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), + snapInfo.getSnapshotID())) { + + String nextPathSnapshot = + chainManager.nextPathSnapshot( + snapInfo.getSnapshotPath(), snapInfo.getSnapshotID()); + + String tableKey = chainManager.getTableKey(nextPathSnapshot); + SnapshotInfo nextSnapshotInfo = + omSnapshotManager.getSnapshotInfo(tableKey); + + if (nextSnapshotInfo.getSnapshotStatus().equals( + SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)) { + return nextSnapshotInfo; + } + } + return null; + } + + private void submitRequest(OMRequest omRequest) { + try { + if (isRatisEnabled()) { + OzoneManagerRatisServer server = ozoneManager.getOmRatisServer(); + + RaftClientRequest raftClientRequest = RaftClientRequest.newBuilder() + .setClientId(clientId) + .setServerId(server.getRaftPeerId()) + .setGroupId(server.getRaftGroupId()) + .setCallId(runCount.get()) + .setMessage(Message.valueOf( + OMRatisHelper.convertRequestToByteString(omRequest))) + .setType(RaftClientRequest.writeRequestType()) + .build(); + + server.submitRequest(omRequest, raftClientRequest); + } else { + ozoneManager.getOmServerProtocol().submitRequest(null, omRequest); + } + } catch (ServiceException e) { + LOG.error("Snapshot Deleting request failed. " + + "Will retry at next run.", e); + } + } + + private boolean isRatisEnabled() { + return ozoneManager.isRatisEnabled(); + } + + } + + @Override + public BackgroundTaskQueue getTasks() { + BackgroundTaskQueue queue = new BackgroundTaskQueue(); + queue.add(new SnapshotDeletingTask()); + return queue; + } + + private boolean shouldRun() { + return !suspended.get() && ozoneManager.isLeaderReady(); + } + + /** + * Suspend the service (for testing). + */ + @VisibleForTesting + public void suspend() { + suspended.set(true); + } + + /** + * Resume the service if suspended (for testing). + */ + @VisibleForTesting + public void resume() { + suspended.set(false); + } + + public long getRunCount() { + return runCount.get(); + } + + public long getSuccessfulRunCount() { + return successRunCount.get(); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java new file mode 100644 index 000000000000..1fbf637c5d0c --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.service; + +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; +import org.apache.hadoop.ozone.om.KeyManager; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmTestManagers; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.helpers.OpenKeySession; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.test.PathUtils; +import org.apache.ozone.test.GenericTestUtils; +import org.apache.ratis.util.ExitUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Test Snapshot Deleting Service. + */ +public class TestSnapshotDeletingService { + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + private OzoneManagerProtocol writeClient; + private OzoneManager om; + private static final Logger LOG = + LoggerFactory.getLogger(SnapshotDeletingService.class); + + private KeyManager keyManager; + private OMMetadataManager omMetadataManager; + private OzoneConfiguration conf; + private OmTestManagers omTestManagers; + private static final String VOLUME_NAME = "vol1"; + private static final String BUCKET_NAME = "bucket1"; + + + @BeforeClass + public static void setup() { + ExitUtils.disableSystemExit(); + } + + @Before + public void createConfAndInitValues() throws Exception { + conf = new OzoneConfiguration(); + File testDir = PathUtils.getTestDir(TestSnapshotDeletingService.class); + System.setProperty(DBConfigFromFile.CONFIG_DIR, "/"); + ServerUtils.setOzoneMetaDirPath(conf, testDir.getPath()); + conf.setTimeDuration(OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL, + 1000, TimeUnit.MILLISECONDS); + conf.setTimeDuration(OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT, + 100000, TimeUnit.MILLISECONDS); + conf.setQuietMode(false); + omTestManagers = new OmTestManagers(conf); + keyManager = omTestManagers.getKeyManager(); + omMetadataManager = omTestManagers.getMetadataManager(); + writeClient = omTestManagers.getWriteClient(); + om = omTestManagers.getOzoneManager(); + } + + @After + public void cleanup() throws Exception { + om.stop(); + } + + @Test + public void testSnapshotKeySpaceReclaim() throws Exception { + int snapshotCount = 0; + + SnapshotDeletingService snapshotDeletingService = (SnapshotDeletingService) + keyManager.getSnapshotDeletingService(); + + snapshotDeletingService.suspend(); + + OmKeyArgs key1 = createVolumeBucketKey(VOLUME_NAME, BUCKET_NAME, + BucketLayout.DEFAULT, "key1"); + + createSnapshot(VOLUME_NAME, BUCKET_NAME, "snap1", ++snapshotCount); + + OmKeyArgs key2 = createKey(VOLUME_NAME, BUCKET_NAME, "key2"); + + // Key 1 cannot be deleted as it is still referenced by Snapshot 1. + writeClient.deleteKey(key1); + //Key 2 is deleted here, which means we can reclaim + // it when snapshot 2 is deleted. + writeClient.deleteKey(key2); + + createSnapshot(VOLUME_NAME, BUCKET_NAME, "snap2", ++snapshotCount); + createKey(VOLUME_NAME, BUCKET_NAME, "key4"); + OmKeyArgs key5 = createKey(VOLUME_NAME, BUCKET_NAME, "key5"); + writeClient.deleteKey(key5); + + createSnapshot(VOLUME_NAME, BUCKET_NAME, "snap3", ++snapshotCount); + + + String snapshotKey2 = "/vol1/bucket1/snap2"; + SnapshotInfo snapshotInfo = om.getMetadataManager() + .getSnapshotInfoTable().get(snapshotKey2); + + snapshotInfo + .setSnapshotStatus(SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); + om.getMetadataManager() + .getSnapshotInfoTable().put(snapshotKey2, snapshotInfo); + snapshotInfo = om.getMetadataManager() + .getSnapshotInfoTable().get(snapshotKey2); + assertEquals(snapshotInfo.getSnapshotStatus(), + SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); + + snapshotDeletingService.resume(); + + GenericTestUtils.waitFor(() -> + snapshotDeletingService.getSuccessfulRunCount() >= 1, + 1000, 10000); + + OmSnapshot nextSnapshot = (OmSnapshot) om.getOmSnapshotManager() + .checkForSnapshot(VOLUME_NAME, BUCKET_NAME, getSnapshotPrefix("snap3")); + + //Check key1 added to next non deleted snapshot db. + RepeatedOmKeyInfo omKeyInfo = + nextSnapshot.getMetadataManager() + .getDeletedTable().get("/vol1/bucket1/key1"); + assertNotNull(omKeyInfo); + + //Check key2 added active db as it can be reclaimed. + RepeatedOmKeyInfo omKeyInfo1 = omMetadataManager + .getDeletedTable().get("/vol1/bucket1/key2"); + + assertNotNull(omKeyInfo1); + + } + + private OmKeyArgs createVolumeBucketKey(String volumeName, String bucketName, + BucketLayout bucketLayout, String keyName) throws IOException { + // cheat here, just create a volume and bucket entry so that we can + // create the keys, we put the same data for key and value since the + // system does not decode the object + OMRequestTestUtils.addVolumeToOM(omMetadataManager, + OmVolumeArgs.newBuilder() + .setOwnerName("owner") + .setAdminName("admin") + .setVolume(volumeName) + .build()); + + OMRequestTestUtils.addBucketToOM(omMetadataManager, + OmBucketInfo.newBuilder().setVolumeName(volumeName) + .setBucketName(bucketName) + .setBucketLayout(bucketLayout) + .build()); + + return createKey(volumeName, bucketName, keyName); + } + + private OmKeyArgs createKey(String volumeName, String bucketName, + String keyName) throws IOException { + OmKeyArgs keyArg = + new OmKeyArgs.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setAcls(Collections.emptyList()) + .setReplicationConfig(StandaloneReplicationConfig.getInstance( + HddsProtos.ReplicationFactor.ONE)) + .setLocationInfoList(new ArrayList<>()) + .build(); + + // Open and write the key without commit it. + OpenKeySession session = writeClient.openKey(keyArg); + writeClient.commitKey(keyArg, session.getId()); + + return keyArg; + } + + private void createSnapshot(String volName, String bucketName, + String snapName, int count) throws Exception { + writeClient.createSnapshot(volName, bucketName, snapName); + + GenericTestUtils.waitFor(() -> { + try { + return omMetadataManager.getSnapshotInfoTable() + .getEstimatedKeyCount() >= count; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + }, 1000, 10000); + } +} From bbd106b5942271ab81fd3552c431af3c4b23a7b3 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Fri, 3 Feb 2023 02:12:32 +0530 Subject: [PATCH 02/20] HDDS-7740: Fix tests. --- .../snapshot/OMSnapshotMoveDeletedKeysResponse.java | 1 - .../om/request/snapshot/TestOMSnapshotCreateRequest.java | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index 3953eeeea345..ac47c8bd6e0b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -40,7 +40,6 @@ @CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) public class OMSnapshotMoveDeletedKeysResponse extends OMClientResponse { - private OmSnapshot fromSnapshot; private OmSnapshot nextSnapshot; private List activeDBKeysList; private List nextDBKeysList; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index 972920e46df8..9ee08ca204f8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; @@ -185,7 +186,11 @@ public void testPreExecuteNameLength() throws Exception { @Test public void testValidateAndUpdateCache() throws Exception { + SnapshotChainManager snapshotChainManager = + new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); + when(ozoneManager.getSnapshotChainManager()) + .thenReturn(snapshotChainManager); OMRequest omRequest = OMRequestTestUtils.createSnapshotRequest( volumeName, bucketName, snapshotName); @@ -224,7 +229,11 @@ public void testValidateAndUpdateCache() throws Exception { @Test public void testEntryExists() throws Exception { + SnapshotChainManager snapshotChainManager = + new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); + when(ozoneManager.getSnapshotChainManager()) + .thenReturn(snapshotChainManager); OMRequest omRequest = OMRequestTestUtils.createSnapshotRequest( volumeName, bucketName, snapshotName); From fb7b7aed61b9a9fb99260670503a5b8678f849ec Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Mon, 27 Feb 2023 13:34:37 -0800 Subject: [PATCH 03/20] HDDS-7740: Address review comments. --- .../src/main/resources/ozone-default.xml | 2 +- .../hadoop/ozone/om/helpers/SnapshotInfo.java | 34 +++++-- .../src/main/proto/OmClientProtocol.proto | 10 +- .../hadoop/ozone/om/SnapshotChainManager.java | 20 ++-- .../snapshot/OMSnapshotCreateRequest.java | 9 +- .../OMSnapshotMoveDeletedKeysRequest.java | 78 ++++++++++----- .../OMSnapshotMoveDeletedKeysResponse.java | 30 ++++-- .../om/service/SnapshotDeletingService.java | 97 ++++++++----------- .../snapshot/TestOMSnapshotDeleteRequest.java | 5 + .../service/TestSnapshotDeletingService.java | 33 +++---- 10 files changed, 184 insertions(+), 134 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index d579ad65fb8b..1269ad18a575 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3169,7 +3169,7 @@ ozone.snapshot.deleting.limit.per.task 10 OZONE, PERFORMANCE, OM - The number of snapshots to be reclaimed by the + The number of maximum snapshots to be reclaimed by the Snapshot Deleting Service per run. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index f15908dcd805..cf18a7180a31 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -368,10 +368,17 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() { .setBucketName(bucketName) .setSnapshotStatus(snapshotStatus.toProto()) .setCreationTime(creationTime) - .setDeletionTime(deletionTime) - .setPathPreviousSnapshotID(pathPreviousSnapshotID) - .setGlobalPreviousSnapshotID(globalPreviousSnapshotID) - .setSnapshotPath(snapshotPath) + .setDeletionTime(deletionTime); + + if (pathPreviousSnapshotID != null) { + sib.setPathPreviousSnapshotID(pathPreviousSnapshotID); + } + + if (globalPreviousSnapshotID != null) { + sib.setGlobalPreviousSnapshotID(globalPreviousSnapshotID); + } + + sib.setSnapshotPath(snapshotPath) .setCheckpointDir(checkpointDir) .setDbTxSequenceNumber(dbTxSequenceNumber); return sib.build(); @@ -392,12 +399,19 @@ public static SnapshotInfo getFromProtobuf( .setSnapshotStatus(SnapshotStatus.valueOf(snapshotInfoProto .getSnapshotStatus())) .setCreationTime(snapshotInfoProto.getCreationTime()) - .setDeletionTime(snapshotInfoProto.getDeletionTime()) - .setPathPreviousSnapshotID(snapshotInfoProto. - getPathPreviousSnapshotID()) - .setGlobalPreviousSnapshotID(snapshotInfoProto. - getGlobalPreviousSnapshotID()) - .setSnapshotPath(snapshotInfoProto.getSnapshotPath()) + .setDeletionTime(snapshotInfoProto.getDeletionTime()); + + if (snapshotInfoProto.hasPathPreviousSnapshotID()) { + osib.setPathPreviousSnapshotID(snapshotInfoProto. + getPathPreviousSnapshotID()); + } + + if (snapshotInfoProto.hasGlobalPreviousSnapshotID()) { + osib.setGlobalPreviousSnapshotID(snapshotInfoProto. + getGlobalPreviousSnapshotID()); + } + + osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()) .setCheckpointDir(snapshotInfoProto.getCheckpointDir()) .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber()); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index cbfafe251f77..c25cc6069ddf 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1695,14 +1695,14 @@ message DeleteSnapshotRequest { } message SnapshotMoveDeletedKeysRequest { - required SnapshotInfo fromSnapshot = 1; + optional SnapshotInfo fromSnapshot = 1; optional SnapshotInfo nextSnapshot = 2; - repeated KeyValuePair activeDBKeys = 3; - repeated KeyValuePair nextDBKeys = 4; + repeated SnapshotMoveKeyInfos activeDBKeys = 3; + repeated SnapshotMoveKeyInfos nextDBKeys = 4; } -message KeyValuePair { - required string key = 1; +message SnapshotMoveKeyInfos { + optional string key = 1; repeated KeyInfo keyInfos = 2; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java index 3a690c0e2d09..9e9766ff4fc5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java @@ -49,7 +49,7 @@ public class SnapshotChainManager { snapshotChainPath; private Map latestPathSnapshotID; private String latestGlobalSnapshotID; - private Map snapshotPathToKey; + private Map snapshotPathToTableKey; private static final Logger LOG = LoggerFactory.getLogger(SnapshotChainManager.class); @@ -58,11 +58,14 @@ public SnapshotChainManager(OMMetadataManager metadataManager) snapshotChainGlobal = new LinkedHashMap<>(); snapshotChainPath = new HashMap<>(); latestPathSnapshotID = new HashMap<>(); - snapshotPathToKey = new HashMap<>(); + snapshotPathToTableKey = new HashMap<>(); latestGlobalSnapshotID = null; loadFromSnapshotInfoTable(metadataManager); } + /** + * Add snapshot to global snapshot chain. + */ private void addSnapshotGlobal(String snapshotID, String prevGlobalID) throws IOException { // set previous snapshotID to null if it is "" for @@ -92,9 +95,13 @@ private void addSnapshotGlobal(String snapshotID, latestGlobalSnapshotID = snapshotID; }; + /** + * Add snapshot to bucket snapshot chain(path based). + */ private void addSnapshotPath(String snapshotPath, - String snapshotID, String prevPathID, String snapTableKey) - throws IOException { + String snapshotID, + String prevPathID, + String snapTableKey) throws IOException { // set previous snapshotID to null if it is "" for // internal in-mem structure if (prevPathID != null && prevPathID.isEmpty()) { @@ -132,8 +139,9 @@ private void addSnapshotPath(String snapshotPath, .put(snapshotID, new SnapshotChainInfo(snapshotID, prevPathID, null)); + // store snapshot ID to snapshot DB table key in the map + snapshotPathToTableKey.put(snapshotID, snapTableKey); // set state variable latestPath snapshot entry to this snapshotID - snapshotPathToKey.put(snapshotID, snapTableKey); latestPathSnapshotID.put(snapshotPath, snapshotID); }; @@ -508,7 +516,7 @@ public String previousPathSnapshot(String snapshotPath, String snapshotID) } public String getTableKey(String snapshotPath) { - return snapshotPathToKey.get(snapshotPath); + return snapshotPathToTableKey.get(snapshotPath); } @VisibleForTesting diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index be76c7f9b434..0222431b3789 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om.request.snapshot; import com.google.common.base.Optional; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; @@ -155,14 +156,14 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, String latestGlobalSnapshot = snapshotChainManager.getLatestGlobalSnapshot(); - if (latestPathSnapshot == null || latestPathSnapshot.isEmpty()) { - snapshotInfo.setPathPreviousSnapshotID(""); + if (StringUtils.isEmpty(latestPathSnapshot)) { + snapshotInfo.setPathPreviousSnapshotID(null); } else { snapshotInfo.setPathPreviousSnapshotID(latestPathSnapshot); } - if (latestGlobalSnapshot == null || latestGlobalSnapshot.isEmpty()) { - snapshotInfo.setGlobalPreviousSnapshotID(""); + if (StringUtils.isEmpty(latestGlobalSnapshot)) { + snapshotInfo.setGlobalPreviousSnapshotID(null); } else { snapshotInfo.setGlobalPreviousSnapshotID(latestGlobalSnapshot); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index 2c958e125c99..086b19645bc9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -22,6 +22,7 @@ import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.OMClientRequest; @@ -29,7 +30,7 @@ import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveDeletedKeysResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyValuePair; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; import org.slf4j.Logger; @@ -56,48 +57,77 @@ public OMSnapshotMoveDeletedKeysRequest(OMRequest omRequest) { public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager(); + SnapshotChainManager snapshotChainManager = + ozoneManager.getSnapshotChainManager(); SnapshotMoveDeletedKeysRequest moveDeletedKeysRequest = getOmRequest().getSnapshotMoveDeletedKeysRequest(); + SnapshotInfo fromSnapshot = SnapshotInfo.getFromProtobuf( + moveDeletedKeysRequest.getFromSnapshot()); // If there is no Non-Deleted Snapshot move the // keys to Active Object Store. SnapshotInfo nextSnapshot = null; - if (moveDeletedKeysRequest.hasNextSnapshot()) { - nextSnapshot = SnapshotInfo - .getFromProtobuf(moveDeletedKeysRequest.getNextSnapshot()); - } + OMClientResponse omClientResponse = null; + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = + OmResponseUtil.getOMResponseBuilder(getOmRequest()); + try { + nextSnapshot = getNextActiveSnapshot(fromSnapshot, + snapshotChainManager, omSnapshotManager); - List activeDBKeysList = - moveDeletedKeysRequest.getActiveDBKeysList(); - List nextDBKeysList = - moveDeletedKeysRequest.getNextDBKeysList(); + // Get next non-deleted snapshot. - OmSnapshot omFromSnapshot = null; - OmSnapshot omNextSnapshot = null; + List activeDBKeysList = + moveDeletedKeysRequest.getActiveDBKeysList(); + List nextDBKeysList = + moveDeletedKeysRequest.getNextDBKeysList(); + + OmSnapshot omNextSnapshot = null; - try { if (nextSnapshot != null) { omNextSnapshot = (OmSnapshot) omSnapshotManager .checkForSnapshot(nextSnapshot.getVolumeName(), nextSnapshot.getBucketName(), getSnapshotPrefix(nextSnapshot.getName())); } - } catch (IOException ex) { - LOG.error("Error occurred when moving keys between snapshots", ex); - } - - OzoneManagerProtocolProtos.OMResponse.Builder omResponse = - OmResponseUtil.getOMResponseBuilder( - getOmRequest()); - OMClientResponse omClientResponse = - new OMSnapshotMoveDeletedKeysResponse(omResponse.build(), - omNextSnapshot, activeDBKeysList, nextDBKeysList); + omClientResponse = new OMSnapshotMoveDeletedKeysResponse( + omResponse.build(), omNextSnapshot, activeDBKeysList, nextDBKeysList); - addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, - omDoubleBufferHelper); + } catch (IOException ex) { + omClientResponse = new OMSnapshotMoveDeletedKeysResponse( + createErrorOMResponse(omResponse, ex)); + } finally { + addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, + omDoubleBufferHelper); + } return omClientResponse; } + + /** + * Get the next non deleted snapshot in the snapshot chain. + */ + private SnapshotInfo getNextActiveSnapshot(SnapshotInfo snapInfo, + SnapshotChainManager chainManager, OmSnapshotManager omSnapshotManager) + throws IOException { + while (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), + snapInfo.getSnapshotID())) { + + String nextPathSnapshot = + chainManager.nextPathSnapshot( + snapInfo.getSnapshotPath(), snapInfo.getSnapshotID()); + + String tableKey = chainManager.getTableKey(nextPathSnapshot); + SnapshotInfo nextSnapshotInfo = + omSnapshotManager.getSnapshotInfo(tableKey); + + if (nextSnapshotInfo.getSnapshotStatus().equals( + SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)) { + return nextSnapshotInfo; + } + } + return null; + } } + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index ac47c8bd6e0b..6d38e35a9881 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -26,9 +26,10 @@ import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyValuePair; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.List; @@ -41,23 +42,32 @@ public class OMSnapshotMoveDeletedKeysResponse extends OMClientResponse { private OmSnapshot nextSnapshot; - private List activeDBKeysList; - private List nextDBKeysList; + private List activeDBKeysList; + private List nextDBKeysList; public OMSnapshotMoveDeletedKeysResponse(OMResponse omResponse, - OmSnapshot omNextSnapshot, List activeDBKeysList, - List nextDBKeysList) { + OmSnapshot omNextSnapshot, List activeDBKeysList, + List nextDBKeysList) { super(omResponse); this.nextSnapshot = omNextSnapshot; this.activeDBKeysList = activeDBKeysList; this.nextDBKeysList = nextDBKeysList; } + /** + * For when the request is not successful. + * For a successful request, the other constructor should be used. + */ + public OMSnapshotMoveDeletedKeysResponse(@Nonnull OMResponse omResponse) { + super(omResponse); + checkStatusNotOK(); + } + @Override protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - for (KeyValuePair activeDBKey : activeDBKeysList) { + for (SnapshotMoveKeyInfos activeDBKey : activeDBKeysList) { RepeatedOmKeyInfo activeDBOmKeyInfo = createRepeatedOmKeyInfo(activeDBKey.getKeyInfosList()); @@ -69,7 +79,7 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, batchOperation, activeDBKey.getKey(), activeDBOmKeyInfo); } - for (KeyValuePair nextDBKey : nextDBKeysList) { + for (SnapshotMoveKeyInfos nextDBKey : nextDBKeysList) { RepeatedOmKeyInfo nextDBOmKeyInfo = createRepeatedOmKeyInfo(nextDBKey.getKeyInfosList()); @@ -88,11 +98,11 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, } } - private RepeatedOmKeyInfo createRepeatedOmKeyInfo(List keyInfosList) + private RepeatedOmKeyInfo createRepeatedOmKeyInfo(List keyInfoList) throws IOException { RepeatedOmKeyInfo result = null; - for (KeyInfo keyInfo: keyInfosList) { + for (KeyInfo keyInfo: keyInfoList) { if (result == null) { result = new RepeatedOmKeyInfo(OmKeyInfo.getFromProtobuf(keyInfo)); } else { @@ -102,4 +112,4 @@ private RepeatedOmKeyInfo createRepeatedOmKeyInfo(List keyInfosList) return result; } -} \ No newline at end of file +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index e8e051886d94..a17261c0736c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -28,17 +28,19 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyValuePair; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; @@ -58,6 +60,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; /** * Background Service to clean-up deleted snapshot and reclaim space. @@ -83,8 +86,9 @@ public class SnapshotDeletingService extends BackgroundService { public SnapshotDeletingService(long interval, long serviceTimeout, OzoneManager ozoneManager) throws IOException { - super("SnapshotDeletingService", interval, TimeUnit.MILLISECONDS, - SNAPSHOT_DELETING_CORE_POOL_SIZE, serviceTimeout); + super(SnapshotDeletingService.class.getSimpleName(), interval, + TimeUnit.MILLISECONDS, SNAPSHOT_DELETING_CORE_POOL_SIZE, + serviceTimeout); this.ozoneManager = ozoneManager; this.omSnapshotManager = ozoneManager.getOmSnapshotManager(); this.chainManager = ozoneManager.getSnapshotChainManager(); @@ -111,7 +115,6 @@ public BackgroundTaskResult call() throws Exception { ozoneManager.getMetadataManager().getSnapshotInfoTable(); try (TableIterator> iterator = snapshotInfoTable.iterator()) { - iterator.seekToFirst(); long snapshotLimit = snapshotDeletionPerTask; @@ -134,7 +137,7 @@ public BackgroundTaskResult call() throws Exception { Table snapshotDeletedTable = omSnapshot.getMetadataManager().getDeletedTable(); - if (snapshotDeletedTable.getEstimatedKeyCount() == 0) { + if (snapshotDeletedTable.isEmpty()) { continue; } @@ -144,11 +147,12 @@ public BackgroundTaskResult call() throws Exception { OmBucketInfo bucketInfo = ozoneManager.getMetadataManager() .getBucketTable().get(dbBucketKey); + if (bucketInfo == null) { + throw new OMException("Bucket " + snapInfo.getBucketName() + + " is not found", BUCKET_NOT_FOUND); + } - chainManager.loadFromSnapshotInfoTable( - ozoneManager.getMetadataManager()); - SnapshotInfo nextSnapshot = getNextNonDeletedSnapshot(snapInfo); - + //TODO: Add lock to deletedTable and Active DB. SnapshotInfo previousSnapshot = getPreviousSnapshot(snapInfo); Table previousKeyTable = null; OmSnapshot omPreviousSnapshot = null; @@ -168,24 +172,34 @@ public BackgroundTaskResult call() throws Exception { // Move key to either next non deleted snapshot's snapshotDeletedTable // or move to active object store deleted table - List toActiveDBList = new ArrayList<>(); - List toNextDBList = new ArrayList<>(); + List toActiveDBList = new ArrayList<>(); + List toNextDBList = new ArrayList<>(); try (TableIterator> deletedIterator = snapshotDeletedTable .iterator()) { - iterator.seekToFirst(); + String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX; + iterator.seek(snapshotBucketKey); while (deletedIterator.hasNext()) { Table.KeyValue deletedKeyValue = deletedIterator.next(); + String deletedKey = deletedKeyValue.getKey(); + + // Exit if it is out of the bucket scope. + if (!deletedKey.contains(snapshotBucketKey)) { + break; + } + RepeatedOmKeyInfo repeatedOmKeyInfo = deletedKeyValue.getValue(); - KeyValuePair.Builder toActiveDb = KeyValuePair.newBuilder() - .setKey(deletedKeyValue.getKey()); - KeyValuePair.Builder toNextDb = KeyValuePair.newBuilder() - .setKey(deletedKeyValue.getKey()); + SnapshotMoveKeyInfos.Builder toActiveDb = SnapshotMoveKeyInfos + .newBuilder() + .setKey(deletedKey); + SnapshotMoveKeyInfos.Builder toNextDb = SnapshotMoveKeyInfos + .newBuilder() + .setKey(deletedKey); for (OmKeyInfo keyInfo: repeatedOmKeyInfo.getOmKeyInfoList()) { splitRepeatedOmKeyInfo(toActiveDb, toNextDb, @@ -197,15 +211,14 @@ public BackgroundTaskResult call() throws Exception { } // Submit Move request to OM. - submitSnapshotMoveDeletedKeys(snapInfo, nextSnapshot, - toActiveDBList, toNextDBList); + submitSnapshotMoveDeletedKeys(snapInfo, toActiveDBList, + toNextDBList); snapshotLimit--; + successRunCount.incrementAndGet(); } catch (IOException ex) { LOG.error("Error while running Snapshot Deleting Service", ex); } } - - successRunCount.incrementAndGet(); } catch (IOException e) { LOG.error("Error while running Snapshot Deleting Service", e); } @@ -213,8 +226,8 @@ public BackgroundTaskResult call() throws Exception { return BackgroundTaskResult.EmptyTaskResult.newResult(); } - private void splitRepeatedOmKeyInfo(KeyValuePair.Builder toActiveDb, - KeyValuePair.Builder toNextDb, OmKeyInfo keyInfo, + private void splitRepeatedOmKeyInfo(SnapshotMoveKeyInfos.Builder toActiveDb, + SnapshotMoveKeyInfos.Builder toNextDb, OmKeyInfo keyInfo, Table previousKeyTable) throws IOException { if (checkKeyExistInPreviousTable(previousKeyTable, keyInfo)) { // Move to next non deleted snapshot's deleted table @@ -228,17 +241,13 @@ private void splitRepeatedOmKeyInfo(KeyValuePair.Builder toActiveDb, } private void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, - SnapshotInfo nextSnapshot, List toActiveDBList, - List toNextDBList) { + List toActiveDBList, + List toNextDBList) { SnapshotMoveDeletedKeysRequest.Builder moveDeletedKeysBuilder = SnapshotMoveDeletedKeysRequest.newBuilder() .setFromSnapshot(snapInfo.getProtobuf()); - if (nextSnapshot != null) { - moveDeletedKeysBuilder.setNextSnapshot(nextSnapshot.getProtobuf()); - } - SnapshotMoveDeletedKeysRequest moveDeletedKeys = moveDeletedKeysBuilder.addAllActiveDBKeys(toActiveDBList) .addAllNextDBKeys(toNextDBList).build(); @@ -286,30 +295,6 @@ private SnapshotInfo getPreviousSnapshot(SnapshotInfo snapInfo) return null; } - /** - * Get the next non deleted snapshot in the snapshot chain. - */ - private SnapshotInfo getNextNonDeletedSnapshot(SnapshotInfo snapInfo) - throws IOException { - while (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), - snapInfo.getSnapshotID())) { - - String nextPathSnapshot = - chainManager.nextPathSnapshot( - snapInfo.getSnapshotPath(), snapInfo.getSnapshotID()); - - String tableKey = chainManager.getTableKey(nextPathSnapshot); - SnapshotInfo nextSnapshotInfo = - omSnapshotManager.getSnapshotInfo(tableKey); - - if (nextSnapshotInfo.getSnapshotStatus().equals( - SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE)) { - return nextSnapshotInfo; - } - } - return null; - } - private void submitRequest(OMRequest omRequest) { try { if (isRatisEnabled()) { @@ -353,18 +338,18 @@ private boolean shouldRun() { } /** - * Suspend the service (for testing). + * Suspend the service. */ @VisibleForTesting - public void suspend() { + void suspend() { suspended.set(true); } /** - * Resume the service if suspended (for testing). + * Resume the service if suspended. */ @VisibleForTesting - public void resume() { + void resume() { suspended.set(false); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java index 14b7955cbe1f..b8d7c74173b0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; @@ -258,6 +259,10 @@ public void testEntryNotExist() throws Exception { */ @Test public void testEntryExists() throws Exception { + SnapshotChainManager snapshotChainManager = + new SnapshotChainManager(omMetadataManager); + when(ozoneManager.getSnapshotChainManager()) + .thenReturn(snapshotChainManager); when(ozoneManager.isAdmin(any())).thenReturn(true); String key = SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java index a478dfab04b9..817b6c5fa307 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java @@ -41,14 +41,12 @@ import org.apache.hadoop.test.PathUtils; import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.util.ExitUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Rule; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.rules.TemporaryFolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -70,8 +68,6 @@ public class TestSnapshotDeletingService { public TemporaryFolder folder = new TemporaryFolder(); private OzoneManagerProtocol writeClient; private OzoneManager om; - private static final Logger LOG = - LoggerFactory.getLogger(SnapshotDeletingService.class); private KeyManager keyManager; private OMMetadataManager omMetadataManager; @@ -81,12 +77,12 @@ public class TestSnapshotDeletingService { private static final String BUCKET_NAME = "bucket1"; - @BeforeClass + @BeforeAll public static void setup() { ExitUtils.disableSystemExit(); } - @Before + @BeforeEach public void createConfAndInitValues() throws Exception { conf = new OzoneConfiguration(); File testDir = PathUtils.getTestDir(TestSnapshotDeletingService.class); @@ -104,9 +100,11 @@ public void createConfAndInitValues() throws Exception { om = omTestManagers.getOzoneManager(); } - @After + @AfterEach public void cleanup() throws Exception { - om.stop(); + if (om != null) { + om.stop(); + } } @Test @@ -125,9 +123,9 @@ public void testSnapshotKeySpaceReclaim() throws Exception { OmKeyArgs key2 = createKey(VOLUME_NAME, BUCKET_NAME, "key2"); - // Key 1 cannot be deleted as it is still referenced by Snapshot 1. + // Key 1 cannot be reclaimed as it is still referenced by Snapshot 1. writeClient.deleteKey(key1); - //Key 2 is deleted here, which means we can reclaim + // Key 2 is deleted here, which means we can reclaim // it when snapshot 2 is deleted. writeClient.deleteKey(key2); @@ -138,7 +136,6 @@ public void testSnapshotKeySpaceReclaim() throws Exception { createSnapshot(VOLUME_NAME, BUCKET_NAME, "snap3", ++snapshotCount); - String snapshotKey2 = "/vol1/bucket1/snap2"; SnapshotInfo snapshotInfo = om.getMetadataManager() .getSnapshotInfoTable().get(snapshotKey2); @@ -161,13 +158,13 @@ public void testSnapshotKeySpaceReclaim() throws Exception { OmSnapshot nextSnapshot = (OmSnapshot) om.getOmSnapshotManager() .checkForSnapshot(VOLUME_NAME, BUCKET_NAME, getSnapshotPrefix("snap3")); - //Check key1 added to next non deleted snapshot db. + // Check key1 added to next non deleted snapshot db. RepeatedOmKeyInfo omKeyInfo = nextSnapshot.getMetadataManager() .getDeletedTable().get("/vol1/bucket1/key1"); assertNotNull(omKeyInfo); - //Check key2 added active db as it can be reclaimed. + // Check key2 added active db as it can be reclaimed. RepeatedOmKeyInfo omKeyInfo1 = omMetadataManager .getDeletedTable().get("/vol1/bucket1/key2"); @@ -209,7 +206,7 @@ private OmKeyArgs createKey(String volumeName, String bucketName, .setLocationInfoList(new ArrayList<>()) .build(); - // Open and write the key without commit it. + // Open and write the key. OpenKeySession session = writeClient.openKey(keyArg); writeClient.commitKey(keyArg, session.getId()); From a31bd8c6c202b0e9a7bf38ab8e2486603a933a46 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Mon, 27 Feb 2023 15:02:53 -0800 Subject: [PATCH 04/20] HDDS-7740. Fix Unit test. --- .../org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index cf18a7180a31..bc297a92fd49 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -513,8 +513,9 @@ public boolean equals(Object o) { name.equals(that.name) && volumeName.equals(that.volumeName) && bucketName.equals(that.bucketName) && snapshotStatus == that.snapshotStatus && - pathPreviousSnapshotID.equals(that.pathPreviousSnapshotID) && - globalPreviousSnapshotID.equals(that.globalPreviousSnapshotID) && + Objects.equals(pathPreviousSnapshotID, that.pathPreviousSnapshotID) && + Objects.equals( + globalPreviousSnapshotID, that.globalPreviousSnapshotID) && snapshotPath.equals(that.snapshotPath) && checkpointDir.equals(that.checkpointDir); } From 2470c119af5610a2cc8a5d46e0f449f2ae9205ad Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Thu, 2 Mar 2023 00:09:06 -0800 Subject: [PATCH 05/20] HDDS-7740. Add Unit test. --- .../om/service/SnapshotDeletingService.java | 9 +- .../service/TestSnapshotDeletingService.java | 136 +++++++++++++----- 2 files changed, 110 insertions(+), 35 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index a17261c0736c..ef4edf0a0dd4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -152,7 +152,7 @@ public BackgroundTaskResult call() throws Exception { " is not found", BUCKET_NOT_FOUND); } - //TODO: Add lock to deletedTable and Active DB. + //TODO: [SNAPSHOT] Add lock to deletedTable and Active DB. SnapshotInfo previousSnapshot = getPreviousSnapshot(snapInfo); Table previousKeyTable = null; OmSnapshot omPreviousSnapshot = null; @@ -270,7 +270,7 @@ private boolean checkKeyExistInPreviousTable( return false; } - //TODO: Handle Renamed Keys + //TODO: [SNAPSHOT] Handle Renamed Keys String dbKey = ozoneManager.getMetadataManager() .getOzoneKey(deletedKeyInfo.getVolumeName(), deletedKeyInfo.getBucketName(), deletedKeyInfo.getKeyName()); @@ -360,4 +360,9 @@ public long getRunCount() { public long getSuccessfulRunCount() { return successRunCount.get(); } + + @VisibleForTesting + public void setSuccessRunCount(long num) { + successRunCount.getAndSet(num); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java index 817b6c5fa307..bd8ac683f028 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java @@ -43,6 +43,7 @@ import org.apache.ratis.util.ExitUtils; import org.junit.Rule; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -57,8 +58,6 @@ import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; /** * Test Snapshot Deleting Service. @@ -74,7 +73,8 @@ public class TestSnapshotDeletingService { private OzoneConfiguration conf; private OmTestManagers omTestManagers; private static final String VOLUME_NAME = "vol1"; - private static final String BUCKET_NAME = "bucket1"; + private static final String BUCKET_NAME_ONE = "bucket1"; + private static final String BUCKET_NAME_TWO = "bucket2"; @BeforeAll @@ -109,34 +109,65 @@ public void cleanup() throws Exception { @Test public void testSnapshotKeySpaceReclaim() throws Exception { - int snapshotCount = 0; - SnapshotDeletingService snapshotDeletingService = (SnapshotDeletingService) keyManager.getSnapshotDeletingService(); + // Suspending SnapshotDeletingService snapshotDeletingService.suspend(); + createSnapshotDataForBucket1(); + snapshotDeletingService.resume(); + + snapshotDeletingService.setSuccessRunCount(0); - OmKeyArgs key1 = createVolumeBucketKey(VOLUME_NAME, BUCKET_NAME, - BucketLayout.DEFAULT, "key1"); + GenericTestUtils.waitFor(() -> + snapshotDeletingService.getSuccessfulRunCount() >= 1, + 1000, 10000); - createSnapshot(VOLUME_NAME, BUCKET_NAME, "snap1", ++snapshotCount); + OmSnapshot nextSnapshot = (OmSnapshot) om.getOmSnapshotManager() + .checkForSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, + getSnapshotPrefix("bucket1snap3")); - OmKeyArgs key2 = createKey(VOLUME_NAME, BUCKET_NAME, "key2"); + // Check bucket1key1 added to next non deleted snapshot db. + RepeatedOmKeyInfo omKeyInfo = + nextSnapshot.getMetadataManager() + .getDeletedTable().get("/vol1/bucket1/bucket1key1"); + Assertions.assertNotNull(omKeyInfo); - // Key 1 cannot be reclaimed as it is still referenced by Snapshot 1. - writeClient.deleteKey(key1); - // Key 2 is deleted here, which means we can reclaim - // it when snapshot 2 is deleted. - writeClient.deleteKey(key2); + // Check bucket1key2 added active db as it can be reclaimed. + RepeatedOmKeyInfo omKeyInfo1 = omMetadataManager + .getDeletedTable().get("/vol1/bucket1/bucket1key2"); - createSnapshot(VOLUME_NAME, BUCKET_NAME, "snap2", ++snapshotCount); - createKey(VOLUME_NAME, BUCKET_NAME, "key4"); - OmKeyArgs key5 = createKey(VOLUME_NAME, BUCKET_NAME, "key5"); - writeClient.deleteKey(key5); + Assertions.assertNotNull(omKeyInfo1); - createSnapshot(VOLUME_NAME, BUCKET_NAME, "snap3", ++snapshotCount); + } + + @Test + public void testMultipleSnapshotKeyReclaim() throws Exception { + + SnapshotDeletingService snapshotDeletingService = (SnapshotDeletingService) + keyManager.getSnapshotDeletingService(); + + // Suspending SnapshotDeletingService + snapshotDeletingService.suspend(); + int snapshotCount = createSnapshotDataForBucket1(); + + OmKeyArgs bucket2key1 = createVolumeBucketKey(VOLUME_NAME, BUCKET_NAME_TWO, + BucketLayout.DEFAULT, "bucket2key1"); + + OmKeyArgs bucket2key2 = createKey(VOLUME_NAME, BUCKET_NAME_TWO, + "bucket2key2"); + + createSnapshot(VOLUME_NAME, BUCKET_NAME_TWO, "bucket2snap1", + ++snapshotCount); - String snapshotKey2 = "/vol1/bucket1/snap2"; + // Both key 1 and key 2 can be reclaimed when Snapshot 1 is deleted. + writeClient.deleteKey(bucket2key1); + writeClient.deleteKey(bucket2key2); + + createSnapshot(VOLUME_NAME, BUCKET_NAME_TWO, "bucket2snap2", + ++snapshotCount); + + String snapshotKey2 = "/vol1/bucket2/bucket2snap1"; SnapshotInfo snapshotInfo = om.getMetadataManager() .getSnapshotInfoTable().get(snapshotKey2); @@ -146,30 +177,26 @@ public void testSnapshotKeySpaceReclaim() throws Exception { .getSnapshotInfoTable().put(snapshotKey2, snapshotInfo); snapshotInfo = om.getMetadataManager() .getSnapshotInfoTable().get(snapshotKey2); - assertEquals(snapshotInfo.getSnapshotStatus(), + Assertions.assertEquals(snapshotInfo.getSnapshotStatus(), SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); snapshotDeletingService.resume(); + snapshotDeletingService.setSuccessRunCount(0L); GenericTestUtils.waitFor(() -> snapshotDeletingService.getSuccessfulRunCount() >= 1, 1000, 10000); - OmSnapshot nextSnapshot = (OmSnapshot) om.getOmSnapshotManager() - .checkForSnapshot(VOLUME_NAME, BUCKET_NAME, getSnapshotPrefix("snap3")); - - // Check key1 added to next non deleted snapshot db. - RepeatedOmKeyInfo omKeyInfo = - nextSnapshot.getMetadataManager() - .getDeletedTable().get("/vol1/bucket1/key1"); - assertNotNull(omKeyInfo); - - // Check key2 added active db as it can be reclaimed. + // Check bucket2key1 added active db as it can be reclaimed. RepeatedOmKeyInfo omKeyInfo1 = omMetadataManager - .getDeletedTable().get("/vol1/bucket1/key2"); + .getDeletedTable().get("/vol1/bucket2/bucket2key1"); - assertNotNull(omKeyInfo1); + // Check bucket2key2 added active db as it can be reclaimed. + RepeatedOmKeyInfo omKeyInfo2 = omMetadataManager + .getDeletedTable().get("/vol1/bucket2/bucket2key2"); + Assertions.assertNotNull(omKeyInfo1); + Assertions.assertNotNull(omKeyInfo2); } private OmKeyArgs createVolumeBucketKey(String volumeName, String bucketName, @@ -193,6 +220,49 @@ private OmKeyArgs createVolumeBucketKey(String volumeName, String bucketName, return createKey(volumeName, bucketName, keyName); } + + private int createSnapshotDataForBucket1() throws Exception { + int snapshotCount = 0; + OmKeyArgs bucket1key1 = createVolumeBucketKey(VOLUME_NAME, BUCKET_NAME_ONE, + BucketLayout.DEFAULT, "bucket1key1"); + + createSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, "bucket1snap1", + ++snapshotCount); + + OmKeyArgs bucket1key2 = createKey(VOLUME_NAME, BUCKET_NAME_ONE, + "bucket1key2"); + + // Key 1 cannot be reclaimed as it is still referenced by Snapshot 1. + writeClient.deleteKey(bucket1key1); + // Key 2 is deleted here, which means we can reclaim + // it when snapshot 2 is deleted. + writeClient.deleteKey(bucket1key2); + + createSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, "bucket1snap2", + ++snapshotCount); + createKey(VOLUME_NAME, BUCKET_NAME_ONE, "bucket1key4"); + OmKeyArgs bucket1key5 = createKey(VOLUME_NAME, BUCKET_NAME_ONE, + "bucket1key5"); + writeClient.deleteKey(bucket1key5); + + createSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, "bucket1snap3", + ++snapshotCount); + + String snapshotKey2 = "/vol1/bucket1/bucket1snap2"; + SnapshotInfo snapshotInfo = om.getMetadataManager() + .getSnapshotInfoTable().get(snapshotKey2); + + snapshotInfo + .setSnapshotStatus(SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); + om.getMetadataManager() + .getSnapshotInfoTable().put(snapshotKey2, snapshotInfo); + snapshotInfo = om.getMetadataManager() + .getSnapshotInfoTable().get(snapshotKey2); + Assertions.assertEquals(snapshotInfo.getSnapshotStatus(), + SnapshotInfo.SnapshotStatus.SNAPSHOT_DELETED); + return snapshotCount; + } + private OmKeyArgs createKey(String volumeName, String bucketName, String keyName) throws IOException { OmKeyArgs keyArg = From a0f58d37e32194227bca1c0295cacf935688ce4f Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Thu, 2 Mar 2023 00:12:08 -0800 Subject: [PATCH 06/20] HDDS-7740. Add suggestion. --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 4b8e1da4be8d..d3d4de607281 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3160,7 +3160,7 @@ ozone.snapshot.deleting.limit.per.task 10 OZONE, PERFORMANCE, OM - The number of maximum snapshots to be reclaimed by the + The maximum number of snapshots that would be reclaimed by Snapshot Deleting Service per run. From c36f0fd6474e5fd85e7f704b32c95f1db542e374 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Thu, 2 Mar 2023 16:28:24 -0800 Subject: [PATCH 07/20] HDDS-7740. Change put to putWithBatch. --- .../response/snapshot/OMSnapshotMoveDeletedKeysResponse.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index 6d38e35a9881..1e98684d4468 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -92,8 +92,8 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, .getDeletedTable().putWithBatch(batchOperation, nextDBKey.getKey(), nextDBOmKeyInfo); } else { - omMetadataManager.getDeletedTable() - .put(nextDBKey.getKey(), nextDBOmKeyInfo); + omMetadataManager.getDeletedTable().putWithBatch( + batchOperation, nextDBKey.getKey(), nextDBOmKeyInfo); } } } From e79b9313a38cb2f7f3b1bdbac903306eb9588587 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Fri, 3 Mar 2023 09:30:09 -0800 Subject: [PATCH 08/20] Trigger Build From b37e860f1b8fcbd2162516c34e994cd7d91390eb Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Fri, 3 Mar 2023 14:29:36 -0800 Subject: [PATCH 09/20] HDDS-7740. Fix UT. --- .../om/request/snapshot/TestOMSnapshotCreateRequest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index ff36334f357d..e530f712cb11 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -254,7 +254,11 @@ public void testValidateAndUpdateCache() throws Exception { @Test public void testEmptyRenamedKeyTable() throws Exception { + SnapshotChainManager snapshotChainManager = + new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); + when(ozoneManager.getSnapshotChainManager()) + .thenReturn(snapshotChainManager); OmKeyInfo toKeyInfo = addKey("key1"); OmKeyInfo fromKeyInfo = addKey("key2"); From 0bc1e1459ed699a2e993bf3b1d3eaa829546bb43 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Sun, 5 Mar 2023 13:19:13 -0800 Subject: [PATCH 10/20] HDDS-7740. Address review comments. --- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 12 ++++++------ .../common/src/main/resources/ozone-default.xml | 4 ++-- .../apache/hadoop/ozone/om/KeyManagerImpl.java | 16 ++++++++-------- .../hadoop/ozone/om/SnapshotChainManager.java | 8 +++++++- .../OMSnapshotMoveDeletedKeysResponse.java | 1 + .../om/service/TestSnapshotDeletingService.java | 9 +++++---- 6 files changed, 29 insertions(+), 21 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index a21ba3f17f68..75be749b3dd1 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -234,15 +234,15 @@ public final class OzoneConfigKeys { OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT_DEFAULT = "300s"; // 300s for default - public static final String OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL = - "ozone.snapshot.deletion.service.interval"; + public static final String OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL = + "ozone.snapshot.deleting.service.interval"; public static final String - OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL_DEFAULT = "30s"; + OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL_DEFAULT = "30s"; - public static final String OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT = - "ozone.snapshot.deletion.service.timeout"; + public static final String OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT = + "ozone.snapshot.deleting.service.timeout"; public static final String - OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT_DEFAULT = "300s"; + OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT_DEFAULT = "300s"; public static final String OZONE_BLOCK_DELETING_SERVICE_WORKERS = "ozone.block.deleting.service.workers"; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 9dd77b056833..226510573418 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3179,7 +3179,7 @@ - ozone.snapshot.deletion.service.timeout + ozone.snapshot.deleting.service.timeout 300s OZONE, PERFORMANCE, OM @@ -3188,7 +3188,7 @@ - ozone.snapshot.deletion.service.interval + ozone.snapshot.deleting.service.interval 30s OZONE, PERFORMANCE, OM diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index fb99495c810d..0cd707d484c5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -111,10 +111,10 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL_DEFAULT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; @@ -270,12 +270,12 @@ public void start(OzoneConfiguration configuration) { if (snapshotDeletingService == null) { long snapshotServiceInterval = configuration.getTimeDuration( - OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL, - OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL_DEFAULT, + OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, + OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); long snapshotServiceTimeout = configuration.getTimeDuration( - OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT, - OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT_DEFAULT, + OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT, + OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS); try { snapshotDeletingService = new SnapshotDeletingService( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java index 9e9766ff4fc5..0df80d299271 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java @@ -257,7 +257,11 @@ private boolean deleteSnapshotPath(String snapshotPath, return status; } - public void loadFromSnapshotInfoTable(OMMetadataManager metadataManager) + /** + * Loads the snapshot chain from SnapshotInfo table. + * @param metadataManager OMMetadataManager + */ + private void loadFromSnapshotInfoTable(OMMetadataManager metadataManager) throws IOException { // read from snapshotInfo table to populate // snapshot chains - both global and local path @@ -267,6 +271,8 @@ public void loadFromSnapshotInfoTable(OMMetadataManager metadataManager) Table.KeyValue< String, SnapshotInfo > kv; snapshotChainGlobal.clear(); snapshotChainPath.clear(); + latestPathSnapshotID.clear(); + snapshotPathToTableKey.clear(); while (keyIter.hasNext()) { kv = keyIter.next(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index 1e98684d4468..ebac4af28917 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -113,3 +113,4 @@ private RepeatedOmKeyInfo createRepeatedOmKeyInfo(List keyInfoList) return result; } } + diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java index bd8ac683f028..ea424aba3121 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java @@ -56,8 +56,8 @@ import java.util.concurrent.TimeUnit; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT; /** * Test Snapshot Deleting Service. @@ -88,9 +88,9 @@ public void createConfAndInitValues() throws Exception { File testDir = PathUtils.getTestDir(TestSnapshotDeletingService.class); System.setProperty(DBConfigFromFile.CONFIG_DIR, "/"); ServerUtils.setOzoneMetaDirPath(conf, testDir.getPath()); - conf.setTimeDuration(OZONE_SNAPSHOT_DELETION_SERVICE_INTERVAL, + conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1000, TimeUnit.MILLISECONDS); - conf.setTimeDuration(OZONE_SNAPSHOT_DELETION_SERVICE_TIMEOUT, + conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT, 100000, TimeUnit.MILLISECONDS); conf.setQuietMode(false); omTestManagers = new OmTestManagers(conf); @@ -298,3 +298,4 @@ private void createSnapshot(String volName, String bucketName, }, 1000, 10000); } } + From 9ab93739cea027e6c4bb81d8d26150786ac6dda8 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Thu, 9 Mar 2023 21:38:44 -0800 Subject: [PATCH 11/20] HDDS-7740. Update current snapshot deletedTable. --- .../org/apache/hadoop/ozone/OzoneConsts.java | 2 + .../hadoop/ozone/om/helpers/WithObjectID.java | 6 +- .../src/main/proto/OmClientProtocol.proto | 5 +- .../ozone/om/request/key/OMKeyRequest.java | 2 + .../OMSnapshotMoveDeletedKeysRequest.java | 13 ++-- .../OMSnapshotMoveDeletedKeysResponse.java | 62 +++++++++++-------- .../om/service/SnapshotDeletingService.java | 48 ++++++++------ .../service/TestSnapshotDeletingService.java | 4 +- 8 files changed, 84 insertions(+), 58 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 13a3319a527f..11ddbedc9bd5 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -237,6 +237,8 @@ public enum Units { TB, GB, MB, KB, B } public static final int INVALID_PORT = -1; + public static final long OBJECT_ID_DEFAULT = 0L; + /** * Default SCM Datanode ID file name. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java index eebb4d87517c..618d7523cfe3 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java @@ -52,7 +52,7 @@ public long getUpdateID() { } /** - * Set the Object ID. If this value is already set then this function throws. + * Set the Object ID. * There is a reason why we cannot use the final here. The object * ({@link OmVolumeArgs}/ {@link OmBucketInfo}/ {@link OmKeyInfo}) is * deserialized from the protobuf in many places in code. We need to set @@ -61,10 +61,6 @@ public long getUpdateID() { * @param obId - long */ public void setObjectID(long obId) { - if (this.objectID != 0) { - throw new UnsupportedOperationException("Attempt to modify object ID " + - "which is not zero. Current Object ID is " + this.objectID); - } this.objectID = obId; } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 343e47e376b1..2df3ae2c0e37 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1706,9 +1706,8 @@ message DeleteSnapshotRequest { message SnapshotMoveDeletedKeysRequest { optional SnapshotInfo fromSnapshot = 1; - optional SnapshotInfo nextSnapshot = 2; - repeated SnapshotMoveKeyInfos activeDBKeys = 3; - repeated SnapshotMoveKeyInfos nextDBKeys = 4; + repeated SnapshotMoveKeyInfos nextDBKeys = 2; + repeated SnapshotMoveKeyInfos reclaimKeys = 3; } message SnapshotMoveKeyInfos { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 830e60600dd8..6d61007708dc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -86,6 +86,7 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto.READ; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto.WRITE; +import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes .BUCKET_NOT_FOUND; @@ -837,6 +838,7 @@ protected OmKeyInfo wrapUncommittedBlocksAsPseudoKey( LOG.info("Detect allocated but uncommitted blocks {} in key {}.", uncommitted, omKeyInfo.getKeyName()); OmKeyInfo pseudoKeyInfo = omKeyInfo.copyObject(); + pseudoKeyInfo.setObjectID(OBJECT_ID_DEFAULT); // TODO dataSize of pseudoKey is not real here List uncommittedGroups = new ArrayList<>(); // version not matters in the current logic of keyDeletingService, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index 086b19645bc9..f1db67846f88 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -72,15 +72,19 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, OzoneManagerProtocolProtos.OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(getOmRequest()); try { + OmSnapshot omFromSnapshot = (OmSnapshot) omSnapshotManager + .checkForSnapshot(fromSnapshot.getVolumeName(), + fromSnapshot.getBucketName(), + getSnapshotPrefix(fromSnapshot.getName())); + nextSnapshot = getNextActiveSnapshot(fromSnapshot, snapshotChainManager, omSnapshotManager); // Get next non-deleted snapshot. - - List activeDBKeysList = - moveDeletedKeysRequest.getActiveDBKeysList(); List nextDBKeysList = moveDeletedKeysRequest.getNextDBKeysList(); + List reclaimKeysList = + moveDeletedKeysRequest.getReclaimKeysList(); OmSnapshot omNextSnapshot = null; @@ -92,7 +96,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } omClientResponse = new OMSnapshotMoveDeletedKeysResponse( - omResponse.build(), omNextSnapshot, activeDBKeysList, nextDBKeysList); + omResponse.build(), omFromSnapshot, omNextSnapshot, + nextDBKeysList, reclaimKeysList); } catch (IOException ex) { omClientResponse = new OMSnapshotMoveDeletedKeysResponse( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index ebac4af28917..ea4a009acd9e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om.response.snapshot; import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -41,17 +42,20 @@ @CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) public class OMSnapshotMoveDeletedKeysResponse extends OMClientResponse { + private OmSnapshot fromSnapshot; private OmSnapshot nextSnapshot; - private List activeDBKeysList; private List nextDBKeysList; + private List reclaimKeyList; public OMSnapshotMoveDeletedKeysResponse(OMResponse omResponse, - OmSnapshot omNextSnapshot, List activeDBKeysList, - List nextDBKeysList) { + @Nonnull OmSnapshot omFromSnapshot, OmSnapshot omNextSnapshot, + List nextDBKeysList, + List reclaimKeysList) { super(omResponse); + this.fromSnapshot = omFromSnapshot; this.nextSnapshot = omNextSnapshot; - this.activeDBKeysList = activeDBKeysList; this.nextDBKeysList = nextDBKeysList; + this.reclaimKeyList = reclaimKeysList; } /** @@ -67,34 +71,40 @@ public OMSnapshotMoveDeletedKeysResponse(@Nonnull OMResponse omResponse) { protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { - for (SnapshotMoveKeyInfos activeDBKey : activeDBKeysList) { - RepeatedOmKeyInfo activeDBOmKeyInfo = - createRepeatedOmKeyInfo(activeDBKey.getKeyInfosList()); - - if (activeDBOmKeyInfo == null) { - continue; + if (nextSnapshot != null) { + DBStore nextSnapshotStore = nextSnapshot.getMetadataManager().getStore(); + // Init Batch Operation for snapshot db. + try (BatchOperation writeBatch = nextSnapshotStore.initBatchOperation()) { + processKeys(writeBatch, nextSnapshot.getMetadataManager(), + nextDBKeysList); + nextSnapshotStore.commitBatchOperation(writeBatch); } - - omMetadataManager.getDeletedTable().putWithBatch( - batchOperation, activeDBKey.getKey(), activeDBOmKeyInfo); + } else { + // Handle case when there is no next Snapshot + processKeys(batchOperation, omMetadataManager, nextDBKeysList); } - for (SnapshotMoveKeyInfos nextDBKey : nextDBKeysList) { - RepeatedOmKeyInfo nextDBOmKeyInfo = - createRepeatedOmKeyInfo(nextDBKey.getKeyInfosList()); + // Update From Snapshot Deleted Table. + DBStore fromSnapshotStore = fromSnapshot.getMetadataManager().getStore(); + try (BatchOperation fromSnapshotBatchOp = + fromSnapshotStore.initBatchOperation()) { + processKeys(fromSnapshotBatchOp, fromSnapshot.getMetadataManager(), + reclaimKeyList); + fromSnapshotStore.commitBatchOperation(fromSnapshotBatchOp); + } + } - if (nextDBOmKeyInfo == null) { + private void processKeys(BatchOperation batchOp, + OMMetadataManager metadataManager, + List keyList) throws IOException { + for (SnapshotMoveKeyInfos dBKey : keyList) { + RepeatedOmKeyInfo omKeyInfos = + createRepeatedOmKeyInfo(dBKey.getKeyInfosList()); + if (omKeyInfos == null) { continue; } - - if (nextSnapshot != null) { - nextSnapshot.getMetadataManager() - .getDeletedTable().putWithBatch(batchOperation, - nextDBKey.getKey(), nextDBOmKeyInfo); - } else { - omMetadataManager.getDeletedTable().putWithBatch( - batchOperation, nextDBKey.getKey(), nextDBOmKeyInfo); - } + metadataManager.getDeletedTable().putWithBatch(batchOp, + dBKey.getKey(), omKeyInfos); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index ef4edf0a0dd4..5d34d750cb17 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -57,6 +57,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; @@ -157,8 +158,8 @@ public BackgroundTaskResult call() throws Exception { Table previousKeyTable = null; OmSnapshot omPreviousSnapshot = null; - // Handle case when the deleted snapshot is the first snapshot. - // Move deleted keys to activeDB's deletedKeyTable + // Split RepeatedOmKeyInfo and update current snapshot deletedKeyTable + // and next snapshot deletedKeyTable. if (previousSnapshot != null) { omPreviousSnapshot = (OmSnapshot) omSnapshotManager .checkForSnapshot(previousSnapshot.getVolumeName(), @@ -169,10 +170,9 @@ public BackgroundTaskResult call() throws Exception { .getMetadataManager().getKeyTable(bucketInfo.getBucketLayout()); } - // Move key to either next non deleted snapshot's snapshotDeletedTable - // or move to active object store deleted table - - List toActiveDBList = new ArrayList<>(); + // Move key to either next non deleted snapshot's deletedTable + // or keep it in current snapshot deleted table. + List toReclaimList = new ArrayList<>(); List toNextDBList = new ArrayList<>(); try (TableIterator previousKeyTable) throws IOException { - if (checkKeyExistInPreviousTable(previousKeyTable, keyInfo)) { + if (checkKeyReclaimable(previousKeyTable, keyInfo)) { // Move to next non deleted snapshot's deleted table toNextDb.addKeyInfos(keyInfo.getProtobuf( ClientVersion.CURRENT_VERSION)); } else { - // Move to active DB Deleted Table. - toActiveDb.addKeyInfos(keyInfo + // Update in current db's deletedKeyTable + toReclaim.addKeyInfos(keyInfo .getProtobuf(ClientVersion.CURRENT_VERSION)); } } private void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, - List toActiveDBList, + List toReclaimList, List toNextDBList) { SnapshotMoveDeletedKeysRequest.Builder moveDeletedKeysBuilder = @@ -249,7 +254,7 @@ private void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, .setFromSnapshot(snapInfo.getProtobuf()); SnapshotMoveDeletedKeysRequest moveDeletedKeys = - moveDeletedKeysBuilder.addAllActiveDBKeys(toActiveDBList) + moveDeletedKeysBuilder.addAllReclaimKeys(toReclaimList) .addAllNextDBKeys(toNextDBList).build(); @@ -262,14 +267,20 @@ private void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, submitRequest(omRequest); } - private boolean checkKeyExistInPreviousTable( + private boolean checkKeyReclaimable( Table previousKeyTable, OmKeyInfo deletedKeyInfo) throws IOException { + // Handle case when the deleted snapshot is the first snapshot. if (previousKeyTable == null) { return false; } + // These are uncommitted blocks wrapped into a pseudo KeyInfo + if (deletedKeyInfo.getObjectID() == OBJECT_ID_DEFAULT) { + return false; + } + //TODO: [SNAPSHOT] Handle Renamed Keys String dbKey = ozoneManager.getMetadataManager() .getOzoneKey(deletedKeyInfo.getVolumeName(), @@ -366,3 +377,4 @@ public void setSuccessRunCount(long num) { successRunCount.getAndSet(num); } } + diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java index ea424aba3121..7c3c81f430d9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java @@ -289,8 +289,8 @@ private void createSnapshot(String volName, String bucketName, GenericTestUtils.waitFor(() -> { try { - return omMetadataManager.getSnapshotInfoTable() - .getEstimatedKeyCount() >= count; + return omMetadataManager.countRowsInTable( + omMetadataManager.getSnapshotInfoTable()) >= count; } catch (IOException e) { e.printStackTrace(); } From ff78d4774777da61ae78727b27a64d9d39828307 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Fri, 10 Mar 2023 16:05:51 -0800 Subject: [PATCH 12/20] HDDS-7740. Disable test. --- .../service/TestSnapshotDeletingService.java | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java index 7c3c81f430d9..e1da3128c66e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java @@ -46,6 +46,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.rules.TemporaryFolder; @@ -108,44 +109,53 @@ public void cleanup() throws Exception { } @Test + @Disabled("HDDS-7974") public void testSnapshotKeySpaceReclaim() throws Exception { SnapshotDeletingService snapshotDeletingService = (SnapshotDeletingService) keyManager.getSnapshotDeletingService(); + KeyDeletingService deletingService = (KeyDeletingService) + keyManager.getDeletingService(); // Suspending SnapshotDeletingService snapshotDeletingService.suspend(); createSnapshotDataForBucket1(); snapshotDeletingService.resume(); - snapshotDeletingService.setSuccessRunCount(0); + deletingService.start(); + GenericTestUtils.waitFor(() -> + deletingService.getRunCount().get() >= 1, + 1000, 10000); GenericTestUtils.waitFor(() -> snapshotDeletingService.getSuccessfulRunCount() >= 1, 1000, 10000); - OmSnapshot nextSnapshot = (OmSnapshot) om.getOmSnapshotManager() + OmSnapshot bucket1snap3 = (OmSnapshot) om.getOmSnapshotManager() .checkForSnapshot(VOLUME_NAME, BUCKET_NAME_ONE, getSnapshotPrefix("bucket1snap3")); // Check bucket1key1 added to next non deleted snapshot db. RepeatedOmKeyInfo omKeyInfo = - nextSnapshot.getMetadataManager() + bucket1snap3.getMetadataManager() .getDeletedTable().get("/vol1/bucket1/bucket1key1"); Assertions.assertNotNull(omKeyInfo); - // Check bucket1key2 added active db as it can be reclaimed. + // Check bucket1key2 not in active DB. As the key is updated + // in bucket1snap2 RepeatedOmKeyInfo omKeyInfo1 = omMetadataManager .getDeletedTable().get("/vol1/bucket1/bucket1key2"); - - Assertions.assertNotNull(omKeyInfo1); - + Assertions.assertNull(omKeyInfo1); + deletingService.shutdown(); } @Test + @Disabled("HDDS-7974") public void testMultipleSnapshotKeyReclaim() throws Exception { SnapshotDeletingService snapshotDeletingService = (SnapshotDeletingService) keyManager.getSnapshotDeletingService(); + KeyDeletingService deletingService = (KeyDeletingService) + keyManager.getDeletingService(); // Suspending SnapshotDeletingService snapshotDeletingService.suspend(); @@ -182,7 +192,11 @@ public void testMultipleSnapshotKeyReclaim() throws Exception { snapshotDeletingService.resume(); - snapshotDeletingService.setSuccessRunCount(0L); + deletingService.start(); + GenericTestUtils.waitFor(() -> + deletingService.getRunCount().get() >= 1, + 1000, 10000); + GenericTestUtils.waitFor(() -> snapshotDeletingService.getSuccessfulRunCount() >= 1, 1000, 10000); @@ -195,8 +209,11 @@ public void testMultipleSnapshotKeyReclaim() throws Exception { RepeatedOmKeyInfo omKeyInfo2 = omMetadataManager .getDeletedTable().get("/vol1/bucket2/bucket2key2"); - Assertions.assertNotNull(omKeyInfo1); - Assertions.assertNotNull(omKeyInfo2); + //TODO: [SNAPSHOT] Check this shouldn't be null when KeyDeletingService + // is modified for Snapshot + Assertions.assertNull(omKeyInfo1); + Assertions.assertNull(omKeyInfo2); + deletingService.shutdown(); } private OmKeyArgs createVolumeBucketKey(String volumeName, String bucketName, From 2144e16ce82fa9b8cb99828a96aab1e8f89a719f Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Tue, 14 Mar 2023 10:21:55 -0700 Subject: [PATCH 13/20] HDDS-7740. Change variable name and adjust condition. --- .../src/main/java/org/apache/hadoop/ozone/OzoneConsts.java | 5 ++++- .../org/apache/hadoop/ozone/om/helpers/WithObjectID.java | 6 ++++++ .../apache/hadoop/ozone/om/request/key/OMKeyRequest.java | 4 ++-- .../hadoop/ozone/om/service/SnapshotDeletingService.java | 4 ++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 11ddbedc9bd5..3acc47bc09e4 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -237,7 +237,10 @@ public enum Units { TB, GB, MB, KB, B } public static final int INVALID_PORT = -1; - public static final long OBJECT_ID_DEFAULT = 0L; + /** + * Object ID to identify reclaimable uncommitted blocks. + */ + public static final long OBJECT_ID_RECLAIM_BLOCKS = 0L; /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java index 618d7523cfe3..0ea1a1c0e6a7 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/WithObjectID.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.ozone.om.helpers; +import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; + /** * Mixin class to handle ObjectID and UpdateID. */ @@ -61,6 +63,10 @@ public long getUpdateID() { * @param obId - long */ public void setObjectID(long obId) { + if (this.objectID != 0 && obId != OBJECT_ID_RECLAIM_BLOCKS) { + throw new UnsupportedOperationException("Attempt to modify object ID " + + "which is not zero. Current Object ID is " + this.objectID); + } this.objectID = obId; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 6d61007708dc..8f790bc34460 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -86,7 +86,7 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto.READ; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto.WRITE; -import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes .BUCKET_NOT_FOUND; @@ -838,7 +838,7 @@ protected OmKeyInfo wrapUncommittedBlocksAsPseudoKey( LOG.info("Detect allocated but uncommitted blocks {} in key {}.", uncommitted, omKeyInfo.getKeyName()); OmKeyInfo pseudoKeyInfo = omKeyInfo.copyObject(); - pseudoKeyInfo.setObjectID(OBJECT_ID_DEFAULT); + pseudoKeyInfo.setObjectID(OBJECT_ID_RECLAIM_BLOCKS); // TODO dataSize of pseudoKey is not real here List uncommittedGroups = new ArrayList<>(); // version not matters in the current logic of keyDeletingService, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 5d34d750cb17..d98acd448616 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -57,7 +57,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; @@ -277,7 +277,7 @@ private boolean checkKeyReclaimable( } // These are uncommitted blocks wrapped into a pseudo KeyInfo - if (deletedKeyInfo.getObjectID() == OBJECT_ID_DEFAULT) { + if (deletedKeyInfo.getObjectID() == OBJECT_ID_RECLAIM_BLOCKS) { return false; } From 0d9f77389f29cae0964d4405eaa564d79a0e9474 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Tue, 14 Mar 2023 13:12:27 -0700 Subject: [PATCH 14/20] HDDS-7740. Add comment and update variable name. --- .../apache/hadoop/ozone/om/request/key/OMKeyRequest.java | 2 ++ .../snapshot/OMSnapshotMoveDeletedKeysResponse.java | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 8f790bc34460..8febf9fcf5dc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -838,6 +838,8 @@ protected OmKeyInfo wrapUncommittedBlocksAsPseudoKey( LOG.info("Detect allocated but uncommitted blocks {} in key {}.", uncommitted, omKeyInfo.getKeyName()); OmKeyInfo pseudoKeyInfo = omKeyInfo.copyObject(); + // This is a special marker to indicate that SnapshotDeletingService + // can reclaim this key's blocks unconditionally. pseudoKeyInfo.setObjectID(OBJECT_ID_RECLAIM_BLOCKS); // TODO dataSize of pseudoKey is not real here List uncommittedGroups = new ArrayList<>(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index ea4a009acd9e..a265b1f50c09 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -45,7 +45,7 @@ public class OMSnapshotMoveDeletedKeysResponse extends OMClientResponse { private OmSnapshot fromSnapshot; private OmSnapshot nextSnapshot; private List nextDBKeysList; - private List reclaimKeyList; + private List reclaimKeysList; public OMSnapshotMoveDeletedKeysResponse(OMResponse omResponse, @Nonnull OmSnapshot omFromSnapshot, OmSnapshot omNextSnapshot, @@ -55,7 +55,7 @@ public OMSnapshotMoveDeletedKeysResponse(OMResponse omResponse, this.fromSnapshot = omFromSnapshot; this.nextSnapshot = omNextSnapshot; this.nextDBKeysList = nextDBKeysList; - this.reclaimKeyList = reclaimKeysList; + this.reclaimKeysList = reclaimKeysList; } /** @@ -89,7 +89,7 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, try (BatchOperation fromSnapshotBatchOp = fromSnapshotStore.initBatchOperation()) { processKeys(fromSnapshotBatchOp, fromSnapshot.getMetadataManager(), - reclaimKeyList); + reclaimKeysList); fromSnapshotStore.commitBatchOperation(fromSnapshotBatchOp); } } From 5f171663de32ec66970933e45cfb81c146b9ceb0 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Wed, 15 Mar 2023 18:13:47 -0700 Subject: [PATCH 15/20] HDDS-7883. [Snapshot] Accommodate FSO, Key renames and implement OMSnapshotPurgeRequest for SnapshotDeletingService --- .../src/main/resources/ozone-default.xml | 9 + .../java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../apache/hadoop/ozone/om/OMConfigKeys.java | 4 + .../src/main/proto/OmClientProtocol.proto | 11 ++ .../ozone/om/OmMetadataManagerImpl.java | 12 ++ .../apache/hadoop/ozone/om/OzoneManager.java | 11 -- .../hadoop/ozone/om/SnapshotChainManager.java | 27 +-- .../ratis/utils/OzoneManagerRatisUtils.java | 3 + .../snapshot/OMSnapshotCreateRequest.java | 7 +- .../OMSnapshotMoveDeletedKeysRequest.java | 5 +- .../snapshot/OMSnapshotPurgeRequest.java | 64 +++++++ .../snapshot/OMSnapshotPurgeResponse.java | 145 +++++++++++++++ .../om/service/SnapshotDeletingService.java | 105 +++++++++-- .../snapshot/TestOMSnapshotCreateRequest.java | 16 +- .../snapshot/TestOMSnapshotDeleteRequest.java | 8 +- ...TestOMSnapshotPurgeRequestAndResponse.java | 170 ++++++++++++++++++ 16 files changed, 534 insertions(+), 64 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 855e21c0111f..8a1466ad7007 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -491,6 +491,15 @@ and DataNode. + + ozone.snapshot.key.deleting.limit.per.task + 20000 + OM, PERFORMANCE + + The maximum number of deleted keys to be scanned by Snapshot + Deleting Service per snapshot run. + + ozone.om.service.ids diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 249bc48e844c..e09025a5c37c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -309,6 +309,7 @@ public static boolean isReadOnly( case CreateSnapshot: case DeleteSnapshot: case SnapshotMoveDeletedKeys: + case SnapshotPurge: return false; default: LOG.error("CmdType {} is not categorized as readOnly or not.", cmdType); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index eddfd389258e..72fa91d4e7d4 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -85,6 +85,10 @@ private OMConfigKeys() { public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK = "ozone.key.deleting.limit.per.task"; public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 20000; + public static final String OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK = + "ozone.snapshot.key.deleting.limit.per.task"; + public static final int OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT + = 20000; public static final String OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL = "ozone.om.open.key.cleanup.service.interval"; diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 8a70f64361eb..1492233a1dae 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -133,6 +133,7 @@ enum Type { SnapshotMoveDeletedKeys = 116; TransferLeadership = 117; + SnapshotPurge = 118; } message OMRequest { @@ -248,6 +249,7 @@ message OMRequest { optional SnapshotMoveDeletedKeysRequest SnapshotMoveDeletedKeysRequest = 116; optional hdds.TransferLeadershipRequestProto TransferOmLeadershipRequest = 117; + optional SnapshotPurgeRequest SnapshotPurgeRequest = 118; } @@ -357,6 +359,7 @@ message OMResponse { optional SnapshotMoveDeletedKeysResponse SnapshotMoveDeletedKeysResponse = 116; optional hdds.TransferLeadershipResponseProto TransferOmLeadershipResponse = 117; + optional SnapshotPurgeResponse SnapshotPurgeResponse = 118; } enum Status { @@ -1717,6 +1720,10 @@ message SnapshotMoveKeyInfos { repeated KeyInfo keyInfos = 2; } +message SnapshotPurgeRequest { + repeated string snapshotDBKeys = 1; +} + message DeleteTenantRequest { optional string tenantId = 1; } @@ -1783,6 +1790,10 @@ message SnapshotMoveDeletedKeysResponse { } +message SnapshotPurgeResponse { + +} + message SnapshotDiffReportProto { optional string volumeName = 1; optional string bucketName = 2; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index ede9b5893a32..30eb7462299c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -292,6 +292,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager, private Map tableMap = new HashMap<>(); private List tableCacheMetrics = new LinkedList<>(); + private SnapshotChainManager snapshotChainManager; public OmMetadataManagerImpl(OzoneConfiguration conf) throws IOException { this.lock = new OzoneManagerLock(conf); @@ -464,6 +465,8 @@ public void start(OzoneConfiguration configuration) throws IOException { initializeOmTables(true); } + + snapshotChainManager = new SnapshotChainManager(this); } public static DBStore loadDB(OzoneConfiguration configuration, File metaDir) @@ -1632,6 +1635,15 @@ public Table getRenamedKeyTable() { return renamedKeyTable; } + /** + * Get Snapshot Chain Manager. + * + * @return SnapshotChainManager. + */ + public SnapshotChainManager getSnapshotChainManager() { + return snapshotChainManager; + } + /** * Update store used by subclass. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 35361ef4bcdd..ac4c71657949 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -452,7 +452,6 @@ private enum State { // This metadata reader points to the active filesystem private OmMetadataReader omMetadataReader; private OmSnapshotManager omSnapshotManager; - private SnapshotChainManager snapshotChainManager; /** A list of property that are reconfigurable at runtime. */ private final SortedSet reconfigurableProperties = @@ -770,7 +769,6 @@ private void instantiateServices(boolean withNewSnapshot) throws IOException { omMetadataReader = new OmMetadataReader(keyManager, prefixManager, this, LOG, AUDIT, metrics); omSnapshotManager = new OmSnapshotManager(this); - snapshotChainManager = new SnapshotChainManager(metadataManager); // Snapshot metrics updateActiveSnapshotMetrics(); @@ -1510,15 +1508,6 @@ public OmSnapshotManager getOmSnapshotManager() { return omSnapshotManager; } - /** - * Get Snapshot Chain Manager. - * - * @return SnapshotChainManager. - */ - public SnapshotChainManager getSnapshotChainManager() { - return snapshotChainManager; - } - /** * Get metadata manager. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java index 0df80d299271..405c243062f8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java @@ -49,7 +49,7 @@ public class SnapshotChainManager { snapshotChainPath; private Map latestPathSnapshotID; private String latestGlobalSnapshotID; - private Map snapshotPathToTableKey; + private Map snapshotIdToTableKey; private static final Logger LOG = LoggerFactory.getLogger(SnapshotChainManager.class); @@ -58,7 +58,7 @@ public SnapshotChainManager(OMMetadataManager metadataManager) snapshotChainGlobal = new LinkedHashMap<>(); snapshotChainPath = new HashMap<>(); latestPathSnapshotID = new HashMap<>(); - snapshotPathToTableKey = new HashMap<>(); + snapshotIdToTableKey = new HashMap<>(); latestGlobalSnapshotID = null; loadFromSnapshotInfoTable(metadataManager); } @@ -100,8 +100,7 @@ private void addSnapshotGlobal(String snapshotID, */ private void addSnapshotPath(String snapshotPath, String snapshotID, - String prevPathID, - String snapTableKey) throws IOException { + String prevPathID) throws IOException { // set previous snapshotID to null if it is "" for // internal in-mem structure if (prevPathID != null && prevPathID.isEmpty()) { @@ -139,8 +138,6 @@ private void addSnapshotPath(String snapshotPath, .put(snapshotID, new SnapshotChainInfo(snapshotID, prevPathID, null)); - // store snapshot ID to snapshot DB table key in the map - snapshotPathToTableKey.put(snapshotID, snapTableKey); // set state variable latestPath snapshot entry to this snapshotID latestPathSnapshotID.put(snapshotPath, snapshotID); }; @@ -272,7 +269,7 @@ private void loadFromSnapshotInfoTable(OMMetadataManager metadataManager) snapshotChainGlobal.clear(); snapshotChainPath.clear(); latestPathSnapshotID.clear(); - snapshotPathToTableKey.clear(); + snapshotIdToTableKey.clear(); while (keyIter.hasNext()) { kv = keyIter.next(); @@ -292,8 +289,9 @@ public void addSnapshot(SnapshotInfo sinfo) throws IOException { sinfo.getGlobalPreviousSnapshotID()); addSnapshotPath(sinfo.getSnapshotPath(), sinfo.getSnapshotID(), - sinfo.getPathPreviousSnapshotID(), - sinfo.getTableKey()); + sinfo.getPathPreviousSnapshotID()); + // store snapshot ID to snapshot DB table key in the map + snapshotIdToTableKey.put(sinfo.getSnapshotID(), sinfo.getTableKey()); } /** @@ -304,9 +302,12 @@ public void addSnapshot(SnapshotInfo sinfo) throws IOException { public boolean deleteSnapshot(SnapshotInfo sinfo) throws IOException { boolean status; - status = deleteSnapshotGlobal(sinfo.getSnapshotID()); - return status && deleteSnapshotPath(sinfo.getSnapshotPath(), - sinfo.getSnapshotID()); + status = deleteSnapshotGlobal(sinfo.getSnapshotID()) && + deleteSnapshotPath(sinfo.getSnapshotPath(), sinfo.getSnapshotID()); + if (status) { + snapshotIdToTableKey.remove(sinfo.getSnapshotID()); + } + return status; } /** @@ -522,7 +523,7 @@ public String previousPathSnapshot(String snapshotPath, String snapshotID) } public String getTableKey(String snapshotPath) { - return snapshotPathToTableKey.get(snapshotPath); + return snapshotIdToTableKey.get(snapshotPath); } @VisibleForTesting diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index bc08a9059811..512c8cb92beb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -73,6 +73,7 @@ import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotDeleteRequest; import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotMoveDeletedKeysRequest; +import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotPurgeRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMCancelPrepareRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMFinalizeUpgradeRequest; import org.apache.hadoop.ozone.om.request.upgrade.OMPrepareRequest; @@ -218,6 +219,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMSnapshotDeleteRequest(omRequest); case SnapshotMoveDeletedKeys: return new OMSnapshotMoveDeletedKeysRequest(omRequest); + case SnapshotPurge: + return new OMSnapshotPurgeRequest(omRequest); case DeleteOpenKeys: BucketLayout bktLayout = BucketLayout.DEFAULT; if (omRequest.getDeleteOpenKeysRequest().hasBucketLayout()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index 2ee67692ff9a..a37f2edd2bba 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -26,8 +26,8 @@ import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; -import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -115,9 +115,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, boolean acquiredBucketLock = false, acquiredSnapshotLock = false; IOException exception = null; - OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) + ozoneManager.getMetadataManager(); SnapshotChainManager snapshotChainManager = - ozoneManager.getSnapshotChainManager(); + omMetadataManager.getSnapshotChainManager(); OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index f1db67846f88..2c1f44e6777a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om.request.snapshot; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; @@ -57,8 +58,10 @@ public OMSnapshotMoveDeletedKeysRequest(OMRequest omRequest) { public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager(); + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) + ozoneManager.getMetadataManager(); SnapshotChainManager snapshotChainManager = - ozoneManager.getSnapshotChainManager(); + omMetadataManager.getSnapshotChainManager(); SnapshotMoveDeletedKeysRequest moveDeletedKeysRequest = getOmRequest().getSnapshotMoveDeletedKeysRequest(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java new file mode 100644 index 000000000000..30409c047342 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.request.snapshot; + +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotPurgeResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; + +import java.util.List; + +/** + * Handles OMSnapshotPurge Request. + */ +public class OMSnapshotPurgeRequest extends OMClientRequest { + + public OMSnapshotPurgeRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, + long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { + + OMClientResponse omClientResponse = null; + + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = + OmResponseUtil.getOMResponseBuilder(getOmRequest()); + SnapshotPurgeRequest snapshotPurgeRequest = getOmRequest() + .getSnapshotPurgeRequest(); + + List snapshotDbKeys = snapshotPurgeRequest + .getSnapshotDBKeysList(); + + omClientResponse = new OMSnapshotPurgeResponse(omResponse.build(), + snapshotDbKeys); + addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, + omDoubleBufferHelper); + + return omClientResponse; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java new file mode 100644 index 000000000000..9d625ea1dc5b --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.ozone.om.response.snapshot; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.SnapshotChainManager; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; + +/** + * Response for OMSnapshotPurgeRequest. + */ +@CleanupTableInfo(cleanupTables = {SNAPSHOT_INFO_TABLE}) +public class OMSnapshotPurgeResponse extends OMClientResponse { + private static final Logger LOG = + LoggerFactory.getLogger(OMSnapshotPurgeResponse.class); + private List snapshotDbKeys; + + public OMSnapshotPurgeResponse(@Nonnull OMResponse omResponse, + @Nonnull List snapshotDbKeys) { + super(omResponse); + this.snapshotDbKeys = snapshotDbKeys; + } + + @Override + protected void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) throws IOException { + + OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) + omMetadataManager; + for (String dbKey: snapshotDbKeys) { + SnapshotInfo snapshotInfo = omMetadataManager + .getSnapshotInfoTable().get(dbKey); + cleanupSnapshotChain(metadataManager, snapshotInfo, batchOperation); + // Delete Snapshot checkpoint directory. + deleteCheckpointDirectory(omMetadataManager, snapshotInfo); + omMetadataManager.getSnapshotInfoTable().deleteWithBatch(batchOperation, + dbKey); + } + } + + /** + * Cleans up the snapshot chain and updates next snapshot's + * previousPath and previousGlobal IDs. + * @param metadataManager + * @param snapInfo + * @param batchOperation + */ + private void cleanupSnapshotChain(OmMetadataManagerImpl metadataManager, + SnapshotInfo snapInfo, BatchOperation batchOperation) throws IOException { + SnapshotChainManager snapshotChainManager = metadataManager + .getSnapshotChainManager(); + + // Updates next path snapshot's previous snapshot ID + if (snapshotChainManager.hasNextPathSnapshot( + snapInfo.getSnapshotPath(), snapInfo.getSnapshotID())) { + String nextPathSnapshotId = + snapshotChainManager.nextPathSnapshot( + snapInfo.getSnapshotPath(), snapInfo.getSnapshotID()); + + String snapshotTableKey = snapshotChainManager + .getTableKey(nextPathSnapshotId); + SnapshotInfo nextPathSnapInfo = + metadataManager.getSnapshotInfoTable().get(snapshotTableKey); + if (nextPathSnapInfo != null) { + nextPathSnapInfo.setPathPreviousSnapshotID( + snapInfo.getPathPreviousSnapshotID()); + metadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, + nextPathSnapInfo.getTableKey(), nextPathSnapInfo); + } + } + + // Updates next global snapshot's previous snapshot ID + if (snapshotChainManager.hasNextGlobalSnapshot( + snapInfo.getSnapshotID())) { + String nextGlobalSnapshotId = + snapshotChainManager.nextGlobalSnapshot(snapInfo.getSnapshotID()); + + String snapshotTableKey = snapshotChainManager + .getTableKey(nextGlobalSnapshotId); + SnapshotInfo nextGlobalSnapInfo = + metadataManager.getSnapshotInfoTable().get(snapshotTableKey); + if (nextGlobalSnapInfo != null) { + nextGlobalSnapInfo.setGlobalPreviousSnapshotID( + snapInfo.getPathPreviousSnapshotID()); + metadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, + nextGlobalSnapInfo.getTableKey(), nextGlobalSnapInfo); + } + } + + // Removes current snapshot from the snapshot chain. + snapshotChainManager.deleteSnapshot(snapInfo); + } + + /** + * Deletes the checkpoint directory for a snapshot. + * @param omMetadataManager + * @param snapshotInfo + */ + private void deleteCheckpointDirectory(OMMetadataManager omMetadataManager, + SnapshotInfo snapshotInfo) { + RDBStore store = (RDBStore) omMetadataManager.getStore(); + String checkpointPrefix = store.getDbLocation().getName(); + Path snapshotDirPath = Paths.get(store.getSnapshotsParentDir(), + checkpointPrefix + snapshotInfo.getCheckpointDir()); + try { + FileUtils.deleteDirectory(snapshotDirPath.toFile()); + } catch (IOException ex) { + LOG.error("Failed to delete snapshot directory {} for snapshot {}", + snapshotDirPath, snapshotInfo.getTableKey(), ex); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index d98acd448616..c3c862310f7f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; @@ -37,12 +38,14 @@ import org.apache.hadoop.ozone.om.helpers.OMRatisHelper; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyRenameInfo; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveDeletedKeysRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotMoveKeyInfos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.protocol.Message; @@ -57,6 +60,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT; import static org.apache.hadoop.ozone.OzoneConsts.OBJECT_ID_RECLAIM_BLOCKS; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT; @@ -84,6 +89,7 @@ public class SnapshotDeletingService extends BackgroundService { private final OzoneConfiguration conf; private final AtomicLong successRunCount; private final long snapshotDeletionPerTask; + private final int keyLimitPerSnapshot; public SnapshotDeletingService(long interval, long serviceTimeout, OzoneManager ozoneManager) throws IOException { @@ -92,7 +98,9 @@ public SnapshotDeletingService(long interval, long serviceTimeout, serviceTimeout); this.ozoneManager = ozoneManager; this.omSnapshotManager = ozoneManager.getOmSnapshotManager(); - this.chainManager = ozoneManager.getSnapshotChainManager(); + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) + ozoneManager.getMetadataManager(); + this.chainManager = omMetadataManager.getSnapshotChainManager(); this.runCount = new AtomicLong(0); this.successRunCount = new AtomicLong(0); this.suspended = new AtomicBoolean(false); @@ -100,6 +108,9 @@ public SnapshotDeletingService(long interval, long serviceTimeout, this.snapshotDeletionPerTask = conf .getLong(SNAPSHOT_DELETING_LIMIT_PER_TASK, SNAPSHOT_DELETING_LIMIT_PER_TASK_DEFAULT); + this.keyLimitPerSnapshot = conf.getInt( + OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK, + OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT); } private class SnapshotDeletingTask implements BackgroundTask { @@ -118,6 +129,7 @@ public BackgroundTaskResult call() throws Exception { > iterator = snapshotInfoTable.iterator()) { long snapshotLimit = snapshotDeletionPerTask; + List purgeSnapshotKeys = new ArrayList<>(); while (iterator.hasNext() && snapshotLimit > 0) { SnapshotInfo snapInfo = iterator.next().getValue(); @@ -142,6 +154,11 @@ public BackgroundTaskResult call() throws Exception { continue; } + Table renamedKeyTable = + omSnapshot.getMetadataManager().getRenamedKeyTable(); + + long volumeId = ozoneManager.getMetadataManager() + .getVolumeId(snapInfo.getVolumeName()); // Get bucketInfo for the snapshot bucket to get bucket layout. String dbBucketKey = ozoneManager.getMetadataManager().getBucketKey( snapInfo.getVolumeName(), snapInfo.getBucketName()); @@ -182,13 +199,18 @@ public BackgroundTaskResult call() throws Exception { String snapshotBucketKey = dbBucketKey + OzoneConsts.OM_KEY_PREFIX; iterator.seek(snapshotBucketKey); - while (deletedIterator.hasNext()) { + int deletionCount = 0; + while (deletedIterator.hasNext() && + deletionCount <= keyLimitPerSnapshot) { Table.KeyValue deletedKeyValue = deletedIterator.next(); String deletedKey = deletedKeyValue.getKey(); // Exit if it is out of the bucket scope. if (!deletedKey.startsWith(snapshotBucketKey)) { + // If snapshot deletedKeyTable doesn't have any + // entry in the snapshot scope it can be reclaimed + purgeSnapshotKeys.add(snapInfo.getTableKey()); break; } @@ -203,7 +225,8 @@ public BackgroundTaskResult call() throws Exception { for (OmKeyInfo keyInfo: repeatedOmKeyInfo.getOmKeyInfoList()) { splitRepeatedOmKeyInfo(toReclaim, toNextDb, - keyInfo, previousKeyTable); + keyInfo, previousKeyTable, renamedKeyTable, + bucketInfo, volumeId); } // If all the KeyInfos are reclaimable in RepeatedOmKeyInfo @@ -213,7 +236,7 @@ public BackgroundTaskResult call() throws Exception { toReclaimList.add(toReclaim.build()); } toNextDBList.add(toNextDb.build()); - + deletionCount++; } // Submit Move request to OM. submitSnapshotMoveDeletedKeys(snapInfo, toReclaimList, @@ -224,6 +247,9 @@ public BackgroundTaskResult call() throws Exception { LOG.error("Error while running Snapshot Deleting Service", ex); } } + // TODO: [SNAPSHOT] Enable it once KeyDeletingService, + // DirectoryDeletingService for snapshots are modified. + // submitSnapshotPurgeRequest(purgeSnapshotKeys); } catch (IOException e) { LOG.error("Error while running Snapshot Deleting Service", e); } @@ -231,10 +257,30 @@ public BackgroundTaskResult call() throws Exception { return BackgroundTaskResult.EmptyTaskResult.newResult(); } + private void submitSnapshotPurgeRequest(List purgeSnapshotKeys) { + if (!purgeSnapshotKeys.isEmpty()) { + SnapshotPurgeRequest snapshotPurgeRequest = SnapshotPurgeRequest + .newBuilder() + .addAllSnapshotDBKeys(purgeSnapshotKeys) + .build(); + + OMRequest omRequest = OMRequest.newBuilder() + .setCmdType(Type.SnapshotPurge) + .setSnapshotPurgeRequest(snapshotPurgeRequest) + .setClientId(clientId.toString()) + .build(); + + submitRequest(omRequest); + } + } + private void splitRepeatedOmKeyInfo(SnapshotMoveKeyInfos.Builder toReclaim, SnapshotMoveKeyInfos.Builder toNextDb, OmKeyInfo keyInfo, - Table previousKeyTable) throws IOException { - if (checkKeyReclaimable(previousKeyTable, keyInfo)) { + Table previousKeyTable, + Table renamedKeyTable, + OmBucketInfo bucketInfo, long volumeId) throws IOException { + if (checkKeyReclaimable(previousKeyTable, renamedKeyTable, + keyInfo, bucketInfo, volumeId)) { // Move to next non deleted snapshot's deleted table toNextDb.addKeyInfos(keyInfo.getProtobuf( ClientVersion.CURRENT_VERSION)); @@ -268,9 +314,12 @@ private void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, } private boolean checkKeyReclaimable( - Table previousKeyTable, OmKeyInfo deletedKeyInfo) - throws IOException { + Table previousKeyTable, + Table renamedKeyTable, + OmKeyInfo deletedKeyInfo, OmBucketInfo bucketInfo, + long volumeId) throws IOException { + String dbKey; // Handle case when the deleted snapshot is the first snapshot. if (previousKeyTable == null) { return false; @@ -281,12 +330,40 @@ private boolean checkKeyReclaimable( return false; } - //TODO: [SNAPSHOT] Handle Renamed Keys - String dbKey = ozoneManager.getMetadataManager() - .getOzoneKey(deletedKeyInfo.getVolumeName(), - deletedKeyInfo.getBucketName(), deletedKeyInfo.getKeyName()); + if (bucketInfo.getBucketLayout().isFileSystemOptimized()) { + // Handle FSO buckets + dbKey = ozoneManager.getMetadataManager().getOzonePathKey(volumeId, + bucketInfo.getObjectID(), deletedKeyInfo.getParentObjectID(), + deletedKeyInfo.getKeyName()); + } else { + dbKey = ozoneManager.getMetadataManager() + .getOzoneKey(deletedKeyInfo.getVolumeName(), + deletedKeyInfo.getBucketName(), deletedKeyInfo.getKeyName()); + } + + // renamedKeyTable: volumeName/bucketName/objectID -> OMRenameKeyInfo + String dbRenameKey = ozoneManager.getMetadataManager().getRenameKey( + deletedKeyInfo.getVolumeName(), deletedKeyInfo.getBucketName(), + deletedKeyInfo.getObjectID()); + + OmKeyRenameInfo renamedKeyInfo = renamedKeyTable.getIfExist(dbRenameKey); + + boolean isKeyRenamed = false; + String dbOriginalKey = null; + // Condition: key should not exist in renamedKeyTable of the current + // snapshot and keyTable of the previous snapshot. + // Check key exists in renamedKeyTable of the Snapshot + if (renamedKeyInfo != null && !renamedKeyInfo + .getOmKeyRenameInfoList().isEmpty()) { + isKeyRenamed = true; + dbOriginalKey = renamedKeyInfo.getOmKeyRenameInfoList().get(0); + } + + // previousKeyTable is fileTable if the bucket is FSO, + // otherwise it is the keyTable. + OmKeyInfo prevKeyInfo = isKeyRenamed ? previousKeyTable + .get(dbOriginalKey) : previousKeyTable.get(dbKey); - OmKeyInfo prevKeyInfo = previousKeyTable.get(dbKey); if (prevKeyInfo != null && prevKeyInfo.getObjectID() == deletedKeyInfo.getObjectID()) { return true; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index e530f712cb11..391d7b99f2c9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -29,11 +29,9 @@ import org.apache.hadoop.ozone.audit.AuditMessage; import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -71,7 +69,7 @@ public class TestOMSnapshotCreateRequest { private OzoneManager ozoneManager; private OMMetrics omMetrics; - private OMMetadataManager omMetadataManager; + private OmMetadataManagerImpl omMetadataManager; private BatchOperation batchOperation; private String volumeName; @@ -211,11 +209,7 @@ public void testPreExecuteNameLength() throws Exception { @Test public void testValidateAndUpdateCache() throws Exception { - SnapshotChainManager snapshotChainManager = - new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); - when(ozoneManager.getSnapshotChainManager()) - .thenReturn(snapshotChainManager); OMRequest omRequest = OMRequestTestUtils.createSnapshotRequest( volumeName, bucketName, snapshotName); @@ -254,11 +248,7 @@ public void testValidateAndUpdateCache() throws Exception { @Test public void testEmptyRenamedKeyTable() throws Exception { - SnapshotChainManager snapshotChainManager = - new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); - when(ozoneManager.getSnapshotChainManager()) - .thenReturn(snapshotChainManager); OmKeyInfo toKeyInfo = addKey("key1"); OmKeyInfo fromKeyInfo = addKey("key2"); @@ -302,11 +292,7 @@ public void testEmptyRenamedKeyTable() throws Exception { @Test public void testEntryExists() throws Exception { - SnapshotChainManager snapshotChainManager = - new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); - when(ozoneManager.getSnapshotChainManager()) - .thenReturn(snapshotChainManager); OMRequest omRequest = OMRequestTestUtils.createSnapshotRequest( volumeName, bucketName, snapshotName); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java index b8d7c74173b0..e380a5281ce3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java @@ -27,11 +27,9 @@ import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.AuditMessage; import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; @@ -68,7 +66,7 @@ public class TestOMSnapshotDeleteRequest { private OzoneManager ozoneManager; private OMMetrics omMetrics; - private OMMetadataManager omMetadataManager; + private OmMetadataManagerImpl omMetadataManager; private String volumeName; private String bucketName; @@ -259,10 +257,6 @@ public void testEntryNotExist() throws Exception { */ @Test public void testEntryExists() throws Exception { - SnapshotChainManager snapshotChainManager = - new SnapshotChainManager(omMetadataManager); - when(ozoneManager.getSnapshotChainManager()) - .thenReturn(snapshotChainManager); when(ozoneManager.isAdmin(any())).thenReturn(true); String key = SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java new file mode 100644 index 000000000000..46e9692770f8 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.om.request.snapshot; + +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.ozone.om.request.key.TestOMKeyRequest; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotCreateResponse; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotPurgeResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.junit.jupiter.api.Assertions; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +/** + * Tests OMSnapshotPurgeRequest class. + */ +public class TestOMSnapshotPurgeRequestAndResponse extends TestOMKeyRequest { + + private BatchOperation batchOperation; + private List checkpointPaths = new ArrayList<>(); + + /** + * Creates volume, bucket and snapshot entries. + */ + private List createSnapshots(int numSnapshotKeys) + throws Exception { + + // Add volume, bucket and key entries to OM DB. + OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, + omMetadataManager); + + // Create Snapshot and CheckpointDir + List purgeSnapshots = new ArrayList<>(numSnapshotKeys); + for (int i = 1; i <= numSnapshotKeys; i++) { + String snapshotName = keyName + "-" + i; + createSnapshotCheckpoint(snapshotName); + purgeSnapshots.add(SnapshotInfo.getTableKey(volumeName, + bucketName, snapshotName)); + } + + return purgeSnapshots; + } + + /** + * Create OMRequest which encapsulates SnapshotPurgeRequest. + * + * @return OMRequest + */ + private OMRequest createPurgeKeysRequest(List purgeSnapshotKeys) { + SnapshotPurgeRequest snapshotPurgeRequest = SnapshotPurgeRequest + .newBuilder() + .addAllSnapshotDBKeys(purgeSnapshotKeys) + .build(); + + OMRequest omRequest = OMRequest.newBuilder() + .setCmdType(Type.SnapshotPurge) + .setSnapshotPurgeRequest(snapshotPurgeRequest) + .setClientId(UUID.randomUUID().toString()) + .build(); + + return omRequest; + } + + /** + * Create snapshot and checkpoint directory. + */ + private void createSnapshotCheckpoint(String snapshotName) throws Exception { + when(ozoneManager.isAdmin(any())).thenReturn(true); + batchOperation = omMetadataManager.getStore().initBatchOperation(); + OMRequest omRequest = OMRequestTestUtils + .createSnapshotRequest(volumeName, bucketName, snapshotName); + // Pre-Execute OMSnapshotCreateRequest. + OMSnapshotCreateRequest omSnapshotCreateRequest = + TestOMSnapshotCreateRequest.doPreExecute(omRequest, ozoneManager); + + // validateAndUpdateCache OMSnapshotCreateResponse. + OMSnapshotCreateResponse omClientResponse = (OMSnapshotCreateResponse) + omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1, + ozoneManagerDoubleBufferHelper); + // Add to batch and commit to DB. + omClientResponse.addToDBBatch(omMetadataManager, batchOperation); + omMetadataManager.getStore().commitBatchOperation(batchOperation); + batchOperation.close(); + + String key = SnapshotInfo.getTableKey(volumeName, + bucketName, snapshotName); + SnapshotInfo snapshotInfo = + omMetadataManager.getSnapshotInfoTable().get(key); + Assertions.assertNotNull(snapshotInfo); + + RDBStore store = (RDBStore) omMetadataManager.getStore(); + String checkpointPrefix = store.getDbLocation().getName(); + Path snapshotDirPath = Paths.get(store.getSnapshotsParentDir(), + checkpointPrefix + snapshotInfo.getCheckpointDir()); + //Check the DB is still there + Assertions.assertTrue(Files.exists(snapshotDirPath)); + checkpointPaths.add(snapshotDirPath); + } + + private OMSnapshotPurgeRequest preExecute(OMRequest originalOmRequest) + throws IOException { + OMSnapshotPurgeRequest omSnapshotPurgeRequest = + new OMSnapshotPurgeRequest(originalOmRequest); + OMRequest modifiedOmRequest = omSnapshotPurgeRequest + .preExecute(ozoneManager); + return new OMSnapshotPurgeRequest(modifiedOmRequest); + } + + @Test + public void testValidateAndUpdateCache() throws Exception { + + List snapshotDbKeysToPurge = createSnapshots(10); + OMRequest snapshotPurgeRequest = createPurgeKeysRequest( + snapshotDbKeysToPurge); + // Pre-Execute OMSnapshotPurgeRequest + OMSnapshotPurgeRequest omSnapshotPurgeRequest = + preExecute(snapshotPurgeRequest); + + // validateAndUpdateCache for OMSnapshotPurgeRequest. + OMSnapshotPurgeResponse omSnapshotPurgeResponse = (OMSnapshotPurgeResponse) + omSnapshotPurgeRequest.validateAndUpdateCache(ozoneManager, 200L, + ozoneManagerDoubleBufferHelper); + + // Commit to DB. + batchOperation = omMetadataManager.getStore().initBatchOperation(); + omSnapshotPurgeResponse.checkAndUpdateDB(omMetadataManager, batchOperation); + omMetadataManager.getStore().commitBatchOperation(batchOperation); + + // Check if the entries are deleted. + Assertions.assertTrue(omMetadataManager.getSnapshotInfoTable().isEmpty()); + + // Check if all the checkpoints are cleared. + for (Path checkpoint : checkpointPaths) { + Assertions.assertFalse(Files.exists(checkpoint)); + } + } + +} From 8dfc27eafde1e9e9d1cec5e31f65dfa216aaf3d0 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Wed, 15 Mar 2023 20:09:05 -0700 Subject: [PATCH 16/20] HDDS-7883. Fix findbugs. --- .../hadoop/ozone/om/service/SnapshotDeletingService.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index c3c862310f7f..17d1d38c1b15 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -247,9 +247,8 @@ public BackgroundTaskResult call() throws Exception { LOG.error("Error while running Snapshot Deleting Service", ex); } } - // TODO: [SNAPSHOT] Enable it once KeyDeletingService, - // DirectoryDeletingService for snapshots are modified. - // submitSnapshotPurgeRequest(purgeSnapshotKeys); + + submitSnapshotPurgeRequest(purgeSnapshotKeys); } catch (IOException e) { LOG.error("Error while running Snapshot Deleting Service", e); } @@ -270,7 +269,9 @@ private void submitSnapshotPurgeRequest(List purgeSnapshotKeys) { .setClientId(clientId.toString()) .build(); - submitRequest(omRequest); + // TODO: [SNAPSHOT] Submit request once KeyDeletingService, + // DirectoryDeletingService for snapshots are modified. + // submitRequest(omRequest); } } From 090f42842bde033ce5b77436a5704b4a07f82a41 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Wed, 15 Mar 2023 20:35:53 -0700 Subject: [PATCH 17/20] Merge master --- .../om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index a265b1f50c09..95d7bc0bafa2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -80,7 +80,7 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, nextSnapshotStore.commitBatchOperation(writeBatch); } } else { - // Handle case when there is no next Snapshot + // Handle the case where there is no next Snapshot. processKeys(batchOperation, omMetadataManager, nextDBKeysList); } From f8180d6b1ac61f449843f4e580e1b29acdaa306c Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Thu, 16 Mar 2023 16:07:13 -0700 Subject: [PATCH 18/20] HDDS-7883. Remove unused declaration. --- .../org/apache/hadoop/ozone/om/OzoneManager.java | 11 ----------- .../snapshot/TestOMSnapshotCreateRequest.java | 13 ------------- .../snapshot/TestOMSnapshotDeleteRequest.java | 5 ----- 3 files changed, 29 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 35361ef4bcdd..ac4c71657949 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -452,7 +452,6 @@ private enum State { // This metadata reader points to the active filesystem private OmMetadataReader omMetadataReader; private OmSnapshotManager omSnapshotManager; - private SnapshotChainManager snapshotChainManager; /** A list of property that are reconfigurable at runtime. */ private final SortedSet reconfigurableProperties = @@ -770,7 +769,6 @@ private void instantiateServices(boolean withNewSnapshot) throws IOException { omMetadataReader = new OmMetadataReader(keyManager, prefixManager, this, LOG, AUDIT, metrics); omSnapshotManager = new OmSnapshotManager(this); - snapshotChainManager = new SnapshotChainManager(metadataManager); // Snapshot metrics updateActiveSnapshotMetrics(); @@ -1510,15 +1508,6 @@ public OmSnapshotManager getOmSnapshotManager() { return omSnapshotManager; } - /** - * Get Snapshot Chain Manager. - * - * @return SnapshotChainManager. - */ - public SnapshotChainManager getSnapshotChainManager() { - return snapshotChainManager; - } - /** * Get metadata manager. * diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index fcb8cb3a95c7..391d7b99f2c9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -32,7 +32,6 @@ import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; @@ -210,11 +209,7 @@ public void testPreExecuteNameLength() throws Exception { @Test public void testValidateAndUpdateCache() throws Exception { - SnapshotChainManager snapshotChainManager = - new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); - when(ozoneManager.getSnapshotChainManager()) - .thenReturn(snapshotChainManager); OMRequest omRequest = OMRequestTestUtils.createSnapshotRequest( volumeName, bucketName, snapshotName); @@ -253,11 +248,7 @@ public void testValidateAndUpdateCache() throws Exception { @Test public void testEmptyRenamedKeyTable() throws Exception { - SnapshotChainManager snapshotChainManager = - new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); - when(ozoneManager.getSnapshotChainManager()) - .thenReturn(snapshotChainManager); OmKeyInfo toKeyInfo = addKey("key1"); OmKeyInfo fromKeyInfo = addKey("key2"); @@ -301,11 +292,7 @@ public void testEmptyRenamedKeyTable() throws Exception { @Test public void testEntryExists() throws Exception { - SnapshotChainManager snapshotChainManager = - new SnapshotChainManager(omMetadataManager); when(ozoneManager.isAdmin(any())).thenReturn(true); - when(ozoneManager.getSnapshotChainManager()) - .thenReturn(snapshotChainManager); OMRequest omRequest = OMRequestTestUtils.createSnapshotRequest( volumeName, bucketName, snapshotName); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java index e243882f2626..e380a5281ce3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java @@ -30,7 +30,6 @@ import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; @@ -258,10 +257,6 @@ public void testEntryNotExist() throws Exception { */ @Test public void testEntryExists() throws Exception { - SnapshotChainManager snapshotChainManager = - new SnapshotChainManager(omMetadataManager); - when(ozoneManager.getSnapshotChainManager()) - .thenReturn(snapshotChainManager); when(ozoneManager.isAdmin(any())).thenReturn(true); String key = SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName); From fa1118d38ae20d63299cb13284a086c0cd492b53 Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Fri, 17 Mar 2023 15:57:28 -0700 Subject: [PATCH 19/20] HDDS-7883. Add test and address comments. --- .../hadoop/ozone/om/SnapshotChainManager.java | 4 +- .../om/service/SnapshotDeletingService.java | 24 +-- ...TestOMSnapshotPurgeRequestAndResponse.java | 163 ++++++++++++++++-- 3 files changed, 168 insertions(+), 23 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java index 405c243062f8..ff637358d0b5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java @@ -522,8 +522,8 @@ public String previousPathSnapshot(String snapshotPath, String snapshotID) .getPreviousSnapshotID(); } - public String getTableKey(String snapshotPath) { - return snapshotIdToTableKey.get(snapshotPath); + public String getTableKey(String snapshotId) { + return snapshotIdToTableKey.get(snapshotId); } @VisibleForTesting diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 17d1d38c1b15..8d01ff89521e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -210,6 +210,7 @@ public BackgroundTaskResult call() throws Exception { if (!deletedKey.startsWith(snapshotBucketKey)) { // If snapshot deletedKeyTable doesn't have any // entry in the snapshot scope it can be reclaimed + // TODO: [SNAPSHOT] Check deletedDirTable to be empty. purgeSnapshotKeys.add(snapInfo.getTableKey()); break; } @@ -331,15 +332,18 @@ private boolean checkKeyReclaimable( return false; } + // Construct keyTable or fileTable DB key depending on the bucket type if (bucketInfo.getBucketLayout().isFileSystemOptimized()) { - // Handle FSO buckets - dbKey = ozoneManager.getMetadataManager().getOzonePathKey(volumeId, - bucketInfo.getObjectID(), deletedKeyInfo.getParentObjectID(), + dbKey = ozoneManager.getMetadataManager().getOzonePathKey( + volumeId, + bucketInfo.getObjectID(), + deletedKeyInfo.getParentObjectID(), deletedKeyInfo.getKeyName()); } else { - dbKey = ozoneManager.getMetadataManager() - .getOzoneKey(deletedKeyInfo.getVolumeName(), - deletedKeyInfo.getBucketName(), deletedKeyInfo.getKeyName()); + dbKey = ozoneManager.getMetadataManager().getOzoneKey( + deletedKeyInfo.getVolumeName(), + deletedKeyInfo.getBucketName(), + deletedKeyInfo.getKeyName()); } // renamedKeyTable: volumeName/bucketName/objectID -> OMRenameKeyInfo @@ -365,11 +369,11 @@ private boolean checkKeyReclaimable( OmKeyInfo prevKeyInfo = isKeyRenamed ? previousKeyTable .get(dbOriginalKey) : previousKeyTable.get(dbKey); - if (prevKeyInfo != null && - prevKeyInfo.getObjectID() == deletedKeyInfo.getObjectID()) { - return true; + if (prevKeyInfo == null) { + return false; } - return false; + + return prevKeyInfo.getObjectID() == deletedKeyInfo.getObjectID(); } private SnapshotInfo getPreviousSnapshot(SnapshotInfo snapInfo) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java index 46e9692770f8..5787f6e1e741 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java @@ -19,44 +19,112 @@ package org.apache.hadoop.ozone.om.request.snapshot; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmMetadataReader; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.ozone.om.request.key.TestOMKeyRequest; import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotCreateResponse; import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotPurgeResponse; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotPurgeRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.Assertions; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mockito; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Random; import java.util.UUID; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; /** * Tests OMSnapshotPurgeRequest class. */ -public class TestOMSnapshotPurgeRequestAndResponse extends TestOMKeyRequest { +public class TestOMSnapshotPurgeRequestAndResponse { private BatchOperation batchOperation; private List checkpointPaths = new ArrayList<>(); + private static OzoneManager ozoneManager; + private static OMMetrics omMetrics; + private static OMMetadataManager omMetadataManager; + private static AuditLogger auditLogger; + + private static String volumeName; + private static String bucketName; + private static String keyName; + + + // Just setting ozoneManagerDoubleBuffer which does nothing. + private static OzoneManagerDoubleBufferHelper ozoneManagerDoubleBufferHelper = + ((response, transactionIndex) -> { + return null; + }); + + @BeforeEach + public void setup() throws Exception { + File testDir = GenericTestUtils.getRandomizedTestDir(); + ozoneManager = Mockito.mock(OzoneManager.class); + OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class); + when(lvm.getMetadataLayoutVersion()).thenReturn(0); + when(ozoneManager.getVersionManager()).thenReturn(lvm); + when(ozoneManager.isRatisEnabled()).thenReturn(true); + auditLogger = Mockito.mock(AuditLogger.class); + when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); + omMetrics = OMMetrics.create(); + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); + ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, + testDir.getAbsolutePath()); + ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS, + testDir.getAbsolutePath()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration); + when(ozoneManager.getMetrics()).thenReturn(omMetrics); + when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); + when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); + when(ozoneManager.isAdmin(any(UserGroupInformation.class))) + .thenReturn(true); + + OmMetadataReader omMetadataReader = Mockito.mock(OmMetadataReader.class); + when(ozoneManager.getOmMetadataReader()).thenReturn(omMetadataReader); + volumeName = UUID.randomUUID().toString(); + bucketName = UUID.randomUUID().toString(); + keyName = UUID.randomUUID().toString(); + } + /** * Creates volume, bucket and snapshot entries. */ private List createSnapshots(int numSnapshotKeys) throws Exception { + Random random = new Random(); // Add volume, bucket and key entries to OM DB. OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager); @@ -64,7 +132,7 @@ private List createSnapshots(int numSnapshotKeys) // Create Snapshot and CheckpointDir List purgeSnapshots = new ArrayList<>(numSnapshotKeys); for (int i = 1; i <= numSnapshotKeys; i++) { - String snapshotName = keyName + "-" + i; + String snapshotName = keyName + "-" + random.nextLong(); createSnapshotCheckpoint(snapshotName); purgeSnapshots.add(SnapshotInfo.getTableKey(volumeName, bucketName, snapshotName)); @@ -138,12 +206,8 @@ private OMSnapshotPurgeRequest preExecute(OMRequest originalOmRequest) return new OMSnapshotPurgeRequest(modifiedOmRequest); } - @Test - public void testValidateAndUpdateCache() throws Exception { - - List snapshotDbKeysToPurge = createSnapshots(10); - OMRequest snapshotPurgeRequest = createPurgeKeysRequest( - snapshotDbKeysToPurge); + private void purgeSnapshots(OMRequest snapshotPurgeRequest) + throws IOException { // Pre-Execute OMSnapshotPurgeRequest OMSnapshotPurgeRequest omSnapshotPurgeRequest = preExecute(snapshotPurgeRequest); @@ -151,12 +215,22 @@ public void testValidateAndUpdateCache() throws Exception { // validateAndUpdateCache for OMSnapshotPurgeRequest. OMSnapshotPurgeResponse omSnapshotPurgeResponse = (OMSnapshotPurgeResponse) omSnapshotPurgeRequest.validateAndUpdateCache(ozoneManager, 200L, - ozoneManagerDoubleBufferHelper); + ozoneManagerDoubleBufferHelper); // Commit to DB. batchOperation = omMetadataManager.getStore().initBatchOperation(); omSnapshotPurgeResponse.checkAndUpdateDB(omMetadataManager, batchOperation); omMetadataManager.getStore().commitBatchOperation(batchOperation); + } + + @Test + public void testValidateAndUpdateCache() throws Exception { + + List snapshotDbKeysToPurge = createSnapshots(10); + Assertions.assertFalse(omMetadataManager.getSnapshotInfoTable().isEmpty()); + OMRequest snapshotPurgeRequest = createPurgeKeysRequest( + snapshotDbKeysToPurge); + purgeSnapshots(snapshotPurgeRequest); // Check if the entries are deleted. Assertions.assertTrue(omMetadataManager.getSnapshotInfoTable().isEmpty()); @@ -167,4 +241,71 @@ public void testValidateAndUpdateCache() throws Exception { } } + @ParameterizedTest + @ValueSource(ints = {0, 1, 2, 3, 4}) + public void testSnapshotChainCleanup(int index) throws Exception { + List snapshots = createSnapshots(5); + String snapShotToPurge = snapshots.get(index); + + // Before purge, check snapshot chain + OmMetadataManagerImpl metadataManager = + (OmMetadataManagerImpl) omMetadataManager; + SnapshotChainManager chainManager = metadataManager + .getSnapshotChainManager(); + SnapshotInfo snapInfo = metadataManager.getSnapshotInfoTable() + .get(snapShotToPurge); + + // Get previous and next snapshotInfos to verify if the SnapInfo + // is changed. + String prevPathSnapId = null; + String prevGlobalSnapId = null; + String nextPathSnapId = null; + String nextGlobalSnapId = null; + + if (chainManager.hasPreviousPathSnapshot(snapInfo.getSnapshotPath(), + snapInfo.getSnapshotID())) { + prevPathSnapId = chainManager.previousPathSnapshot( + snapInfo.getSnapshotPath(), snapInfo.getSnapshotID()); + } + if (chainManager.hasPreviousGlobalSnapshot(snapInfo.getSnapshotID())) { + prevGlobalSnapId = chainManager.previousGlobalSnapshot( + snapInfo.getSnapshotID()); + } + if (chainManager.hasNextPathSnapshot(snapInfo.getSnapshotPath(), + snapInfo.getSnapshotID())) { + nextPathSnapId = chainManager.nextPathSnapshot( + snapInfo.getSnapshotPath(), snapInfo.getSnapshotID()); + } + if (chainManager.hasNextGlobalSnapshot(snapInfo.getSnapshotID())) { + nextGlobalSnapId = chainManager.nextGlobalSnapshot( + snapInfo.getSnapshotID()); + } + + long rowsInTableBeforePurge = omMetadataManager + .countRowsInTable(omMetadataManager.getSnapshotInfoTable()); + // Purge Snapshot of the given index. + List toPurgeList = Collections.singletonList(snapShotToPurge); + OMRequest snapshotPurgeRequest = createPurgeKeysRequest( + toPurgeList); + purgeSnapshots(snapshotPurgeRequest); + + // After purge, check snapshot chain. + if (nextPathSnapId != null) { + SnapshotInfo nextPathSnapshotInfoAfterPurge = metadataManager + .getSnapshotInfoTable().get(chainManager.getTableKey(nextPathSnapId)); + Assertions.assertEquals(nextPathSnapshotInfoAfterPurge + .getGlobalPreviousSnapshotID(), prevPathSnapId); + } + + if (nextGlobalSnapId != null) { + SnapshotInfo nextGlobalSnapshotInfoAfterPurge = metadataManager + .getSnapshotInfoTable().get(chainManager + .getTableKey(nextGlobalSnapId)); + Assertions.assertEquals(nextGlobalSnapshotInfoAfterPurge + .getGlobalPreviousSnapshotID(), prevGlobalSnapId); + } + + Assertions.assertNotEquals(rowsInTableBeforePurge, omMetadataManager + .countRowsInTable(omMetadataManager.getSnapshotInfoTable())); + } } From eeaa9f78da0133138ca714914f27430d213acc2c Mon Sep 17 00:00:00 2001 From: Aswin Shakil Balasubramanian Date: Tue, 21 Mar 2023 09:21:30 -0700 Subject: [PATCH 20/20] Fix Findbugs --- .../TestOMSnapshotPurgeRequestAndResponse.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java index 5787f6e1e741..02814c788e17 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java @@ -72,14 +72,14 @@ public class TestOMSnapshotPurgeRequestAndResponse { private BatchOperation batchOperation; private List checkpointPaths = new ArrayList<>(); - private static OzoneManager ozoneManager; - private static OMMetrics omMetrics; - private static OMMetadataManager omMetadataManager; - private static AuditLogger auditLogger; - - private static String volumeName; - private static String bucketName; - private static String keyName; + private OzoneManager ozoneManager; + private OMMetrics omMetrics; + private OMMetadataManager omMetadataManager; + private AuditLogger auditLogger; + + private String volumeName; + private String bucketName; + private String keyName; // Just setting ozoneManagerDoubleBuffer which does nothing.