From 831cd46ff48d75c0ea4f1c21f907deebbb5f3262 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 9 Sep 2024 17:24:14 -0700 Subject: [PATCH 1/7] HDDS-11440. Add a lastTransactionInfo field in SnapshotInfo to check for transactions in flight on the snapshot Change-Id: Ifa01e118427bd37000eef801a6dc8b12ccfe5d0a --- .../hadoop/hdds/utils/TransactionInfo.java | 11 +++- .../hadoop/ozone/om/helpers/SnapshotInfo.java | 62 ++++++++++++------- .../src/main/proto/OmClientProtocol.proto | 1 + .../hadoop/ozone/om/OmSnapshotManager.java | 35 +++++++++++ .../key/OMDirectoriesPurgeRequestWithFSO.java | 11 ++-- .../om/request/key/OMKeyPurgeRequest.java | 28 ++++++--- .../snapshot/OMSnapshotCreateRequest.java | 3 +- .../OMSnapshotMoveDeletedKeysRequest.java | 25 +++++--- .../snapshot/OMSnapshotPurgeRequest.java | 9 ++- .../OMDirectoriesPurgeResponseWithFSO.java | 4 +- .../om/response/key/OMKeyPurgeResponse.java | 4 +- .../OMSnapshotMoveDeletedKeysResponse.java | 5 ++ .../ozone/om/snapshot/SnapshotUtils.java | 10 +++ .../ozone/om/snapshot/TestSnapshotInfo.java | 44 +++++++++++++ 14 files changed, 203 insertions(+), 49 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java index e7c4ec4ce3d6..68d05af0c6ff 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java @@ -162,7 +162,16 @@ public String toString() { */ public static TransactionInfo readTransactionInfo( DBStoreHAManager metadataManager) throws IOException { - return metadataManager.getTransactionInfoTable().get(TRANSACTION_INFO_KEY); + return readTransactionInfo(metadataManager, false); + } + + /** + * Return transaction info persisted in OM DB skipping cache. + */ + public static TransactionInfo readTransactionInfo( + DBStoreHAManager metadataManager, boolean skipCache) throws IOException { + return skipCache ? metadataManager.getTransactionInfoTable().getSkipCache(TRANSACTION_INFO_KEY) : + metadataManager.getTransactionInfoTable().get(TRANSACTION_INFO_KEY); } public SnapshotInfo toSnapshotInfo() { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index 47a48c37e8e0..54af86348937 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -19,6 +19,7 @@ */ import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.ByteString; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.CopyObject; @@ -124,6 +125,7 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) { private long exclusiveSize; private long exclusiveReplicatedSize; private boolean deepCleanedDeletedDir; + private ByteString lastTransactionInfo; private SnapshotInfo(Builder b) { this.snapshotId = b.snapshotId; @@ -145,6 +147,7 @@ private SnapshotInfo(Builder b) { this.exclusiveSize = b.exclusiveSize; this.exclusiveReplicatedSize = b.exclusiveReplicatedSize; this.deepCleanedDeletedDir = b.deepCleanedDeletedDir; + this.lastTransactionInfo = b.lastTransactionInfo; } public void setName(String name) { @@ -261,13 +264,15 @@ public SnapshotInfo.Builder toBuilder() { .setGlobalPreviousSnapshotId(globalPreviousSnapshotId) .setSnapshotPath(snapshotPath) .setCheckpointDir(checkpointDir) + .setDbTxSequenceNumber(dbTxSequenceNumber) .setDeepClean(deepClean) .setSstFiltered(sstFiltered) .setReferencedSize(referencedSize) .setReferencedReplicatedSize(referencedReplicatedSize) .setExclusiveSize(exclusiveSize) .setExclusiveReplicatedSize(exclusiveReplicatedSize) - .setDeepCleanedDeletedDir(deepCleanedDeletedDir); + .setDeepCleanedDeletedDir(deepCleanedDeletedDir) + .setLastTransactionInfo(lastTransactionInfo); } /** @@ -293,6 +298,7 @@ public static class Builder { private long exclusiveSize; private long exclusiveReplicatedSize; private boolean deepCleanedDeletedDir; + private ByteString lastTransactionInfo; public Builder() { // default values @@ -411,6 +417,16 @@ public Builder setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) { return this; } + public Builder setLastTransactionInfo(byte[] lastTransactionInfo) { + this.lastTransactionInfo = lastTransactionInfo == null ? null : ByteString.copyFrom(lastTransactionInfo); + return this; + } + + public Builder setLastTransactionInfo(ByteString lastTransactionInfo) { + this.lastTransactionInfo = lastTransactionInfo; + return this; + } + public SnapshotInfo build() { Preconditions.checkNotNull(name); return new SnapshotInfo(this); @@ -445,6 +461,10 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() { sib.setGlobalPreviousSnapshotID(toProtobuf(globalPreviousSnapshotId)); } + if (lastTransactionInfo != null) { + sib.setLastTransactionInfo(lastTransactionInfo); + } + sib.setSnapshotPath(snapshotPath) .setCheckpointDir(checkpointDir) .setDbTxSequenceNumber(dbTxSequenceNumber) @@ -513,6 +533,10 @@ public static SnapshotInfo getFromProtobuf( snapshotInfoProto.getDeepCleanedDeletedDir()); } + if (snapshotInfoProto.hasLastTransactionInfo()) { + osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo().toByteArray()); + } + osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()) .setCheckpointDir(snapshotInfoProto.getCheckpointDir()) .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber()); @@ -605,6 +629,14 @@ public void setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) { this.deepCleanedDeletedDir = deepCleanedDeletedDir; } + public ByteString getLastTransactionInfo() { + return lastTransactionInfo; + } + + public void setLastTransactionInfo(byte[] lastTransactionInfo) { + this.lastTransactionInfo = lastTransactionInfo == null ? null : ByteString.copyFrom(lastTransactionInfo); + } + /** * Generate default name of snapshot, (used if user doesn't provide one). */ @@ -673,7 +705,8 @@ public boolean equals(Object o) { referencedReplicatedSize == that.referencedReplicatedSize && exclusiveSize == that.exclusiveSize && exclusiveReplicatedSize == that.exclusiveReplicatedSize && - deepCleanedDeletedDir == that.deepCleanedDeletedDir; + deepCleanedDeletedDir == that.deepCleanedDeletedDir && + Objects.equals(lastTransactionInfo, that.lastTransactionInfo); } @Override @@ -684,7 +717,7 @@ public int hashCode() { globalPreviousSnapshotId, snapshotPath, checkpointDir, deepClean, sstFiltered, referencedSize, referencedReplicatedSize, - exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir); + exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir, lastTransactionInfo); } /** @@ -692,27 +725,7 @@ public int hashCode() { */ @Override public SnapshotInfo copyObject() { - return new Builder() - .setSnapshotId(snapshotId) - .setName(name) - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setSnapshotStatus(snapshotStatus) - .setCreationTime(creationTime) - .setDeletionTime(deletionTime) - .setPathPreviousSnapshotId(pathPreviousSnapshotId) - .setGlobalPreviousSnapshotId(globalPreviousSnapshotId) - .setSnapshotPath(snapshotPath) - .setCheckpointDir(checkpointDir) - .setDbTxSequenceNumber(dbTxSequenceNumber) - .setDeepClean(deepClean) - .setSstFiltered(sstFiltered) - .setReferencedSize(referencedSize) - .setReferencedReplicatedSize(referencedReplicatedSize) - .setExclusiveSize(exclusiveSize) - .setExclusiveReplicatedSize(exclusiveReplicatedSize) - .setDeepCleanedDeletedDir(deepCleanedDeletedDir) - .build(); + return this.toBuilder().build(); } @Override @@ -737,6 +750,7 @@ public String toString() { ", exclusiveSize: '" + exclusiveSize + '\'' + ", exclusiveReplicatedSize: '" + exclusiveReplicatedSize + '\'' + ", deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' + + ", lastTransactionInfo: '" + lastTransactionInfo + '\'' + '}'; } } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index e79797993c13..eefcfa7552ca 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -886,6 +886,7 @@ message SnapshotInfo { optional uint64 exclusiveReplicatedSize = 18; // note: shared sizes can be calculated from: referenced - exclusive optional bool deepCleanedDeletedDir = 19; + optional bytes lastTransactionInfo = 20; } message SnapshotDiffJobProto { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 18b29118182d..5c9333859a2c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.server.ServerUtils; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.CodecRegistry; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; @@ -674,6 +675,40 @@ private ReferenceCounted getSnapshot(String snapshotTableKey, boolea return snapshotCache.get(snapshotInfo.getSnapshotId()); } + /** + * Checks if the last transaction performed on the snapshot has been flushed to disk. + * @param metadataManager Metadatamanager of Active OM. + * @param snapshotTableKey table key corresponding to snapshot in snapshotInfoTable. + * @return True if the changes have been flushed to DB otherwise false + * @throws IOException + */ + public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataManager, String snapshotTableKey) + throws IOException { + // Need this info from cache since the snapshot could have been updated only on cache and not on disk. + SnapshotInfo snapshotInfo = metadataManager.getSnapshotInfoTable().get(snapshotTableKey); + return areSnapshotChangesFlushedToDB(metadataManager, snapshotInfo); + } + + /** + * Checks if the last transaction performed on the snapshot has been flushed to disk. + * @param metadataManager Metadatamanager of Active OM. + * @param snapshotInfo table key corresponding to snapshot in snapshotInfoTable, this should be a value from cache + * and not from disk. + * @return True if the changes have been flushed to DB otherwise false + * @throws IOException + */ + public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataManager, SnapshotInfo snapshotInfo) + throws IOException { + if (snapshotInfo != null) { + TransactionInfo snapshotTransactionInfo = snapshotInfo.getLastTransactionInfo() != null ? + TransactionInfo.getCodec().fromPersistedFormat(snapshotInfo.getLastTransactionInfo().toByteArray()) : null; + TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager, true); + return snapshotTransactionInfo == null || omTransactionInfo.compareTo(snapshotTransactionInfo) >= 0; + } + return false; + } + + /** * Returns OmSnapshot object and skips active check. * This should only be used for API calls initiated by background service e.g. purgeKeys, purgeSnapshot, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index cb10c0d2e40a..422c872b4e18 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -42,7 +42,6 @@ import org.apache.hadoop.ozone.om.response.key.OMDirectoriesPurgeResponseWithFSO; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; @@ -71,7 +70,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn Map, OmBucketInfo> volBucketInfoMap = new HashMap<>(); OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); Map openKeyInfoMap = new HashMap<>(); - + OzoneManagerProtocolProtos.OMResponse.Builder omResponse = + OmResponseUtil.getOMResponseBuilder(getOmRequest()); OMMetrics omMetrics = ozoneManager.getMetrics(); try { if (fromSnapshot != null) { @@ -149,6 +149,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } } } + if (fromSnapshotInfo != null) { + SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshotInfo, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshotInfo.getTableKey()), + CacheValue.get(termIndex.getIndex(), fromSnapshotInfo)); + } } catch (IOException ex) { // Case of IOException for fromProtobuf will not happen // as this is created and send within OM @@ -164,8 +169,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } } - OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( - getOmRequest()); OMClientResponse omClientResponse = new OMDirectoriesPurgeResponseWithFSO( omResponse.build(), purgeRequests, ozoneManager.isRatisEnabled(), getBucketLayout(), volBucketInfoMap, fromSnapshotInfo, openKeyInfoMap); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java index 5738d7945bfe..ab3692bcdd17 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.util.ArrayList; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OzoneManager; @@ -61,6 +64,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn String fromSnapshot = purgeKeysRequest.hasSnapshotTableKey() ? purgeKeysRequest.getSnapshotTableKey() : null; List keysToBePurgedList = new ArrayList<>(); + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( getOmRequest()); @@ -71,17 +75,27 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn keysToBePurgedList.add(deletedKey); } } + final SnapshotInfo fromSnapshotInfo; try { - SnapshotInfo fromSnapshotInfo = null; - if (fromSnapshot != null) { - fromSnapshotInfo = SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot); - } - omClientResponse = new OMKeyPurgeResponse(omResponse.build(), - keysToBePurgedList, fromSnapshotInfo, keysToUpdateList); + fromSnapshotInfo = fromSnapshot == null ? null : SnapshotUtils.getSnapshotInfo(ozoneManager, fromSnapshot); } catch (IOException ex) { - omClientResponse = new OMKeyPurgeResponse(createErrorOMResponse(omResponse, ex)); + return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, ex)); + } + + // Setting transaction info for snapshot, this is to prevent duplicate purge requests to OM from background + // services. + try { + if (fromSnapshotInfo != null) { + SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshotInfo, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshotInfo.getTableKey()), + CacheValue.get(termIndex.getIndex(), fromSnapshotInfo)); + } + } catch (IOException e) { + return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, e)); } + omClientResponse = new OMKeyPurgeResponse(omResponse.build(), keysToBePurgedList, fromSnapshotInfo, + keysToUpdateList); return omClientResponse; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index 3aa4151cea32..c55539070a5e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -166,7 +167,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn ((RDBStore) omMetadataManager.getStore()).getDb() .getLatestSequenceNumber(); snapshotInfo.setDbTxSequenceNumber(dbLatestSequenceNumber); - + SnapshotUtils.setTransactionInfoInSnapshot(snapshotInfo, termIndex); // Snapshot referenced size should be bucket's used bytes OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index df4781bb0ca6..f2fe481b0750 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.om.request.snapshot; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OzoneManager; @@ -82,15 +84,20 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn nextSnapshot = SnapshotUtils.getNextActiveSnapshot(fromSnapshot, snapshotChainManager, ozoneManager); // Get next non-deleted snapshot. - List nextDBKeysList = - moveDeletedKeysRequest.getNextDBKeysList(); - List reclaimKeysList = - moveDeletedKeysRequest.getReclaimKeysList(); - List renamedKeysList = - moveDeletedKeysRequest.getRenamedKeysList(); - List movedDirs = - moveDeletedKeysRequest.getDeletedDirsToMoveList(); - + List nextDBKeysList = moveDeletedKeysRequest.getNextDBKeysList(); + List reclaimKeysList = moveDeletedKeysRequest.getReclaimKeysList(); + List renamedKeysList = moveDeletedKeysRequest.getRenamedKeysList(); + List movedDirs = moveDeletedKeysRequest.getDeletedDirsToMoveList(); + + // Update lastTransactionInfo for fromSnapshot and the nextSnapshot. + SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshot, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), + CacheValue.get(termIndex.getIndex(), fromSnapshot)); + if (nextSnapshot != null) { + SnapshotUtils.setTransactionInfoInSnapshot(nextSnapshot, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(nextSnapshot.getTableKey()), + CacheValue.get(termIndex.getIndex(), nextSnapshot)); + } omClientResponse = new OMSnapshotMoveDeletedKeysResponse( omResponse.build(), fromSnapshot, nextSnapshot, nextDBKeysList, reclaimKeysList, renamedKeysList, movedDirs); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index 47304b416aeb..ef266cc06c0e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -110,9 +110,16 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager, trxnLogIndex); // Step 2: Update the snapshot chain. updateSnapshotChainAndCache(omMetadataManager, fromSnapshot, trxnLogIndex); - // Step 3: Purge the snapshot from SnapshotInfoTable cache. + // Step 3: Purge the snapshot from SnapshotInfoTable cache and also remove from the map. omMetadataManager.getSnapshotInfoTable() .addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), CacheValue.get(trxnLogIndex)); + updatedSnapshotInfos.remove(fromSnapshot.getTableKey()); + } + + for (SnapshotInfo snapshotInfo : updatedSnapshotInfos.values()) { + SnapshotUtils.setTransactionInfoInSnapshot(snapshotInfo, termIndex); + omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(snapshotInfo.getTableKey()), + CacheValue.get(termIndex.getIndex(), snapshotInfo)); } omClientResponse = new OMSnapshotPurgeResponse(omResponse.build(), snapshotDbKeys, updatedSnapshotInfos); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java index 138e942e2b60..28c3e3d758e2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java @@ -48,12 +48,13 @@ import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; /** * Response for {@link OMDirectoriesPurgeRequestWithFSO} request. */ @CleanupTableInfo(cleanupTables = {DELETED_TABLE, DELETED_DIR_TABLE, - DIRECTORY_TABLE, FILE_TABLE}) + DIRECTORY_TABLE, FILE_TABLE, SNAPSHOT_INFO_TABLE}) public class OMDirectoriesPurgeResponseWithFSO extends OmKeyResponse { private static final Logger LOG = LoggerFactory.getLogger(OMDirectoriesPurgeResponseWithFSO.class); @@ -97,6 +98,7 @@ public void addToDBBatch(OMMetadataManager metadataManager, fromSnapshotStore.commitBatchOperation(writeBatch); } } + metadataManager.getSnapshotInfoTable().putWithBatch(batchOp, fromSnapshotInfo.getTableKey(), fromSnapshotInfo); } else { processPaths(metadataManager, batchOp); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java index b59c7d18d408..cd2f7d190f45 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyPurgeResponse.java @@ -39,12 +39,13 @@ import jakarta.annotation.Nonnull; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DELETED_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE; import static org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveDeletedKeysResponse.createRepeatedOmKeyInfo; /** * Response for {@link OMKeyPurgeRequest} request. */ -@CleanupTableInfo(cleanupTables = {DELETED_TABLE}) +@CleanupTableInfo(cleanupTables = {DELETED_TABLE, SNAPSHOT_INFO_TABLE}) public class OMKeyPurgeResponse extends OmKeyResponse { private List purgeKeyList; private SnapshotInfo fromSnapshot; @@ -90,6 +91,7 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, fromSnapshotStore.commitBatchOperation(writeBatch); } } + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot); } else { processKeys(batchOperation, omMetadataManager); processKeysToUpdate(batchOperation, omMetadataManager); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java index 3b5a7454f9db..f39d5827a0cc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotMoveDeletedKeysResponse.java @@ -133,6 +133,11 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, } } + // Flush snapshot info to rocksDB. + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot); + if (nextSnapshot != null) { + omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, nextSnapshot.getTableKey(), nextSnapshot); + } } private void deleteDirsFromSnapshot(BatchOperation batchOp, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index e0f40dabd8a7..94f1a4e7fce8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.snapshot; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; @@ -26,6 +27,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; +import org.apache.ratis.server.protocol.TermIndex; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.slf4j.Logger; @@ -138,6 +140,14 @@ public static void checkSnapshotActive(SnapshotInfo snapInfo, } } + /** + * Set transactionInfo in snapshotInfo. + */ + public static void setTransactionInfoInSnapshot(SnapshotInfo snapshot, TermIndex termIndex) throws IOException { + TransactionInfo transactionInfo = TransactionInfo.valueOf(termIndex); + snapshot.setLastTransactionInfo(TransactionInfo.getCodec().toPersistedFormat(transactionInfo)); + } + /** * Get the next non deleted snapshot in the snapshot chain. */ diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java index dc00433e179b..39d81bf898b5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java @@ -19,12 +19,18 @@ package org.apache.hadoop.ozone.om.snapshot; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; import org.apache.hadoop.util.Time; +import org.apache.ratis.server.protocol.TermIndex; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -114,4 +120,42 @@ public void testSnapshotSSTFilteredFlag() throws Exception { snapshotInfo.put(EXPECTED_SNAPSHOT_KEY, info); assertTrue(snapshotInfo.get(EXPECTED_SNAPSHOT_KEY).isSstFiltered()); } + + @Test + public void testLastTransactionInfo() throws Exception { + Table snapshotInfo = + omMetadataManager.getSnapshotInfoTable(); + SnapshotInfo info = createSnapshotInfo(); + snapshotInfo.put(EXPECTED_SNAPSHOT_KEY, info); + assertEquals(snapshotInfo.get(EXPECTED_SNAPSHOT_KEY).getLastTransactionInfo(), null); + omMetadataManager.getTransactionInfoTable().put(OzoneConsts.TRANSACTION_INFO_KEY, TransactionInfo.valueOf(0, 0)); + // Checking if changes have been flushed when lastTransactionInfo is null + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, info)); + TermIndex termIndex = TermIndex.valueOf(1, 1); + SnapshotUtils.setTransactionInfoInSnapshot(info, termIndex); + // Checking if changes to snapshot object has been updated but not updated on cache or disk. + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + snapshotInfo.addCacheEntry(new CacheKey<>(EXPECTED_SNAPSHOT_KEY), CacheValue.get(termIndex.getIndex(), info)); + + assertEquals(snapshotInfo.get(EXPECTED_SNAPSHOT_KEY).getLastTransactionInfo(), info.getLastTransactionInfo()); + + // Checking if changes have not been flushed when snapshot last transaction info is behind OmTransactionTable value. + assertFalse(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + omMetadataManager.getTransactionInfoTable().addCacheEntry(new CacheKey<>(OzoneConsts.TRANSACTION_INFO_KEY), + CacheValue.get(termIndex.getIndex(), TransactionInfo.valueOf(1, 1))); + assertFalse(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + + // Checking changes are flushed when transaction is equal. + omMetadataManager.getTransactionInfoTable().put(OzoneConsts.TRANSACTION_INFO_KEY, + TransactionInfo.valueOf(1, 1)); + + + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + // Checking changes are flushed when transactionIndex is greater . + omMetadataManager.getTransactionInfoTable().put(OzoneConsts.TRANSACTION_INFO_KEY, TransactionInfo.valueOf(1, 2)); + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + // Checking changes are flushed when both term & transactionIndex is greater. + omMetadataManager.getTransactionInfoTable().put(OzoneConsts.TRANSACTION_INFO_KEY, TransactionInfo.valueOf(2, 2)); + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); + } } From 4ff6a6eaebb7c74445ed4f5a74a7fc40a6ed7edc Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 10 Sep 2024 19:38:04 -0700 Subject: [PATCH 2/7] HDDS-11440. Address review comments Change-Id: I414e862601ceab9dff051484c3bba72546e0bf5a --- .../apache/hadoop/hdds/utils/TransactionInfo.java | 12 ++++-------- .../apache/hadoop/ozone/om/helpers/SnapshotInfo.java | 11 +++-------- .../apache/hadoop/ozone/om/OmSnapshotManager.java | 2 +- .../key/OMDirectoriesPurgeRequestWithFSO.java | 6 ++++-- .../hadoop/ozone/om/snapshot/SnapshotUtils.java | 2 +- 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java index 68d05af0c6ff..d7b55fc7ae69 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Objects; +import com.google.protobuf.ByteString; import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.DelegatedCodec; import org.apache.hadoop.hdds.utils.db.StringCodec; @@ -162,16 +163,11 @@ public String toString() { */ public static TransactionInfo readTransactionInfo( DBStoreHAManager metadataManager) throws IOException { - return readTransactionInfo(metadataManager, false); + return metadataManager.getTransactionInfoTable().getSkipCache(TRANSACTION_INFO_KEY); } - /** - * Return transaction info persisted in OM DB skipping cache. - */ - public static TransactionInfo readTransactionInfo( - DBStoreHAManager metadataManager, boolean skipCache) throws IOException { - return skipCache ? metadataManager.getTransactionInfoTable().getSkipCache(TRANSACTION_INFO_KEY) : - metadataManager.getTransactionInfoTable().get(TRANSACTION_INFO_KEY); + public ByteString toByteString() throws IOException { + return ByteString.copyFrom(getCodec().toPersistedFormat(this)); } public SnapshotInfo toSnapshotInfo() { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index 54af86348937..8584796c2e95 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -417,11 +417,6 @@ public Builder setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) { return this; } - public Builder setLastTransactionInfo(byte[] lastTransactionInfo) { - this.lastTransactionInfo = lastTransactionInfo == null ? null : ByteString.copyFrom(lastTransactionInfo); - return this; - } - public Builder setLastTransactionInfo(ByteString lastTransactionInfo) { this.lastTransactionInfo = lastTransactionInfo; return this; @@ -534,7 +529,7 @@ public static SnapshotInfo getFromProtobuf( } if (snapshotInfoProto.hasLastTransactionInfo()) { - osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo().toByteArray()); + osib.setLastTransactionInfo(snapshotInfoProto.getLastTransactionInfo()); } osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()) @@ -633,8 +628,8 @@ public ByteString getLastTransactionInfo() { return lastTransactionInfo; } - public void setLastTransactionInfo(byte[] lastTransactionInfo) { - this.lastTransactionInfo = lastTransactionInfo == null ? null : ByteString.copyFrom(lastTransactionInfo); + public void setLastTransactionInfo(ByteString lastTransactionInfo) { + this.lastTransactionInfo = lastTransactionInfo; } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 5c9333859a2c..f946335da649 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -702,7 +702,7 @@ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataMa if (snapshotInfo != null) { TransactionInfo snapshotTransactionInfo = snapshotInfo.getLastTransactionInfo() != null ? TransactionInfo.getCodec().fromPersistedFormat(snapshotInfo.getLastTransactionInfo().toByteArray()) : null; - TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager, true); + TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager); return snapshotTransactionInfo == null || omTransactionInfo.compareTo(snapshotTransactionInfo) >= 0; } return false; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index 422c872b4e18..632b78f77ba8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -42,6 +42,7 @@ import org.apache.hadoop.ozone.om.response.key.OMDirectoriesPurgeResponseWithFSO; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import static org.apache.hadoop.ozone.OzoneConsts.DELETED_HSYNC_KEY; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; @@ -70,8 +71,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn Map, OmBucketInfo> volBucketInfoMap = new HashMap<>(); OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); Map openKeyInfoMap = new HashMap<>(); - OzoneManagerProtocolProtos.OMResponse.Builder omResponse = - OmResponseUtil.getOMResponseBuilder(getOmRequest()); + OMMetrics omMetrics = ozoneManager.getMetrics(); try { if (fromSnapshot != null) { @@ -169,6 +169,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } } + OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( + getOmRequest()); OMClientResponse omClientResponse = new OMDirectoriesPurgeResponseWithFSO( omResponse.build(), purgeRequests, ozoneManager.isRatisEnabled(), getBucketLayout(), volBucketInfoMap, fromSnapshotInfo, openKeyInfoMap); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index 94f1a4e7fce8..aa7cd6e3ba2b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -145,7 +145,7 @@ public static void checkSnapshotActive(SnapshotInfo snapInfo, */ public static void setTransactionInfoInSnapshot(SnapshotInfo snapshot, TermIndex termIndex) throws IOException { TransactionInfo transactionInfo = TransactionInfo.valueOf(termIndex); - snapshot.setLastTransactionInfo(TransactionInfo.getCodec().toPersistedFormat(transactionInfo)); + snapshot.setLastTransactionInfo(transactionInfo.toByteString()); } /** From dc55f37442c68eb0dd6ea4561e84c58a73291562 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 10 Sep 2024 19:47:11 -0700 Subject: [PATCH 3/7] HDDS-11440. Address review comments Change-Id: I55b6d5f4503d3a427ddd13c6e6ae4eac091655e0 --- .../java/org/apache/hadoop/hdds/utils/TransactionInfo.java | 4 ++++ .../java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java index d7b55fc7ae69..29531f315184 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java @@ -170,6 +170,10 @@ public ByteString toByteString() throws IOException { return ByteString.copyFrom(getCodec().toPersistedFormat(this)); } + public static TransactionInfo fromByteString(ByteString byteString) throws IOException { + return byteString == null ? null : getCodec().fromPersistedFormat(byteString.toByteArray()); + } + public SnapshotInfo toSnapshotInfo() { return snapshotInfo; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index f946335da649..b63f129a412c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -701,7 +701,7 @@ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataMa throws IOException { if (snapshotInfo != null) { TransactionInfo snapshotTransactionInfo = snapshotInfo.getLastTransactionInfo() != null ? - TransactionInfo.getCodec().fromPersistedFormat(snapshotInfo.getLastTransactionInfo().toByteArray()) : null; + TransactionInfo.fromByteString(snapshotInfo.getLastTransactionInfo()) : null; TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager); return snapshotTransactionInfo == null || omTransactionInfo.compareTo(snapshotTransactionInfo) >= 0; } From e55612959663bf46b217604bc74cee94c14964be Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 10 Sep 2024 19:59:15 -0700 Subject: [PATCH 4/7] HDDS-11440. Address review comments Change-Id: I34decc69bf5b4a1c5d5a5e58865c9ede74dae2dc --- .../java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index b63f129a412c..02db3adbc949 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -694,7 +694,8 @@ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataMa * @param metadataManager Metadatamanager of Active OM. * @param snapshotInfo table key corresponding to snapshot in snapshotInfoTable, this should be a value from cache * and not from disk. - * @return True if the changes have been flushed to DB otherwise false + * @return True if the changes have been flushed to DB otherwise false. It would return true if the snapshot + * provided is null meaning the snapshot doesn't exist. * @throws IOException */ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataManager, SnapshotInfo snapshotInfo) @@ -703,9 +704,10 @@ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataMa TransactionInfo snapshotTransactionInfo = snapshotInfo.getLastTransactionInfo() != null ? TransactionInfo.fromByteString(snapshotInfo.getLastTransactionInfo()) : null; TransactionInfo omTransactionInfo = TransactionInfo.readTransactionInfo(metadataManager); + // If transactionInfo field is null then return true to keep things backward compatible. return snapshotTransactionInfo == null || omTransactionInfo.compareTo(snapshotTransactionInfo) >= 0; } - return false; + return true; } From b32d074af41f13bb8c5975612939be2b9871edd4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 10 Sep 2024 20:01:06 -0700 Subject: [PATCH 5/7] HDDS-11440. Address review comments Change-Id: Idbaf521397c170b381530f86f7fe9f215db5169d --- .../java/org/apache/hadoop/ozone/om/OmSnapshotManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 02db3adbc949..dde5b22e7937 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -692,8 +692,7 @@ public static boolean areSnapshotChangesFlushedToDB(OMMetadataManager metadataMa /** * Checks if the last transaction performed on the snapshot has been flushed to disk. * @param metadataManager Metadatamanager of Active OM. - * @param snapshotInfo table key corresponding to snapshot in snapshotInfoTable, this should be a value from cache - * and not from disk. + * @param snapshotInfo SnapshotInfo value. * @return True if the changes have been flushed to DB otherwise false. It would return true if the snapshot * provided is null meaning the snapshot doesn't exist. * @throws IOException From 331acddb734ac86970318864502c30159a69578e Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 11 Sep 2024 08:06:12 -0700 Subject: [PATCH 6/7] HDDS-11440. Address review comments Change-Id: I789b587865e4416b61eeb4c2f88d2a838d5f3cb2 --- .../apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java index 39d81bf898b5..98290e44fb3a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java @@ -41,6 +41,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -127,7 +128,9 @@ public void testLastTransactionInfo() throws Exception { omMetadataManager.getSnapshotInfoTable(); SnapshotInfo info = createSnapshotInfo(); snapshotInfo.put(EXPECTED_SNAPSHOT_KEY, info); - assertEquals(snapshotInfo.get(EXPECTED_SNAPSHOT_KEY).getLastTransactionInfo(), null); + assertNull(snapshotInfo.get(EXPECTED_SNAPSHOT_KEY).getLastTransactionInfo()); + // checking if true value is returned when snapshot is null. + assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, (SnapshotInfo)null)); omMetadataManager.getTransactionInfoTable().put(OzoneConsts.TRANSACTION_INFO_KEY, TransactionInfo.valueOf(0, 0)); // Checking if changes have been flushed when lastTransactionInfo is null assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, info)); From 36aed3770a85546a69197e65566075d4e4be8a92 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 11 Sep 2024 16:48:12 -0700 Subject: [PATCH 7/7] HDDS-11440. Add test cases for each request method Change-Id: Ib455f6d871411fdee434ac142e6e0bcee14baab8 --- .../key/OMDirectoriesPurgeRequestWithFSO.java | 3 +- .../om/request/key/OMKeyPurgeRequest.java | 3 +- .../snapshot/OMSnapshotCreateRequest.java | 4 +- .../OMSnapshotMoveDeletedKeysRequest.java | 5 +- .../snapshot/OMSnapshotPurgeRequest.java | 3 +- .../ozone/om/snapshot/SnapshotUtils.java | 10 -- ...tOMDirectoriesPurgeRequestAndResponse.java | 91 +++++++++++++++---- .../key/TestOMKeyPurgeRequestAndResponse.java | 48 +++------- .../om/request/key/TestOMKeyRequest.java | 39 +++++++- .../snapshot/TestOMSnapshotCreateRequest.java | 4 +- ...TestOMSnapshotPurgeRequestAndResponse.java | 36 ++++++-- .../TestOMSnapshotCreateResponse.java | 4 +- .../ozone/om/snapshot/TestSnapshotInfo.java | 2 +- 13 files changed, 169 insertions(+), 83 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index 632b78f77ba8..dd08ff171654 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneConsts; @@ -150,7 +151,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } } if (fromSnapshotInfo != null) { - SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshotInfo, termIndex); + fromSnapshotInfo.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshotInfo.getTableKey()), CacheValue.get(termIndex.getIndex(), fromSnapshotInfo)); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java index ab3692bcdd17..14c80bb7a93b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.ArrayList; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; @@ -87,7 +88,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn // services. try { if (fromSnapshotInfo != null) { - SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshotInfo, termIndex); + fromSnapshotInfo.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshotInfo.getTableKey()), CacheValue.get(termIndex.getIndex(), fromSnapshotInfo)); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index c55539070a5e..2ded4f6a83e9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -21,7 +21,7 @@ import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -167,7 +167,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn ((RDBStore) omMetadataManager.getStore()).getDb() .getLatestSequenceNumber(); snapshotInfo.setDbTxSequenceNumber(dbLatestSequenceNumber); - SnapshotUtils.setTransactionInfoInSnapshot(snapshotInfo, termIndex); + snapshotInfo.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); // Snapshot referenced size should be bucket's used bytes OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index f2fe481b0750..58fdb1232d31 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.om.request.snapshot; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.ratis.server.protocol.TermIndex; @@ -90,11 +91,11 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn List movedDirs = moveDeletedKeysRequest.getDeletedDirsToMoveList(); // Update lastTransactionInfo for fromSnapshot and the nextSnapshot. - SnapshotUtils.setTransactionInfoInSnapshot(fromSnapshot, termIndex); + fromSnapshot.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(fromSnapshot.getTableKey()), CacheValue.get(termIndex.getIndex(), fromSnapshot)); if (nextSnapshot != null) { - SnapshotUtils.setTransactionInfoInSnapshot(nextSnapshot, termIndex); + nextSnapshot.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(nextSnapshot.getTableKey()), CacheValue.get(termIndex.getIndex(), nextSnapshot)); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index ef266cc06c0e..6602f52514b5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om.request.snapshot; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OMMetrics; import org.apache.ratis.server.protocol.TermIndex; @@ -117,7 +118,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn } for (SnapshotInfo snapshotInfo : updatedSnapshotInfos.values()) { - SnapshotUtils.setTransactionInfoInSnapshot(snapshotInfo, termIndex); + snapshotInfo.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); omMetadataManager.getSnapshotInfoTable().addCacheEntry(new CacheKey<>(snapshotInfo.getTableKey()), CacheValue.get(termIndex.getIndex(), snapshotInfo)); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java index aa7cd6e3ba2b..e0f40dabd8a7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotUtils.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om.snapshot; -import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; @@ -27,7 +26,6 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus; -import org.apache.ratis.server.protocol.TermIndex; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.slf4j.Logger; @@ -140,14 +138,6 @@ public static void checkSnapshotActive(SnapshotInfo snapInfo, } } - /** - * Set transactionInfo in snapshotInfo. - */ - public static void setTransactionInfoInSnapshot(SnapshotInfo snapshot, TermIndex termIndex) throws IOException { - TransactionInfo transactionInfo = TransactionInfo.valueOf(termIndex); - snapshot.setLastTransactionInfo(transactionInfo.toByteString()); - } - /** * Get the next non deleted snapshot in the snapshot chain. */ diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java index cbb782e184fe..9eb8738b9d48 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java @@ -26,18 +26,23 @@ import java.util.List; import java.util.UUID; import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.response.key.OMDirectoriesPurgeResponseWithFSO; import org.apache.hadoop.ozone.om.response.key.OMKeyPurgeResponse; +import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import jakarta.annotation.Nonnull; @@ -109,7 +114,7 @@ private void updateBlockInfo(OmKeyInfo omKeyInfo) throws IOException { * Create OMRequest which encapsulates DeleteKeyRequest. * @return OMRequest */ - private OMRequest createPurgeKeysRequest(String purgeDeletedDir, + private OMRequest createPurgeKeysRequest(String fromSnapshot, String purgeDeletedDir, List keyList, OmBucketInfo bucketInfo) throws IOException { List purgePathRequestList = new ArrayList<>(); @@ -127,7 +132,9 @@ private OMRequest createPurgeKeysRequest(String purgeDeletedDir, OzoneManagerProtocolProtos.PurgeDirectoriesRequest.Builder purgeDirRequest = OzoneManagerProtocolProtos.PurgeDirectoriesRequest.newBuilder(); purgeDirRequest.addAllDeletedPath(purgePathRequestList); - + if (fromSnapshot != null) { + purgeDirRequest.setSnapshotTableKey(fromSnapshot); + } OzoneManagerProtocolProtos.OMRequest omRequest = OzoneManagerProtocolProtos.OMRequest.newBuilder() .setCmdType(OzoneManagerProtocolProtos.Type.PurgeDirectories) @@ -138,8 +145,7 @@ private OMRequest createPurgeKeysRequest(String purgeDeletedDir, } private OzoneManagerProtocolProtos.PurgePathRequest wrapPurgeRequest( final long volumeId, final long bucketId, final String purgeDeletedDir, - final List purgeDeletedFiles, - final List markDirsAsDeleted) { + final List purgeDeletedFiles, final List markDirsAsDeleted) { // Put all keys to be purged in a list OzoneManagerProtocolProtos.PurgePathRequest.Builder purgePathsRequest = OzoneManagerProtocolProtos.PurgePathRequest.newBuilder(); @@ -182,13 +188,13 @@ public void testValidateAndUpdateCacheCheckQuota() throws Exception { // Create and Delete keys. The keys should be moved to DeletedKeys table List deletedKeyInfos = createAndDeleteKeys(1, null); // The keys should be present in the DeletedKeys table before purging - List deletedKeyNames = validateDeletedKeysTable(deletedKeyInfos); + List deletedKeyNames = validateDeletedKeysTable(omMetadataManager, deletedKeyInfos, true); // Create PurgeKeysRequest to purge the deleted keys String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get( bucketKey); - OMRequest omRequest = createPurgeKeysRequest( + OMRequest omRequest = createPurgeKeysRequest(null, null, deletedKeyInfos, omBucketInfo); OMRequest preExecutedRequest = preExecute(omRequest); OMDirectoriesPurgeRequestWithFSO omKeyPurgeRequest = @@ -205,7 +211,59 @@ public void testValidateAndUpdateCacheCheckQuota() throws Exception { performBatchOperationCommit(omClientResponse); // The keys should exist in the DeletedKeys table after dir delete - validateDeletedKeys(deletedKeyNames); + validateDeletedKeys(omMetadataManager, deletedKeyNames); + } + + @Test + public void testValidateAndUpdateCacheSnapshotLastTransactionInfoUpdated() throws Exception { + // Create and Delete keys. The keys should be moved to DeletedKeys table + List deletedKeyInfos = createAndDeleteKeys(1, null); + // The keys should be present in the DeletedKeys table before purging + List deletedKeyNames = validateDeletedKeysTable(omMetadataManager, deletedKeyInfos, true); + + String snapshotName = "snap1"; + SnapshotInfo snapshotInfo = createSnapshot(snapshotName); + ReferenceCounted rcOmSnapshot = ozoneManager.getOmSnapshotManager() + .getSnapshot(snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), snapshotInfo.getName()); + // Keys should be present in snapshot + validateDeletedKeysTable(rcOmSnapshot.get().getMetadataManager(), deletedKeyInfos, true); + // keys should have been moved from AOS + validateDeletedKeysTable(omMetadataManager, deletedKeyInfos, false); + + // Create PurgeKeysRequest to purge the deleted keys + assertEquals(snapshotInfo.getLastTransactionInfo(), + TransactionInfo.valueOf(TransactionInfo.getTermIndex(1L)).toByteString()); + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get( + bucketKey); + OMRequest omRequest = createPurgeKeysRequest(snapshotInfo.getTableKey(), + null, deletedKeyInfos, omBucketInfo); + OMRequest preExecutedRequest = preExecute(omRequest); + OMDirectoriesPurgeRequestWithFSO omKeyPurgeRequest = + new OMDirectoriesPurgeRequestWithFSO(preExecutedRequest); + + assertEquals(1000L * deletedKeyNames.size(), omBucketInfo.getUsedBytes()); + OMDirectoriesPurgeResponseWithFSO omClientResponse + = (OMDirectoriesPurgeResponseWithFSO) omKeyPurgeRequest + .validateAndUpdateCache(ozoneManager, 100L); + + SnapshotInfo snapshotInfoOnDisk = omMetadataManager.getSnapshotInfoTable().getSkipCache(snapshotInfo.getTableKey()); + SnapshotInfo updatedSnapshotInfo = omMetadataManager.getSnapshotInfoTable().get(snapshotInfo.getTableKey()); + + assertEquals(snapshotInfoOnDisk, snapshotInfo); + snapshotInfo.setLastTransactionInfo(TransactionInfo.valueOf(TransactionInfo.getTermIndex(100L)) + .toByteString()); + assertEquals(snapshotInfo, updatedSnapshotInfo); + omBucketInfo = omMetadataManager.getBucketTable().get(bucketKey); + assertEquals(0L * deletedKeyNames.size(), omBucketInfo.getUsedBytes()); + + performBatchOperationCommit(omClientResponse); + + // The keys should exist in the DeletedKeys table after dir delete + validateDeletedKeys(rcOmSnapshot.get().getMetadataManager(), deletedKeyNames); + snapshotInfoOnDisk = omMetadataManager.getSnapshotInfoTable().getSkipCache(snapshotInfo.getTableKey()); + assertEquals(snapshotInfo, snapshotInfoOnDisk); + rcOmSnapshot.close(); } @Test @@ -214,13 +272,13 @@ public void testValidateAndUpdateCacheQuotaBucketRecreated() // Create and Delete keys. The keys should be moved to DeletedKeys table List deletedKeyInfos = createAndDeleteKeys(1, null); // The keys should be present in the DeletedKeys table before purging - List deletedKeyNames = validateDeletedKeysTable(deletedKeyInfos); + List deletedKeyNames = validateDeletedKeysTable(omMetadataManager, deletedKeyInfos, true); // Create PurgeKeysRequest to purge the deleted keys String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get( bucketKey); - OMRequest omRequest = createPurgeKeysRequest( + OMRequest omRequest = createPurgeKeysRequest(null, null, deletedKeyInfos, omBucketInfo); OMRequest preExecutedRequest = preExecute(omRequest); OMDirectoriesPurgeRequestWithFSO omKeyPurgeRequest = @@ -258,35 +316,32 @@ public void testValidateAndUpdateCacheQuotaBucketRecreated() performBatchOperationCommit(omClientResponse); // The keys should exist in the DeletedKeys table after dir delete - validateDeletedKeys(deletedKeyNames); + validateDeletedKeys(omMetadataManager, deletedKeyNames); } - private void performBatchOperationCommit( - OMDirectoriesPurgeResponseWithFSO omClientResponse) throws IOException { + private void performBatchOperationCommit(OMDirectoriesPurgeResponseWithFSO omClientResponse) throws IOException { try (BatchOperation batchOperation = omMetadataManager.getStore().initBatchOperation()) { - omClientResponse.addToDBBatch(omMetadataManager, batchOperation); - // Do manual commit and see whether addToBatch is successful or not. omMetadataManager.getStore().commitBatchOperation(batchOperation); } } @Nonnull - private List validateDeletedKeysTable( - List deletedKeyInfos) throws IOException { + private List validateDeletedKeysTable(OMMetadataManager omMetadataManager, + List deletedKeyInfos, boolean keyExists) throws IOException { List deletedKeyNames = new ArrayList<>(); for (OmKeyInfo deletedKey : deletedKeyInfos) { String keyName = omMetadataManager.getOzoneKey(deletedKey.getVolumeName(), deletedKey.getBucketName(), deletedKey.getKeyName()); - assertTrue(omMetadataManager.getDeletedTable().isExist(keyName)); + assertEquals(omMetadataManager.getDeletedTable().isExist(keyName), keyExists); deletedKeyNames.add(keyName); } return deletedKeyNames; } - private void validateDeletedKeys( + private void validateDeletedKeys(OMMetadataManager omMetadataManager, List deletedKeyNames) throws IOException { for (String deletedKey : deletedKeyNames) { assertTrue(omMetadataManager.getDeletedTable().isExist( diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java index 2cd0de920be4..c323fecd5015 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyPurgeRequestAndResponse.java @@ -23,12 +23,10 @@ import java.util.List; import java.util.UUID; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; -import org.apache.hadoop.ozone.om.request.snapshot.TestOMSnapshotCreateRequest; -import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotCreateResponse; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.junit.jupiter.api.Test; @@ -42,12 +40,10 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.hdds.utils.db.BatchOperation; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.when; /** * Tests {@link OMKeyPurgeRequest} and {@link OMKeyPurgeResponse}. @@ -115,35 +111,6 @@ private OMRequest createPurgeKeysRequest(List deletedKeys, .build(); } - /** - * Create snapshot and checkpoint directory. - */ - private SnapshotInfo createSnapshot(String snapshotName) throws Exception { - when(ozoneManager.isAdmin(any())).thenReturn(true); - BatchOperation batchOperation = omMetadataManager.getStore() - .initBatchOperation(); - OMRequest omRequest = OMRequestTestUtils - .createSnapshotRequest(volumeName, bucketName, snapshotName); - // Pre-Execute OMSnapshotCreateRequest. - OMSnapshotCreateRequest omSnapshotCreateRequest = - TestOMSnapshotCreateRequest.doPreExecute(omRequest, ozoneManager); - - // validateAndUpdateCache OMSnapshotCreateResponse. - OMSnapshotCreateResponse omClientResponse = (OMSnapshotCreateResponse) - omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1L); - // Add to batch and commit to DB. - omClientResponse.addToDBBatch(omMetadataManager, batchOperation); - omMetadataManager.getStore().commitBatchOperation(batchOperation); - batchOperation.close(); - - String key = SnapshotInfo.getTableKey(volumeName, - bucketName, snapshotName); - SnapshotInfo snapshotInfo = - omMetadataManager.getSnapshotInfoTable().get(key); - assertNotNull(snapshotInfo); - return snapshotInfo; - } - private OMRequest preExecute(OMRequest originalOmRequest) throws IOException { OMKeyPurgeRequest omKeyPurgeRequest = new OMKeyPurgeRequest(originalOmRequest); @@ -205,6 +172,8 @@ public void testKeyPurgeInSnapshot() throws Exception { List deletedKeyNames = createAndDeleteKeys(1, null); SnapshotInfo snapInfo = createSnapshot("snap1"); + assertEquals(snapInfo.getLastTransactionInfo(), + TransactionInfo.valueOf(TransactionInfo.getTermIndex(1L)).toByteString()); // The keys should be not present in the active Db's deletedTable for (String deletedKey : deletedKeyNames) { assertFalse(omMetadataManager.getDeletedTable().isExist(deletedKey)); @@ -230,6 +199,12 @@ public void testKeyPurgeInSnapshot() throws Exception { omKeyPurgeRequest.validateAndUpdateCache(ozoneManager, 100L); + SnapshotInfo snapshotInfoOnDisk = omMetadataManager.getSnapshotInfoTable().getSkipCache(snapInfo.getTableKey()); + SnapshotInfo updatedSnapshotInfo = omMetadataManager.getSnapshotInfoTable().get(snapInfo.getTableKey()); + assertEquals(snapshotInfoOnDisk, snapInfo); + snapInfo.setLastTransactionInfo(TransactionInfo.valueOf(TransactionInfo.getTermIndex(100L)) + .toByteString()); + assertEquals(snapInfo, updatedSnapshotInfo); OMResponse omResponse = OMResponse.newBuilder() .setPurgeKeysResponse(PurgeKeysResponse.getDefaultInstance()) .setCmdType(Type.PurgeKeys) @@ -245,7 +220,8 @@ public void testKeyPurgeInSnapshot() throws Exception { // Do manual commit and see whether addToBatch is successful or not. omMetadataManager.getStore().commitBatchOperation(batchOperation); } - + snapshotInfoOnDisk = omMetadataManager.getSnapshotInfoTable().getSkipCache(snapInfo.getTableKey()); + assertEquals(snapshotInfoOnDisk, snapInfo); // The keys should not exist in the DeletedKeys table for (String deletedKey : deletedKeyNames) { assertFalse(omSnapshot.getMetadataManager() diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index c1b0e45e6d60..e13508ebd5f1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.om.IOmMetadataReader; import org.apache.hadoop.ozone.om.OMPerformanceMetrics; @@ -43,9 +44,15 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.ozone.om.request.snapshot.OMSnapshotCreateRequest; +import org.apache.hadoop.ozone.om.request.snapshot.TestOMSnapshotCreateRequest; +import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotCreateResponse; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer; import org.apache.hadoop.security.UserGroupInformation; @@ -236,7 +243,7 @@ public void setup() throws Exception { .thenReturn(bucket); when(ozoneManager.resolveBucketLink(any(Pair.class))) .thenReturn(bucket); - OmSnapshotManager omSnapshotManager = new OmSnapshotManager(ozoneManager); + OmSnapshotManager omSnapshotManager = Mockito.spy(new OmSnapshotManager(ozoneManager)); when(ozoneManager.getOmSnapshotManager()) .thenReturn(omSnapshotManager); @@ -284,4 +291,34 @@ public void stop() { omMetrics.unRegister(); framework().clearInlineMocks(); } + + /** + * Create snapshot and checkpoint directory. + */ + protected SnapshotInfo createSnapshot(String snapshotName) throws Exception { + when(ozoneManager.isAdmin(any())).thenReturn(true); + BatchOperation batchOperation = omMetadataManager.getStore() + .initBatchOperation(); + OzoneManagerProtocolProtos.OMRequest omRequest = OMRequestTestUtils + .createSnapshotRequest(volumeName, bucketName, snapshotName); + // Pre-Execute OMSnapshotCreateRequest. + OMSnapshotCreateRequest omSnapshotCreateRequest = + TestOMSnapshotCreateRequest.doPreExecute(omRequest, ozoneManager); + + // validateAndUpdateCache OMSnapshotCreateResponse. + OMSnapshotCreateResponse omClientResponse = (OMSnapshotCreateResponse) + omSnapshotCreateRequest.validateAndUpdateCache(ozoneManager, 1L); + // Add to batch and commit to DB. + omClientResponse.addToDBBatch(omMetadataManager, batchOperation); + omMetadataManager.getStore().commitBatchOperation(batchOperation); + batchOperation.close(); + + String key = SnapshotInfo.getTableKey(volumeName, + bucketName, snapshotName); + SnapshotInfo snapshotInfo = + omMetadataManager.getSnapshotInfoTable().get(key); + assertNotNull(snapshotInfo); + return snapshotInfo; + } + } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index 3997f39d7bd5..806c1b90f7f3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.audit.AuditLogger; @@ -229,7 +230,8 @@ public void testValidateAndUpdateCache() throws Exception { omMetadataManager.getSnapshotInfoTable().get(key); assertNotNull(snapshotInfoInCache); assertEquals(snapshotInfoFromProto, snapshotInfoInCache); - + assertEquals(snapshotInfoInCache.getLastTransactionInfo(), + TransactionInfo.valueOf(TransactionInfo.getTermIndex(1L)).toByteString()); assertEquals(0, omMetrics.getNumSnapshotCreateFails()); assertEquals(1, omMetrics.getNumSnapshotActive()); assertEquals(1, omMetrics.getNumSnapshotCreates()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java index 8edd096e766c..d0a5559a87b8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java @@ -19,7 +19,9 @@ package org.apache.hadoop.ozone.om.request.snapshot; +import com.google.protobuf.ByteString; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.Table; @@ -57,6 +59,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; @@ -450,12 +453,29 @@ public void testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge( .countRowsInTable(omMetadataManager.getSnapshotInfoTable()); assertEquals(totalKeys, numberOfSnapshotBeforePurge); assertEquals(totalKeys, chainManager.getGlobalSnapshotChain().size()); - - validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain(snapshotInfoList); - + Map expectedTransactionInfos = new HashMap<>(); + // Ratis transaction uses term index 1 while creating snapshot. + ByteString expectedLastTransactionVal = TransactionInfo.valueOf(TransactionInfo.getTermIndex(1L)) + .toByteString(); + for (SnapshotInfo snapshotInfo : snapshotInfoList) { + expectedTransactionInfos.put(snapshotInfo.getSnapshotId(), expectedLastTransactionVal); + } + validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain(snapshotInfoList, expectedTransactionInfos); + // Ratis transaction uses term index 200 while purging snapshot. + expectedLastTransactionVal = TransactionInfo.valueOf(TransactionInfo.getTermIndex(200L)) + .toByteString(); List purgeSnapshotKeys = new ArrayList<>(); for (int i = fromIndex; i <= toIndex; i++) { SnapshotInfo purgeSnapshotInfo = snapshotInfoList.get(i); + UUID snapId = purgeSnapshotInfo.getSnapshotId(); + // expecting nextPathSnapshot & nextGlobalSnapshot in chain gets updated. + if (chainManager.hasNextGlobalSnapshot(snapId)) { + expectedTransactionInfos.put(chainManager.nextGlobalSnapshot(snapId), expectedLastTransactionVal); + } + if (chainManager.hasNextPathSnapshot(purgeSnapshotInfo.getSnapshotPath(), snapId)) { + expectedTransactionInfos.put(chainManager.nextPathSnapshot(purgeSnapshotInfo.getSnapshotPath(), snapId), + expectedLastTransactionVal); + } String purgeSnapshotKey = SnapshotInfo.getTableKey(volumeName, purgeSnapshotInfo.getBucketName(), purgeSnapshotInfo.getName()); @@ -484,17 +504,17 @@ public void testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge( actualNumberOfSnapshotAfterPurge); assertEquals(expectNumberOfSnapshotAfterPurge, chainManager .getGlobalSnapshotChain().size()); - validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain( - snapshotInfoListAfterPurge); + validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain(snapshotInfoListAfterPurge, expectedTransactionInfos); } private void validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain( - List snapshotInfoList - ) throws IOException { + List snapshotInfoList, Map expectedTransactionInfos) throws IOException { if (snapshotInfoList.isEmpty()) { return; } - + for (SnapshotInfo snapshotInfo : snapshotInfoList) { + assertEquals(snapshotInfo.getLastTransactionInfo(), expectedTransactionInfos.get(snapshotInfo.getSnapshotId())); + } OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl) omMetadataManager; SnapshotChainManager chainManager = metadataManager diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java index 7f74f3d17ecf..a370c20ad1b6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java @@ -27,6 +27,7 @@ import java.util.UUID; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -98,7 +99,8 @@ public void testAddToDBBatch(int numberOfKeys) throws Exception { snapshotName, snapshotId, Time.now()); - + snapshotInfo.setLastTransactionInfo( + TransactionInfo.valueOf(TransactionInfo.getTermIndex(1L)).toByteString()); // confirm table is empty assertEquals(0, omMetadataManager .countRowsInTable(omMetadataManager.getSnapshotInfoTable())); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java index 98290e44fb3a..29e0115861fe 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotInfo.java @@ -135,7 +135,7 @@ public void testLastTransactionInfo() throws Exception { // Checking if changes have been flushed when lastTransactionInfo is null assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, info)); TermIndex termIndex = TermIndex.valueOf(1, 1); - SnapshotUtils.setTransactionInfoInSnapshot(info, termIndex); + info.setLastTransactionInfo(TransactionInfo.valueOf(termIndex).toByteString()); // Checking if changes to snapshot object has been updated but not updated on cache or disk. assertTrue(OmSnapshotManager.areSnapshotChangesFlushedToDB(omMetadataManager, EXPECTED_SNAPSHOT_KEY)); snapshotInfo.addCacheEntry(new CacheKey<>(EXPECTED_SNAPSHOT_KEY), CacheValue.get(termIndex.getIndex(), info));