diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 660b21c45bcf..596eb1276560 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@ -272,6 +272,8 @@ public enum ResultCodes { TOO_MANY_BUCKETS, KEY_UNDER_LEASE_RECOVERY, KEY_ALREADY_CLOSED, - KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD + KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD, + + TOO_MANY_SNAPSHOTS, } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java index 874044c75ddb..11de3873c459 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerHASnapshot.java @@ -342,6 +342,29 @@ public void testKeyAndSnapshotDeletionService() throws IOException, InterruptedE checkSnapshotIsPurgedFromDB(omFollower, tableKey); } + @Test + public void testSnapshotInFlightCount() throws Exception { + // snapshot inflight count should be reset to 0 when leader changes + + // first do some snapshot creations + String snapshotName1 = UUID.randomUUID().toString(); + store.createSnapshot(volumeName, bucketName, snapshotName1); + + // then shutdown the leader + OzoneManager omLeader = cluster.getOMLeader(); + cluster.shutdownOzoneManager(omLeader); + + // wait for the new leader to be elected + cluster.waitForLeaderOM(); + + // check the inflight count on the new leader is 0 + OzoneManager newLeader = cluster.getOMLeader(); + assertEquals(0, newLeader.getOmSnapshotManager().getInFlightSnapshotCount()); + + // restart the previous shutdowned node + cluster.restartOzoneManager(omLeader, true); + } + private void createSnapshot(String volName, String buckName, String snapName) throws IOException { store.createSnapshot(volName, buckName, snapName); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 4108af198bff..92edafa9426d 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -564,6 +564,7 @@ enum Status { KEY_ALREADY_CLOSED = 96; KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 97; + TOO_MANY_SNAPSHOTS = 98; } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 6629e911af98..788271042212 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -24,6 +24,8 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE; @@ -63,6 +65,8 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.StringUtils; @@ -173,6 +177,9 @@ public final class OmSnapshotManager implements AutoCloseable { // Soft limit of the snapshot cache size. private final int softCacheSize; + private int fsSnapshotMaxLimit; + private final AtomicInteger inFlightSnapshotCount = new AtomicInteger(0); + public OmSnapshotManager(OzoneManager ozoneManager) { boolean isFilesystemSnapshotEnabled = @@ -247,6 +254,9 @@ public OmSnapshotManager(OzoneManager ozoneManager) { this.softCacheSize = ozoneManager.getConfiguration().getInt( OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE, OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE_DEFAULT); + + fsSnapshotMaxLimit = ozoneManager.getConfiguration().getInt(OZONE_OM_FS_SNAPSHOT_MAX_LIMIT, + OZONE_OM_FS_SNAPSHOT_MAX_LIMIT_DEFAULT); CacheLoader loader = createCacheLoader(); @@ -859,6 +869,42 @@ private void validateSnapshotsExistAndActive(final String volumeName, checkSnapshotActive(toSnapInfo, false); } + public void snapshotLimitCheck() throws IOException, OMException { + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) ozoneManager.getMetadataManager(); + SnapshotChainManager snapshotChainManager = omMetadataManager.getSnapshotChainManager(); + int currentSnapshotNum = snapshotChainManager.getGlobalSnapshotChain().size(); + + AtomicReference exceptionRef = new AtomicReference<>(null); + inFlightSnapshotCount.updateAndGet(count -> { + if (currentSnapshotNum + count >= fsSnapshotMaxLimit) { + exceptionRef.set(new OMException( + String.format("Snapshot limit of %d reached. Cannot create more snapshots. " + + "Current snapshots: %d, In-flight creations: %d", + fsSnapshotMaxLimit, currentSnapshotNum, count) + + " If you already deleted some snapshots, " + + "please wait for the background service to complete the cleanup.", + OMException.ResultCodes.TOO_MANY_SNAPSHOTS)); + return count; + } + return count + 1; + }); + if (exceptionRef.get() != null) { + throw exceptionRef.get(); + } + } + + public void decrementInFlightSnapshotCount() { + inFlightSnapshotCount.decrementAndGet(); + } + + public void resetInFlightSnapshotCount() { + inFlightSnapshotCount.set(0); + } + + public int getInFlightSnapshotCount() { + return inFlightSnapshotCount.get(); + } + private int getIndexFromToken(final String token) throws IOException { if (isBlank(token)) { return 0; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java index 8eadd5bff06a..0ad3844716ce 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java @@ -156,6 +156,11 @@ public SnapshotInfo getLatestSnapshot() { return snapshotInfo; } + @Override + public void notifyLeaderReady() { + ozoneManager.getOmSnapshotManager().resetInFlightSnapshotCount(); + } + @Override public void notifyLeaderChanged(RaftGroupMemberId groupMemberId, RaftPeerId newLeaderId) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index 900419ad4223..8efa517a750b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -120,6 +120,8 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { "Only bucket owners and Ozone admins can create snapshots", OMException.ResultCodes.PERMISSION_DENIED); } + // verify snapshot limit + ozoneManager.getOmSnapshotManager().snapshotLimitCheck(); CreateSnapshotRequest.Builder createSnapshotRequest = omRequest.getCreateSnapshotRequest().toBuilder() .setSnapshotId(toProtobuf(UUID.randomUUID())) .setVolumeName(volumeName) @@ -188,7 +190,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut // pre-replicated key size counter in OmBucketInfo. snapshotInfo.setReferencedSize(estimateBucketDataSize(omBucketInfo)); - addSnapshotInfoToSnapshotChainAndCache(omMetadataManager, context.getIndex()); + addSnapshotInfoToSnapshotChainAndCache(ozoneManager, omMetadataManager, context.getIndex()); omResponse.setCreateSnapshotResponse( CreateSnapshotResponse.newBuilder() @@ -250,6 +252,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut * it was removed at T-5. */ private void addSnapshotInfoToSnapshotChainAndCache( + OzoneManager ozoneManager, OmMetadataManagerImpl omMetadataManager, long transactionLogIndex ) throws IOException { @@ -288,6 +291,8 @@ private void addSnapshotInfoToSnapshotChainAndCache( removeSnapshotInfoFromSnapshotChainManager(snapshotChainManager, snapshotInfo); throw new IOException(exception.getMessage(), exception); + } finally { + ozoneManager.getOmSnapshotManager().decrementInFlightSnapshotCount(); } } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index ef5ae4ab5900..f1c42449b182 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -32,6 +32,7 @@ import static org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.getINode; import static org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.truncateFileName; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -54,6 +55,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -66,6 +68,7 @@ import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; import org.apache.hadoop.hdds.utils.db.RDBStore; +import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TypedTable; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -76,6 +79,7 @@ import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -90,6 +94,9 @@ class TestOmSnapshotManager { private OzoneManager om; + private SnapshotChainManager snapshotChainManager; + private OmMetadataManagerImpl omMetadataManager; + private OmSnapshotManager omSnapshotManager; private static final String CANDIDATE_DIR_NAME = OM_DB_NAME + SNAPSHOT_CANDIDATE_DIR; private File leaderDir; @@ -114,9 +121,15 @@ void init(@TempDir File tempDir) throws Exception { OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE, 1); configuration.setBoolean( OMConfigKeys.OZONE_OM_SNAPSHOT_ROCKSDB_METRICS_ENABLED, false); + // Allow 2 fs snapshots + configuration.setInt( + OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT, 2); OmTestManagers omTestManagers = new OmTestManagers(configuration); om = omTestManagers.getOzoneManager(); + omMetadataManager = (OmMetadataManagerImpl) om.getMetadataManager(); + omSnapshotManager = om.getOmSnapshotManager(); + snapshotChainManager = omMetadataManager.getSnapshotChainManager(); } @AfterAll @@ -124,6 +137,21 @@ void stop() { om.stop(); } + @AfterEach + void cleanup() throws IOException { + Table snapshotInfoTable = omMetadataManager.getSnapshotInfoTable(); + + Iterator iter = snapshotChainManager.iterator(true); + while (iter.hasNext()) { + UUID snapshotId = iter.next(); + String snapshotInfoKey = snapshotChainManager.getTableKey(snapshotId); + SnapshotInfo snapshotInfo = snapshotInfoTable.get(snapshotInfoKey); + snapshotChainManager.deleteSnapshot(snapshotInfo); + snapshotInfoTable.delete(snapshotInfoKey); + } + omSnapshotManager.invalidateCache(); + } + @Test public void testSnapshotFeatureFlagSafetyCheck() throws IOException { // Verify that the snapshot feature config safety check method @@ -150,11 +178,11 @@ public void testCloseOnEviction() throws IOException, final TypedTable bucketTable = mock(TypedTable.class); final TypedTable snapshotInfoTable = mock(TypedTable.class); HddsWhiteboxTestUtils.setInternalState( - om.getMetadataManager(), VOLUME_TABLE, volumeTable); + omMetadataManager, VOLUME_TABLE, volumeTable); HddsWhiteboxTestUtils.setInternalState( - om.getMetadataManager(), BUCKET_TABLE, bucketTable); + omMetadataManager, BUCKET_TABLE, bucketTable); HddsWhiteboxTestUtils.setInternalState( - om.getMetadataManager(), SNAPSHOT_INFO_TABLE, snapshotInfoTable); + omMetadataManager, SNAPSHOT_INFO_TABLE, snapshotInfoTable); final String volumeName = UUID.randomUUID().toString(); final String dbVolumeKey = om.getMetadataManager().getVolumeKey(volumeName); @@ -184,8 +212,8 @@ public void testCloseOnEviction() throws IOException, when(snapshotInfoTable.get(first.getTableKey())).thenReturn(first); when(snapshotInfoTable.get(second.getTableKey())).thenReturn(second); - ((OmMetadataManagerImpl) om.getMetadataManager()).getSnapshotChainManager().addSnapshot(first); - ((OmMetadataManagerImpl) om.getMetadataManager()).getSnapshotChainManager().addSnapshot(second); + snapshotChainManager.addSnapshot(first); + snapshotChainManager.addSnapshot(second); RDBBatchOperation rdbBatchOperation = new RDBBatchOperation(); // create the first snapshot checkpoint OmSnapshotManager.createOmSnapshotCheckpoint(om.getMetadataManager(), @@ -193,7 +221,6 @@ public void testCloseOnEviction() throws IOException, om.getMetadataManager().getStore().commitBatchOperation(rdbBatchOperation); // retrieve it and setup store mock - OmSnapshotManager omSnapshotManager = om.getOmSnapshotManager(); OmSnapshot firstSnapshot = omSnapshotManager .getActiveSnapshot(first.getVolumeName(), first.getBucketName(), first.getName()) .get(); @@ -229,6 +256,37 @@ public void testCloseOnEviction() throws IOException, }, 100, 30_000); } + @Test + public void testValidateSnapshotLimit() throws IOException { + TypedTable snapshotInfoTable = mock(TypedTable.class); + HddsWhiteboxTestUtils.setInternalState( + omMetadataManager, SNAPSHOT_INFO_TABLE, snapshotInfoTable); + + SnapshotInfo first = createSnapshotInfo("vol1", "buck1"); + SnapshotInfo second = createSnapshotInfo("vol1", "buck1"); + + first.setGlobalPreviousSnapshotId(null); + first.setPathPreviousSnapshotId(null); + second.setGlobalPreviousSnapshotId(first.getSnapshotId()); + second.setPathPreviousSnapshotId(first.getSnapshotId()); + + when(snapshotInfoTable.get(first.getTableKey())).thenReturn(first); + when(snapshotInfoTable.get(second.getTableKey())).thenReturn(second); + + snapshotChainManager.addSnapshot(first); + assertDoesNotThrow(() -> omSnapshotManager.snapshotLimitCheck()); + omSnapshotManager.decrementInFlightSnapshotCount(); + + snapshotChainManager.addSnapshot(second); + + OMException exception = assertThrows(OMException.class, () -> omSnapshotManager.snapshotLimitCheck()); + assertEquals(OMException.ResultCodes.TOO_MANY_SNAPSHOTS, exception.getResult()); + + snapshotChainManager.deleteSnapshot(second); + + assertDoesNotThrow(() -> omSnapshotManager.snapshotLimitCheck()); + } + @BeforeEach void setupData(@TempDir File testDir) throws IOException { // Set up the leader with the following files: @@ -332,16 +390,12 @@ public void testHardLinkCreation() throws IOException { @Test public void testGetSnapshotInfo() throws IOException { SnapshotInfo s1 = createSnapshotInfo("vol", "buck"); - UUID latestGlobalSnapId = - ((OmMetadataManagerImpl) om.getMetadataManager()).getSnapshotChainManager() - .getLatestGlobalSnapshotId(); + UUID latestGlobalSnapId = snapshotChainManager.getLatestGlobalSnapshotId(); UUID latestPathSnapId = - ((OmMetadataManagerImpl) om.getMetadataManager()).getSnapshotChainManager() - .getLatestPathSnapshotId(String.join("/", "vol", "buck")); + snapshotChainManager.getLatestPathSnapshotId(String.join("/", "vol", "buck")); s1.setPathPreviousSnapshotId(latestPathSnapId); s1.setGlobalPreviousSnapshotId(latestGlobalSnapId); - ((OmMetadataManagerImpl) om.getMetadataManager()).getSnapshotChainManager() - .addSnapshot(s1); + snapshotChainManager.addSnapshot(s1); OMException ome = assertThrows(OMException.class, () -> om.getOmSnapshotManager().getSnapshot(s1.getSnapshotId())); assertEquals(OMException.ResultCodes.FILE_NOT_FOUND, ome.getResult()); @@ -350,6 +404,12 @@ public void testGetSnapshotInfo() throws IOException { ome = assertThrows(OMException.class, () -> om.getOmSnapshotManager().getSnapshot(s2.getSnapshotId())); assertEquals(OMException.ResultCodes.FILE_NOT_FOUND, ome.getResult()); + + // add to make cleanup work + TypedTable snapshotInfoTable = mock(TypedTable.class); + HddsWhiteboxTestUtils.setInternalState( + omMetadataManager, SNAPSHOT_INFO_TABLE, snapshotInfoTable); + when(snapshotInfoTable.get(s1.getTableKey())).thenReturn(s1); } /* diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index 7842bc1cab25..a45991816b1c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om.request.snapshot; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_FS_SNAPSHOT_MAX_LIMIT; import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.getFromProtobuf; import static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.getTableKey; import static org.apache.hadoop.ozone.om.request.OMRequestTestUtils.createSnapshotRequest; @@ -37,6 +38,7 @@ import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -64,11 +66,17 @@ public class TestOMSnapshotCreateRequest extends TestSnapshotRequestAndResponse { private String snapshotName1; private String snapshotName2; + private String snapshotName3; + private String snapshotName4; + private String snapshotName5; @BeforeEach public void setup() throws Exception { snapshotName1 = UUID.randomUUID().toString(); snapshotName2 = UUID.randomUUID().toString(); + snapshotName3 = UUID.randomUUID().toString(); + snapshotName4 = UUID.randomUUID().toString(); + snapshotName5 = UUID.randomUUID().toString(); } @ValueSource(strings = { @@ -264,6 +272,52 @@ public void testEntryExists() throws Exception { assertEquals(2, getOmMetrics().getNumSnapshotCreates()); } + @Test + public void testSnapshotLimit() throws Exception { + when(getOzoneManager().isAdmin(any())).thenReturn(true); + getOzoneManager().getOmSnapshotManager().close(); + getOzoneManager().getConfiguration().setInt(OZONE_OM_FS_SNAPSHOT_MAX_LIMIT, 3); + OmSnapshotManager omSnapshotManager = new OmSnapshotManager(getOzoneManager()); + when(getOzoneManager().getOmSnapshotManager()).thenReturn(omSnapshotManager); + + // Test Case 1: No snapshots in chain, no in-flight + String key1 = getTableKey(getVolumeName(), getBucketName(), snapshotName1); + OMRequest omRequest = createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName1); + OMSnapshotCreateRequest omSnapshotCreateRequest = doPreExecute(omRequest); + assertNull(getOmMetadataManager().getSnapshotInfoTable().get(key1)); + omSnapshotCreateRequest.validateAndUpdateCache(getOzoneManager(), 1); + assertNotNull(getOmMetadataManager().getSnapshotInfoTable().get(key1)); + + // Test Case 2: One snapshot in chain, no in-flight + String key2 = getTableKey(getVolumeName(), getBucketName(), snapshotName2); + OMRequest snapshotRequest2 = createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName2); + OMSnapshotCreateRequest omSnapshotCreateRequest2 = doPreExecute(snapshotRequest2); + omSnapshotCreateRequest2.validateAndUpdateCache(getOzoneManager(), 2); + assertNotNull(getOmMetadataManager().getSnapshotInfoTable().get(key2)); + + // Test Case 3: Two snapshots in chain, one in-flight + // First create an in-flight snapshot + String key3 = getTableKey(getVolumeName(), getBucketName(), snapshotName3); + OMRequest snapshotRequest3 = createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName3); + OMSnapshotCreateRequest omSnapshotCreateRequest3 = doPreExecute(snapshotRequest3); + // Don't call validateAndUpdateCache to keep it in-flight + + // Try to create another snapshot - should fail as total would be 4 (2 in chain + 1 in-flight + 1 new) + OMRequest snapshotRequest4 = createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName4); + OMException omException = assertThrows(OMException.class, () -> doPreExecute(snapshotRequest4)); + assertEquals(OMException.ResultCodes.TOO_MANY_SNAPSHOTS, omException.getResult()); + + // Complete the in-flight snapshot + omSnapshotCreateRequest3.validateAndUpdateCache(getOzoneManager(), 3); + assertNotNull(getOmMetadataManager().getSnapshotInfoTable().get(key3)); + + // Test Case 4: Three snapshots in chain, no in-flight + // Try to create another snapshot - should fail as we've reached the limit + OMRequest snapshotRequest5 = createSnapshotRequest(getVolumeName(), getBucketName(), snapshotName5); + omException = assertThrows(OMException.class, () -> doPreExecute(snapshotRequest5)); + assertEquals(OMException.ResultCodes.TOO_MANY_SNAPSHOTS, omException.getResult()); + } + private void renameKey(String fromKey, String toKey, long offset) throws IOException { OmKeyInfo toKeyInfo = addKey(toKey, offset + 1L);