diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 132fdb0a4c9b..0169e9d4e51a 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3200,6 +3200,16 @@ + + ozone.filesystem.snapshot.enabled + true + OZONE, OM + + Enables Ozone filesystem snapshot feature if set to true on the OM side. + Disables it otherwise. + + + ozone.snapshot.deleting.service.timeout 300s diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 2adf5ec20e7e..01c87be106b1 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -45,6 +45,10 @@ public final class OMConfigKeys { private OMConfigKeys() { } + public static final String OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY = + "ozone.filesystem.snapshot.enabled"; + public static final boolean OZONE_FILESYSTEM_SNAPSHOT_ENABLED_DEFAULT = true; + // Location where the OM stores its DB files. In the future we may support // multiple entries for performance (sharding).. public static final String OZONE_OM_DB_DIRS = "ozone.om.db.dirs"; diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config index 4642680394d6..1f4029d490e9 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config @@ -93,3 +93,6 @@ OZONE_CONF_DIR=/etc/hadoop OZONE_LOG_DIR=/var/log/hadoop no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1 + +# Explicitly enable filesystem snapshot feature for this Docker compose cluster +OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config index adedb04ca4f6..2809bcbdfd55 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -53,3 +53,6 @@ OZONE_CONF_DIR=/etc/hadoop OZONE_LOG_DIR=/var/log/hadoop no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1 + +# Explicitly enable filesystem snapshot feature for this Docker compose cluster +OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config index 45efd7b52021..9be6e14a2973 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config @@ -159,3 +159,6 @@ OZONE_CONF_DIR=/etc/hadoop OZONE_LOG_DIR=/var/log/hadoop no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1 + +# Explicitly enable filesystem snapshot feature for this Docker compose cluster +OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config index a49a00a08d7e..f279e75ca4d9 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config @@ -149,3 +149,6 @@ OZONE_CONF_DIR=/etc/hadoop OZONE_LOG_DIR=/var/log/hadoop no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1 + +# Explicitly enable filesystem snapshot feature for this Docker compose cluster +OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config index da08c5c3eb7e..d58b098f8527 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config @@ -182,3 +182,6 @@ OZONE-SITE.XML_ozone.om.multitenancy.ranger.sync.timeout=10s # change or let all OMs write to AccessController if this dev flag is set. # OZONE-SITE.XML_ozone.om.tenant.dev.skip.ranger=true + +# Explicitly enable filesystem snapshot feature for this Docker compose cluster +OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config index e241b69be5a5..119e33e8e9e3 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config @@ -61,3 +61,7 @@ OZONE-SITE.XML_ozone.recon.address=recon:9891 no_proxy=om1,om2,om3,scm1,scm2,scm3,s3g,kdc,localhost,127.0.0.1 OM_SERVICE_ID=omservice + +# Explicitly enable filesystem snapshot feature for this Docker compose cluster +# Does not take effect on Ozone versions < 1.4.0 +OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config index 1a7419c52bc2..a7e9d96e686b 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config @@ -40,3 +40,7 @@ OZONE_CONF_DIR=/etc/hadoop OZONE_LOG_DIR=/var/log/hadoop no_proxy=om,scm,s3g,kdc,localhost,127.0.0.1 + +# Explicitly enable filesystem snapshot feature for this Docker compose cluster +# Does not take effect on Ozone versions < 1.4.0 +OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config index e2540baaa49b..44552cbd1663 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/om-ha/docker-config @@ -56,3 +56,7 @@ OZONE-SITE.XML_ozone.recon.address=recon:9891 no_proxy=om1,om2,om3,scm,s3g,kdc,localhost,127.0.0.1 OM_SERVICE_ID=omservice + +# Explicitly enable filesystem snapshot feature for this Docker compose cluster +# Does not take effect on Ozone versions < 1.4.0 +OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java index b6a4bb1fa1c0..20a728719e5e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java @@ -29,6 +29,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.util.ToolRunner; @@ -76,6 +77,8 @@ public class TestOzoneFsSnapshot { @BeforeAll public static void initClass() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); + // Enable filesystem snapshot feature for the test regardless of the default + conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); // Start the cluster cluster = MiniOzoneCluster.newOMHABuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java index 041383264f87..a4e808be41e6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java @@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmSnapshotManager; @@ -98,6 +99,8 @@ public static void init() throws Exception { raftClientConfig.setRpcRequestTimeout(Duration.ofSeconds(3)); raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(3)); conf.setFromObject(raftClientConfig); + // Enable filesystem snapshot feature for the test regardless of the default + conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); // Set DB CF write buffer to a much lower value so that flush and compaction // happens much more frequently without having to create a lot of keys. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java index bdfca04e6126..acd9237c2c89 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java @@ -163,6 +163,8 @@ private void init() throws Exception { conf.setBoolean(OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF, forceFullSnapshotDiff); conf.setEnum(HDDS_DB_PROFILE, DBProfile.TEST); + // Enable filesystem snapshot feature for the test regardless of the default + conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); cluster = MiniOzoneCluster.newOMHABuilder(conf) .setClusterId(clusterId) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotDisabled.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotDisabled.java new file mode 100644 index 000000000000..ef8012fc6af9 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotDisabled.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.om; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.hdds.utils.db.DBProfile; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.ozone.test.LambdaTestUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.UUID; + +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE; + +/** + * Integration test to verify Ozone snapshot RPCs throw exception when called. + */ +public class TestOmSnapshotDisabled { + + private static MiniOzoneCluster cluster = null; + private static OzoneClient client; + private static ObjectStore store; + + @BeforeAll + @Timeout(60) + public static void init() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + String clusterId = UUID.randomUUID().toString(); + String scmId = UUID.randomUUID().toString(); + conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, + BucketLayout.LEGACY.name()); + conf.setEnum(HDDS_DB_PROFILE, DBProfile.TEST); + // Disable filesystem snapshot feature for this test + conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, false); + + cluster = MiniOzoneCluster.newOMHABuilder(conf) + .setClusterId(clusterId) + .setScmId(scmId) + .setOMServiceId("om-service-test1") + .setNumOfOzoneManagers(3) + .build(); + cluster.waitForClusterToBeReady(); + client = cluster.newClient(); + + OzoneManager leaderOzoneManager = + ((MiniOzoneHAClusterImpl) cluster).getOMLeader(); + OzoneConfiguration leaderConfig = leaderOzoneManager.getConfiguration(); + cluster.setConf(leaderConfig); + store = client.getObjectStore(); + } + + @AfterAll + public static void tearDown() throws Exception { + IOUtils.closeQuietly(client); + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testExceptionThrown() throws Exception { + String volumeName = "vol-" + RandomStringUtils.randomNumeric(5); + String bucketName = "buck-" + RandomStringUtils.randomNumeric(5); + String snapshotName = "snap-" + RandomStringUtils.randomNumeric(5); + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + + // create snapshot should throw + LambdaTestUtils.intercept(OMException.class, "FEATURE_NOT_ENABLED", + () -> store.createSnapshot(volumeName, bucketName, snapshotName)); + // delete snapshot should throw + LambdaTestUtils.intercept(OMException.class, "FEATURE_NOT_ENABLED", + () -> store.deleteSnapshot(volumeName, bucketName, snapshotName)); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java index b5f62a67a0e9..81a555a07c84 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java @@ -108,8 +108,6 @@ public class TestOmSnapshotFileSystem { private static final Logger LOG = LoggerFactory.getLogger(TestOmSnapshot.class); - - @Rule public Timeout timeout = new Timeout(120, TimeUnit.SECONDS); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java index d5faf7a8ccd0..92cf5ec2e109 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java @@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.om.KeyManagerImpl; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMStorage; import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; @@ -91,6 +92,9 @@ private static Stream bucketTypesCombinations() { @BeforeEach public void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); + // Enable filesystem snapshot feature for the test regardless of the default + conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); + String clusterId = UUID.randomUUID().toString(); String scmId = UUID.randomUUID().toString(); String serviceID = OM_SERVICE_ID + RandomStringUtils.randomNumeric(5); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 0ee4b59396aa..4d6a91e358f2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -255,7 +255,9 @@ public void start(OzoneConfiguration configuration) { openKeyCleanupService.start(); } - if (snapshotSstFilteringService == null) { + if (snapshotSstFilteringService == null && + ozoneManager.isFilesystemSnapshotEnabled()) { + long serviceInterval = configuration.getTimeDuration( OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL_DEFAULT, @@ -272,7 +274,9 @@ public void start(OzoneConfiguration configuration) { } } - if (snapshotDeletingService == null) { + if (snapshotDeletingService == null && + ozoneManager.isFilesystemSnapshotEnabled()) { + long snapshotServiceInterval = configuration.getTimeDuration( OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL_DEFAULT, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index fcd3b2cde286..eadf8f7321e8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1550,7 +1550,7 @@ public List getPendingDeletionKeys(final int keyCount, * Get the latest OmSnapshot for a snapshot path. */ public OmSnapshot getLatestSnapshot(String volumeName, String bucketName, - OmSnapshotManager snapshotManager) + OmSnapshotManager snapshotManager) throws IOException { String latestPathSnapshot = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 9f1a2747f29e..0425f8ef79b4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -158,7 +158,18 @@ public final class OmSnapshotManager implements AutoCloseable { // Soft limit of the snapshot cache size. private final int softCacheSize; + /** + * TODO: [SNAPSHOT] HDDS-8529: Refactor the constructor in a way that when + * ozoneManager.isFilesystemSnapshotEnabled() returns false, + * no snapshot-related background job or initialization would run, + * except for applying previously committed Ratis transactions in e.g.: + * 1. {@link OMKeyPurgeRequest#validateAndUpdateCache} + * 2. {@link OMDirectoriesPurgeRequestWithFSO#validateAndUpdateCache} + */ public OmSnapshotManager(OzoneManager ozoneManager) { + LOG.info("Ozone filesystem snapshot feature is {}.", + ozoneManager.isFilesystemSnapshotEnabled() ? "enabled" : "disabled"); + this.options = new ManagedDBOptions(); this.options.setCreateIfMissing(true); this.columnFamilyOptions = new ManagedColumnFamilyOptions(); @@ -259,17 +270,21 @@ public OmSnapshotManager(OzoneManager ozoneManager) { OZONE_OM_SNAPSHOT_DIFF_CLEANUP_SERVICE_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS); - this.snapshotDiffCleanupService = new SnapshotDiffCleanupService( - diffCleanupServiceInterval, - diffCleanupServiceTimeout, - ozoneManager, - snapshotDiffDb, - snapDiffJobCf, - snapDiffPurgedJobCf, - snapDiffReportCf, - codecRegistry - ); - this.snapshotDiffCleanupService.start(); + if (ozoneManager.isFilesystemSnapshotEnabled()) { + this.snapshotDiffCleanupService = new SnapshotDiffCleanupService( + diffCleanupServiceInterval, + diffCleanupServiceTimeout, + ozoneManager, + snapshotDiffDb, + snapDiffJobCf, + snapDiffPurgedJobCf, + snapDiffReportCf, + codecRegistry + ); + this.snapshotDiffCleanupService.start(); + } else { + this.snapshotDiffCleanupService = null; + } } private CacheLoader createCacheLoader() { @@ -479,7 +494,7 @@ private static void deleteKeysInSnapshotScopeFromDTableInternal( public IOmMetadataReader checkForSnapshot(String volumeName, String bucketName, String keyname) throws IOException { - if (keyname == null) { + if (keyname == null || !ozoneManager.isFilesystemSnapshotEnabled()) { return ozoneManager.getOmMetadataReader(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 1bdfb3140acf..c8d08b6b0363 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -434,6 +434,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private final OzoneLockProvider ozoneLockProvider; private OMPerformanceMetrics perfMetrics; + private boolean fsSnapshotEnabled; + /** * OM Startup mode. */ @@ -544,6 +546,10 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); + fsSnapshotEnabled = configuration.getBoolean( + OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, + OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_DEFAULT); + String defaultBucketLayoutString = configuration.getTrimmed(OZONE_DEFAULT_BUCKET_LAYOUT, OZONE_DEFAULT_BUCKET_LAYOUT_DEFAULT); @@ -783,6 +789,8 @@ private void instantiateServices(boolean withNewSnapshot) throws IOException { perfMetrics); omMetadataReader = new OmMetadataReader(keyManager, prefixManager, this, LOG, AUDIT, metrics); + + // TODO: [SNAPSHOT] Revisit this in HDDS-8529. omSnapshotManager = new OmSnapshotManager(this); // Snapshot metrics @@ -3994,6 +4002,13 @@ public boolean isRatisEnabled() { return isRatisEnabled; } + /** + * @return true if Ozone filesystem snapshot is enabled, false otherwise. + */ + public boolean isFilesystemSnapshotEnabled() { + return fsSnapshotEnabled; + } + /** * Get DB updates since a specific sequence number. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java index 90ea43a53ee9..6926de1fedaa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java @@ -26,6 +26,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmSnapshot; +import org.apache.hadoop.ozone.om.OmSnapshotManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -58,6 +59,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { OzoneManagerProtocolProtos.PurgeDirectoriesRequest purgeDirsRequest = getOmRequest().getPurgeDirectoriesRequest(); + OmSnapshotManager omSnapshotManager = ozoneManager.getOmSnapshotManager(); String fromSnapshot = purgeDirsRequest.hasSnapshotTableKey() ? purgeDirsRequest.getSnapshotTableKey() : null; @@ -73,7 +75,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, SnapshotInfo snapshotInfo = ozoneManager.getMetadataManager().getSnapshotInfoTable() .get(fromSnapshot); - omFromSnapshot = (OmSnapshot) ozoneManager.getOmSnapshotManager() + // TODO: [SNAPSHOT] Revisit in HDDS-8529. + omFromSnapshot = (OmSnapshot) omSnapshotManager .checkForSnapshot(snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), getSnapshotPrefix(snapshotInfo.getName())); @@ -127,7 +130,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, } } } catch (IOException ex) { - // Case of IOException for fromProtobuf will not hanppen + // Case of IOException for fromProtobuf will not happen // as this is created and send within OM // only case of upgrade where compatibility is broken can have throw new IllegalStateException(ex); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java index f429afb95374..1f49a731d587 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java @@ -80,6 +80,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, SnapshotInfo snapshotInfo = ozoneManager.getMetadataManager().getSnapshotInfoTable() .get(fromSnapshot); + // TODO: [SNAPSHOT] Revisit in HDDS-8529. omFromSnapshot = (OmSnapshot) omSnapshotManager .checkForSnapshot(snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index fb082bbe3e96..0bed12118d04 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotCreateResponse; +import org.apache.hadoop.ozone.om.snapshot.RequireSnapshotFeatureState; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateSnapshotRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CreateSnapshotResponse; @@ -87,6 +88,7 @@ public OMSnapshotCreateRequest(OMRequest omRequest) { } @Override + @RequireSnapshotFeatureState(true) public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { final OMRequest omRequest = super.preExecute(ozoneManager); // Verify name diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java index eaaf63cf645b..33ec0876179f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java @@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotDeleteResponse; +import org.apache.hadoop.ozone.om.snapshot.RequireSnapshotFeatureState; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteSnapshotRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteSnapshotResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -63,6 +64,7 @@ public OMSnapshotDeleteRequest(OMRequest omRequest) { } @Override + @RequireSnapshotFeatureState(true) public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { final OMRequest omRequest = super.preExecute(ozoneManager); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java index 1b0488a326c7..10a5b19e1c5b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotMoveDeletedKeysRequest.java @@ -45,6 +45,7 @@ /** * Handles OMSnapshotMoveDeletedKeys Request. + * This is an OM internal request. Does not need @RequireSnapshotFeatureState. */ public class OMSnapshotMoveDeletedKeysRequest extends OMClientRequest { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java index 30409c047342..20ce2cc240e6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java @@ -33,6 +33,7 @@ /** * Handles OMSnapshotPurge Request. + * This is an OM internal request. Does not need @RequireSnapshotFeatureState. */ public class OMSnapshotPurgeRequest extends OMClientRequest { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RequireSnapshotFeatureState.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RequireSnapshotFeatureState.java new file mode 100644 index 000000000000..19d3f2805489 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RequireSnapshotFeatureState.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation used to check that the snapshot feature in desired state. + * Can be generalized into checking arbitrary config state. + */ +@Target({ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface RequireSnapshotFeatureState { + boolean value(); +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RequireSnapshotFeatureStateAspect.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RequireSnapshotFeatureStateAspect.java new file mode 100644 index 000000000000..5f14e841c600 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RequireSnapshotFeatureStateAspect.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.apache.commons.lang3.NotImplementedException; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.protocolPB.OzoneManagerRequestHandler; +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; +import org.aspectj.lang.reflect.MethodSignature; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.lang.reflect.Method; + +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FEATURE_NOT_ENABLED; + +/** + * 'Aspect' for checking whether snapshot feature is enabled. + * All methods annotated with the specific annotation will have pre-processing + * done here to check layout version compatibility. + * Note: Append class to + * hadoop-ozone/ozone-manager/src/main/resources/META-INF/aop.xml + * if the annotation doesn't seem to take affect on other classes' methods. + */ +@Aspect +public class RequireSnapshotFeatureStateAspect { + + private static final Logger LOG = LoggerFactory + .getLogger(RequireSnapshotFeatureStateAspect.class); + + @Before("@annotation(RequireSnapshotFeatureState) && execution(* *(..))") + public void checkFeatureState(JoinPoint joinPoint) throws IOException { + boolean desiredFeatureState = ((MethodSignature) joinPoint.getSignature()) + .getMethod().getAnnotation(RequireSnapshotFeatureState.class) + .value(); + boolean isFeatureEnabled; + final Object[] args = joinPoint.getArgs(); + if (LOG.isDebugEnabled()) { + LOG.debug("joinPoint.getTarget() = {}", joinPoint.getTarget()); + } + if (joinPoint.getTarget() instanceof OzoneManagerRequestHandler) { + OzoneManager ozoneManager = ((OzoneManagerRequestHandler) + joinPoint.getTarget()).getOzoneManager(); + isFeatureEnabled = ozoneManager.isFilesystemSnapshotEnabled(); + } else if (joinPoint.getTarget() instanceof OMClientRequest && + joinPoint.toShortString().endsWith(".preExecute(..))")) { + // Get OzoneManager instance from preExecute first argument + OzoneManager ozoneManager = (OzoneManager) args[0]; + isFeatureEnabled = ozoneManager.isFilesystemSnapshotEnabled(); + } else { + // This case is used in UT, where: + // joinPoint.getTarget() instanceof SnapshotFeatureEnabledUtil + try { + Method method = joinPoint.getTarget().getClass() + .getMethod("isFilesystemSnapshotEnabled"); + isFeatureEnabled = (boolean) method.invoke(joinPoint.getTarget()); + } catch (Exception ex) { + // Exception being thrown here means this is not called from the UT. + // Thus this is an unhandled usage. + // Add more case handling logic as needed. + throw new NotImplementedException( + "Unhandled use case. Please implement."); + } + } + checkIsAllowed(joinPoint.getSignature().toShortString(), + isFeatureEnabled, desiredFeatureState); + } + + private void checkIsAllowed(String operationName, + boolean isFeatureEnabled, + boolean desiredFeatureState) throws OMException { + + if (desiredFeatureState) { + if (!isFeatureEnabled) { + throw new OMException(String.format( + "Operation %s cannot be invoked because %s.", + operationName, + "Ozone snapshot feature is disabled"), + FEATURE_NOT_ENABLED); + } else { + // Pass the check: feature is enabled, desired feature state is enabled + return; + } + } else { + // Add implementation if needed later + // desiredFeatureState=true is the only case being used right now + throw new NotImplementedException("Check not implemented for case: " + + "isFeatureEnabled=" + isFeatureEnabled + + ", desiredFeatureState=" + desiredFeatureState); + } + } + + /** + * Note: Without this, it occasionally throws NoSuchMethodError when running + * the test. + */ + public static RequireSnapshotFeatureStateAspect aspectOf() { + return new RequireSnapshotFeatureStateAspect(); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/main/resources/META-INF/aop.xml b/hadoop-ozone/ozone-manager/src/main/resources/META-INF/aop.xml index f92d820ad27a..a96b5e8b71bc 100644 --- a/hadoop-ozone/ozone-manager/src/main/resources/META-INF/aop.xml +++ b/hadoop-ozone/ozone-manager/src/main/resources/META-INF/aop.xml @@ -15,8 +15,9 @@ + - + diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index 541d0a5c969e..6287cc00cbdb 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -70,6 +70,9 @@ public void init() throws Exception { testDir = GenericTestUtils.getRandomizedTestDir(); configuration.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.toString()); + // Enable filesystem snapshot feature for the test regardless of the default + configuration.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, + true); // Only allow one entry in cache so each new one causes an eviction configuration.setInt( diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index 462b6c731d73..0a9eda229128 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -137,6 +137,7 @@ public void setup() throws Exception { when(lvm.getMetadataLayoutVersion()).thenReturn(0); when(ozoneManager.getVersionManager()).thenReturn(lvm); when(ozoneManager.isRatisEnabled()).thenReturn(true); + when(ozoneManager.isFilesystemSnapshotEnabled()).thenReturn(true); auditLogger = Mockito.mock(AuditLogger.class); when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); when(ozoneManager.isAdmin(any(UserGroupInformation.class))) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java index a67814156db3..38df5c8006c0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java @@ -97,6 +97,7 @@ public void setup() throws Exception { when(ozoneManager.getMetrics()).thenReturn(omMetrics); when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); when(ozoneManager.isRatisEnabled()).thenReturn(true); + when(ozoneManager.isFilesystemSnapshotEnabled()).thenReturn(true); when(ozoneManager.isAdmin(any())).thenReturn(false); when(ozoneManager.isOwner(any(), any())).thenReturn(false); when(ozoneManager.getBucketOwner(any(), any(), diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java index 0012e9494f0a..ae28a9cd3152 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java @@ -89,6 +89,7 @@ public void setup() throws Exception { when(ozoneManager.getMetrics()).thenReturn(omMetrics); when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); when(ozoneManager.isRatisEnabled()).thenReturn(true); + when(ozoneManager.isFilesystemSnapshotEnabled()).thenReturn(true); when(ozoneManager.isAdmin(any())).thenReturn(false); when(ozoneManager.isOwner(any(), any())).thenReturn(false); when(ozoneManager.getBucketOwner(any(), any(), diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java index 02814c788e17..648e62ad13fe 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java @@ -110,6 +110,7 @@ public void setup() throws Exception { when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration); when(ozoneManager.isAdmin(any(UserGroupInformation.class))) .thenReturn(true); + when(ozoneManager.isFilesystemSnapshotEnabled()).thenReturn(true); OmMetadataReader omMetadataReader = Mockito.mock(OmMetadataReader.class); when(ozoneManager.getOmMetadataReader()).thenReturn(omMetadataReader); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java index e1da3128c66e..39cbdd27667a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; import org.apache.hadoop.ozone.om.KeyManager; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmSnapshot; import org.apache.hadoop.ozone.om.OmTestManagers; @@ -94,6 +95,8 @@ public void createConfAndInitValues() throws Exception { conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_TIMEOUT, 100000, TimeUnit.MILLISECONDS); conf.setQuietMode(false); + // Enable filesystem snapshot feature for the test regardless of the default + conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); omTestManagers = new OmTestManagers(conf); keyManager = omTestManagers.getKeyManager(); omMetadataManager = omTestManagers.getMetadataManager(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java index eb7b77f328eb..60eae2c1c8e8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java @@ -54,6 +54,8 @@ import java.util.concurrent.TimeUnit; import static org.apache.hadoop.hdds.utils.db.DBStoreBuilder.DEFAULT_COLUMN_FAMILY_NAME; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_JOB_REPORT_PERSISTENT_TIME; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_JOB_REPORT_PERSISTENT_TIME_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_MAX_JOBS_PURGE_PER_TASK; @@ -151,6 +153,9 @@ public void init() throws RocksDBException, IOException { TimeUnit.MILLISECONDS) ).thenReturn(TimeUnit.DAYS.toMillis(7)); + when(config.getBoolean(OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, + OZONE_FILESYSTEM_SNAPSHOT_ENABLED_DEFAULT)).thenReturn(true); + when(ozoneManager.getConfiguration()).thenReturn(config); jobTableCfd = new ColumnFamilyDescriptor(jobTableNameBytes, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/SnapshotFeatureEnabledUtil.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/SnapshotFeatureEnabledUtil.java new file mode 100644 index 000000000000..3eea670dcaab --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/SnapshotFeatureEnabledUtil.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +/** + * TestSnapshotFeatureEnabledAspect util class. + */ +public class SnapshotFeatureEnabledUtil { + + /** + * This is an example of an "API" that requires snapshot feature enabled. + */ + @RequireSnapshotFeatureState(true) + public String snapshotMethod() { + return "yay"; + } + + /** + * Method needed for the Aspect to get current feature state. Emulates + * {@link org.apache.hadoop.ozone.om.OzoneManager#isFilesystemSnapshotEnabled} + * Note: Method has to be `public` for reflection invocation to work. + * @return false + */ + public boolean isFilesystemSnapshotEnabled() { + return false; + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestRequireSnapshotFeatureStateAspect.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestRequireSnapshotFeatureStateAspect.java new file mode 100644 index 000000000000..b94977455fd0 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestRequireSnapshotFeatureStateAspect.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.snapshot; + +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.ozone.test.LambdaTestUtils; +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.reflect.MethodSignature; +import org.junit.Test; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Class to test annotation based interceptor that checks whether + * Ozone snapshot feature is enabled. + */ +public class TestRequireSnapshotFeatureStateAspect { + + /** + * Check Aspect implementation with SnapshotFeatureEnabledUtil. + */ + @Test + public void testSnapshotFeatureEnabledAnnotation() throws Exception { + SnapshotFeatureEnabledUtil testObj = new SnapshotFeatureEnabledUtil(); + RequireSnapshotFeatureStateAspect + aspect = new RequireSnapshotFeatureStateAspect(); + + JoinPoint joinPoint = mock(JoinPoint.class); + when(joinPoint.getTarget()).thenReturn(testObj); + + MethodSignature methodSignature = mock(MethodSignature.class); + when(methodSignature.getMethod()).thenReturn( + SnapshotFeatureEnabledUtil.class.getMethod("snapshotMethod")); + when(methodSignature.toShortString()).thenReturn("snapshotMethod"); + when(joinPoint.getSignature()).thenReturn(methodSignature); + + LambdaTestUtils.intercept(OMException.class, + "Operation snapshotMethod cannot be invoked because " + + "Ozone snapshot feature is disabled", + () -> aspect.checkFeatureState(joinPoint)); + } +}