diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index ce65be861dee..ba0e549bb39e 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3961,6 +3961,15 @@ + + ozone.om.snapshot.diff.disable.native.libs + false + OZONE, OM + + Flag to perform snapshot diff without using native libs(can be slow). + + + ozone.om.snapshot.diff.max.page.size 1000 diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestManagedSstFileReader.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestManagedSstFileReader.java index 5db091cafc28..ffed57594a7f 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestManagedSstFileReader.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestManagedSstFileReader.java @@ -33,8 +33,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.rocksdb.RocksDBException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -58,9 +56,6 @@ */ class TestManagedSstFileReader { - private static final Logger LOG = - LoggerFactory.getLogger(TestManagedSstFileReader.class); - // Key prefix containing all characters, to check if all characters can be // written & read from rocksdb through SSTDumptool private static final String KEY_PREFIX = IntStream.range(0, 256).boxed() @@ -158,6 +153,5 @@ public void testGetKeyStreamWithTombstone(int numberOfFiles) } finally { executorService.shutdown(); } - } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java index 27b1964ea0b2..69e4f68eb60b 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java @@ -668,19 +668,23 @@ private List getNextListOfSnapshots(String prevSnapshot) * @param token to get the index to return diff report from. * @param pageSize maximum entries returned to the report. * @param forceFullDiff request to force full diff, skipping DAG optimization + * @param disableNativeDiff request to force diff to perform diffs without + * native lib * @return the difference report between two snapshots * @throws IOException in case of any exception while generating snapshot diff */ + @SuppressWarnings("parameternumber") public SnapshotDiffResponse snapshotDiff(String volumeName, String bucketName, String fromSnapshot, String toSnapshot, String token, int pageSize, - boolean forceFullDiff) + boolean forceFullDiff, + boolean disableNativeDiff) throws IOException { return proxy.snapshotDiff(volumeName, bucketName, fromSnapshot, toSnapshot, - token, pageSize, forceFullDiff); + token, pageSize, forceFullDiff, disableNativeDiff); } /** diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 87c2290ad49c..2c780d93bae1 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -1071,10 +1071,12 @@ List listSnapshot( * @return the difference report between two snapshots * @throws IOException in case of any exception while generating snapshot diff */ + @SuppressWarnings("parameternumber") SnapshotDiffResponse snapshotDiff(String volumeName, String bucketName, String fromSnapshot, String toSnapshot, String token, int pageSize, - boolean forceFullDiff) + boolean forceFullDiff, + boolean disableNativeDiff) throws IOException; /** diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 569901b667f8..f6e9b2a3287f 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -987,14 +987,16 @@ public SnapshotDiffResponse snapshotDiff(String volumeName, String toSnapshot, String token, int pageSize, - boolean forceFullDiff) + boolean forceFullDiff, + boolean disableNativeDiff) throws IOException { Preconditions.checkArgument(StringUtils.isNotBlank(volumeName), "volume can't be null or empty."); Preconditions.checkArgument(StringUtils.isNotBlank(bucketName), "bucket can't be null or empty."); return ozoneManagerClient.snapshotDiff(volumeName, bucketName, - fromSnapshot, toSnapshot, token, pageSize, forceFullDiff); + fromSnapshot, toSnapshot, token, pageSize, forceFullDiff, + disableNativeDiff); } @Override diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index c5c07e285763..2d824de3bbbd 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -489,6 +489,11 @@ private OMConfigKeys() { public static final boolean OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT = false; + public static final String OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS = + "ozone.om.snapshot.diff.disable.native.libs"; + + public static final boolean + OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS_DEFAULT = false; public static final String OZONE_OM_SNAPSHOT_DIFF_DB_DIR = "ozone.om.snapshot.diff.db.dir"; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfoGroup.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfoGroup.java index 991f36666400..931657e8e7f4 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfoGroup.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfoGroup.java @@ -179,7 +179,7 @@ void addAll(long versionToAdd, List locationInfoList) { public String toString() { StringBuilder sb = new StringBuilder(); sb.append("version:").append(version).append(" "); - sb.append("isMultipartKey:").append(isMultipartKey); + sb.append("isMultipartKey:").append(isMultipartKey).append(" "); for (List kliList : locationVersionMap.values()) { for (OmKeyLocationInfo kli: kliList) { sb.append(kli.getLocalID()).append(" || "); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java index 0465c4e1ae93..0d221dc1cd4b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java @@ -48,6 +48,7 @@ public static Codec getCodec() { private String fromSnapshot; private String toSnapshot; private boolean forceFullDiff; + private boolean disableNativeDiff; private long totalDiffEntries; // Reason tells why the job was FAILED. It should be set only if job status @@ -68,6 +69,7 @@ public SnapshotDiffJob(long creationTime, String fromSnapshot, String toSnapshot, boolean forceFullDiff, + boolean disableNativeDiff, long totalDiffEntries) { this.creationTime = creationTime; this.jobId = jobId; @@ -77,6 +79,7 @@ public SnapshotDiffJob(long creationTime, this.fromSnapshot = fromSnapshot; this.toSnapshot = toSnapshot; this.forceFullDiff = forceFullDiff; + this.disableNativeDiff = disableNativeDiff; this.totalDiffEntries = totalDiffEntries; this.reason = StringUtils.EMPTY; } @@ -161,6 +164,14 @@ public void setReason(String reason) { this.reason = reason; } + public boolean isNativeDiffDisabled() { + return disableNativeDiff; + } + + public void disableNativeDiff(boolean disableNativeDiffVal) { + this.disableNativeDiff = disableNativeDiffVal; + } + @Override public String toString() { StringBuilder sb = new StringBuilder("creationTime : ").append(creationTime) @@ -171,6 +182,7 @@ public String toString() { .append(", fromSnapshot: ").append(fromSnapshot) .append(", toSnapshot: ").append(toSnapshot) .append(", forceFullDiff: ").append(forceFullDiff) + .append(", disableNativeDiff: ").append(disableNativeDiff) .append(", totalDiffEntries: ").append(totalDiffEntries); if (StringUtils.isNotEmpty(reason)) { @@ -196,7 +208,8 @@ public boolean equals(Object other) { Objects.equals(this.toSnapshot, otherJob.toSnapshot) && Objects.equals(this.forceFullDiff, otherJob.forceFullDiff) && Objects.equals(this.totalDiffEntries, otherJob.totalDiffEntries) && - Objects.equals(this.reason, otherJob.reason); + Objects.equals(this.disableNativeDiff, otherJob.disableNativeDiff) + && Objects.equals(this.reason, otherJob.reason); } return false; } @@ -204,7 +217,8 @@ public boolean equals(Object other) { @Override public int hashCode() { return Objects.hash(creationTime, jobId, status, volume, bucket, - fromSnapshot, toSnapshot, forceFullDiff, totalDiffEntries, reason); + fromSnapshot, toSnapshot, forceFullDiff, disableNativeDiff, + totalDiffEntries, reason); } public SnapshotDiffJobProto toProtoBuf() { @@ -217,6 +231,7 @@ public SnapshotDiffJobProto toProtoBuf() { .setFromSnapshot(fromSnapshot) .setToSnapshot(toSnapshot) .setForceFullDiff(forceFullDiff) + .setDisableNativeDiff(disableNativeDiff) .setTotalDiffEntries(totalDiffEntries) .build(); } @@ -232,6 +247,7 @@ public static SnapshotDiffJob getFromProtoBuf( diffJobProto.getFromSnapshot(), diffJobProto.getToSnapshot(), diffJobProto.getForceFullDiff(), + diffJobProto.getDisableNativeDiff(), diffJobProto.getTotalDiffEntries()); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index da9e45ebc3fb..962eb9eb5757 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -720,13 +720,15 @@ default List listSnapshot( * @return the difference report between two snapshots * @throws IOException in case of any exception while generating snapshot diff */ + @SuppressWarnings("parameternumber") default SnapshotDiffResponse snapshotDiff(String volumeName, String bucketName, String fromSnapshot, String toSnapshot, String token, int pageSize, - boolean forceFullDiff) + boolean forceFullDiff, + boolean disableNativeDiff) throws IOException { throw new UnsupportedOperationException("OzoneManager does not require " + "this to be implemented"); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 2c0aef893b14..348e364a421f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1232,7 +1232,8 @@ public SnapshotDiffResponse snapshotDiff(String volumeName, String toSnapshot, String token, int pageSize, - boolean forceFullDiff) + boolean forceFullDiff, + boolean disableNativeDiff) throws IOException { final OzoneManagerProtocolProtos.SnapshotDiffRequest.Builder requestBuilder = @@ -1242,7 +1243,8 @@ public SnapshotDiffResponse snapshotDiff(String volumeName, .setFromSnapshot(fromSnapshot) .setToSnapshot(toSnapshot) .setPageSize(pageSize) - .setForceFullDiff(forceFullDiff); + .setForceFullDiff(forceFullDiff) + .setDisableNativeDiff(disableNativeDiff); if (!StringUtils.isBlank(token)) { requestBuilder.setToken(token); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java index dd58791c48d8..4ae27046c997 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java @@ -16,10 +16,19 @@ */ package org.apache.hadoop.ozone.om; +import java.net.URI; +import java.net.URISyntaxException; import java.time.Duration; import java.util.List; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; @@ -30,8 +39,11 @@ import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectUtils; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -46,6 +58,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.service.SnapshotDiffCleanupService; @@ -79,6 +92,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -106,8 +120,8 @@ import static org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_SUCCEEDED; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.CANCELLED; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE; -import static org.awaitility.Awaitility.with; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.IN_PROGRESS; +import static org.awaitility.Awaitility.with; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -136,6 +150,7 @@ public class TestOmSnapshot { private static BucketLayout bucketLayout = BucketLayout.LEGACY; private static boolean enabledFileSystemPaths; private static boolean forceFullSnapshotDiff; + private static boolean disableNativeDiff; private static ObjectStore store; private static OzoneManager ozoneManager; private static RDBStore rdbStore; @@ -157,14 +172,15 @@ public class TestOmSnapshot { @Parameterized.Parameters public static Collection data() { - return Arrays.asList( - new Object[]{OBJECT_STORE, false, false}, - new Object[]{FILE_SYSTEM_OPTIMIZED, false, false}, - new Object[]{BucketLayout.LEGACY, true, true}); + return Arrays.asList(new Object[]{OBJECT_STORE, false, false, false}, + new Object[]{FILE_SYSTEM_OPTIMIZED, false, false, false}, + new Object[]{FILE_SYSTEM_OPTIMIZED, false, false, true}, + new Object[]{BucketLayout.LEGACY, true, true, false}); } public TestOmSnapshot(BucketLayout newBucketLayout, - boolean newEnableFileSystemPaths, boolean forceFullSnapDiff) + boolean newEnableFileSystemPaths, boolean forceFullSnapDiff, + boolean disableNativeDiff) throws Exception { // Checking whether 'newBucketLayout' and // 'newEnableFileSystemPaths' flags represents next parameter @@ -173,19 +189,23 @@ public TestOmSnapshot(BucketLayout newBucketLayout, // Parameterized.Parameters. if (TestOmSnapshot.enabledFileSystemPaths != newEnableFileSystemPaths || TestOmSnapshot.bucketLayout != newBucketLayout || - TestOmSnapshot.forceFullSnapshotDiff != forceFullSnapDiff) { + TestOmSnapshot.forceFullSnapshotDiff != forceFullSnapDiff || + TestOmSnapshot.disableNativeDiff + != disableNativeDiff) { setConfig(newBucketLayout, newEnableFileSystemPaths, - forceFullSnapDiff); + forceFullSnapDiff, disableNativeDiff); tearDown(); init(); } } private static void setConfig(BucketLayout newBucketLayout, - boolean newEnableFileSystemPaths, boolean forceFullSnapDiff) { + boolean newEnableFileSystemPaths, boolean forceFullSnapDiff, + boolean forceNonNativeSnapDiff) { TestOmSnapshot.enabledFileSystemPaths = newEnableFileSystemPaths; TestOmSnapshot.bucketLayout = newBucketLayout; TestOmSnapshot.forceFullSnapshotDiff = forceFullSnapDiff; + TestOmSnapshot.disableNativeDiff = forceNonNativeSnapDiff; } /** @@ -195,6 +215,14 @@ private void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); String clusterId = UUID.randomUUID().toString(); String scmId = UUID.randomUUID().toString(); + conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, + enabledFileSystemPaths); + conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, + bucketLayout.name()); + conf.setBoolean(OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF, + forceFullSnapshotDiff); + conf.setBoolean(OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS, + disableNativeDiff); String omId = UUID.randomUUID().toString(); conf.setBoolean(OZONE_OM_ENABLE_FILESYSTEM_PATHS, enabledFileSystemPaths); conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, bucketLayout.name()); @@ -227,10 +255,10 @@ private void init() throws Exception { KeyManagerImpl keyManager = (KeyManagerImpl) HddsWhiteboxTestUtils .getInternalState(ozoneManager, "keyManager"); - + counter = new AtomicInteger(0); // stop the deletion services so that keys can still be read keyManager.stop(); - preFinalizationChecks(); +// preFinalizationChecks(); finalizeOMUpgrade(); counter = new AtomicInteger(); } @@ -256,7 +284,7 @@ private static void preFinalizationChecks() throws Exception { store.snapshotDiff(volumeName, bucketName, UUID.randomUUID().toString(), UUID.randomUUID().toString(), - "", 1000, false)); + "", 1000, false, disableNativeDiff)); expectFailurePreFinalization(() -> store.deleteSnapshot(volumeName, bucketName, UUID.randomUUID().toString())); @@ -306,8 +334,7 @@ public static void tearDown() throws Exception { @Test // based on TestOzoneRpcClientAbstract:testListKey - public void testListKey() - throws IOException, InterruptedException, TimeoutException { + public void testListKey() throws Exception { String volumeA = "vol-a-" + counter.incrementAndGet(); String volumeB = "vol-b-" + counter.incrementAndGet(); String bucketA = "buc-a-" + counter.incrementAndGet(); @@ -472,8 +499,7 @@ public void checkKey() throws Exception { } @Test - public void testListDeleteKey() - throws IOException, InterruptedException, TimeoutException { + public void testListDeleteKey() throws Exception { String volume = "vol-" + counter.incrementAndGet(); String bucket = "buc-" + counter.incrementAndGet(); store.createVolume(volume); @@ -498,8 +524,7 @@ public void testListDeleteKey() } @Test - public void testListAddNewKey() - throws IOException, InterruptedException, TimeoutException { + public void testListAddNewKey() throws Exception { String volume = "vol-" + counter.incrementAndGet(); String bucket = "buc-" + counter.incrementAndGet(); store.createVolume(volume); @@ -571,6 +596,587 @@ public void testCreateSnapshotMissingMandatoryParams() throws Exception { () -> createSnapshot(nullstr, bucket)); } + private Set getDeletedKeysFromRocksDb( + OMMetadataManager metadataManager) throws IOException { + Set deletedKeys = Sets.newHashSet(); + try (TableIterator> + deletedTableIterator = metadataManager.getDeletedTable() + .iterator()) { + while (deletedTableIterator.hasNext()) { + Table.KeyValue val = + deletedTableIterator.next(); + deletedKeys.addAll(val.getValue().getOmKeyInfoList()); + } + } + + try (TableIterator> + deletedDirTableIterator = metadataManager.getDeletedDirTable() + .iterator()) { + while (deletedDirTableIterator.hasNext()) { + deletedKeys.add(deletedDirTableIterator.next().getValue()); + } + } + return deletedKeys; + } + + private OmKeyInfo getOmKeyInfo(String volume, String bucket, + String key) throws IOException { + return cluster.getOzoneManager().getKeyManager() + .getKeyInfo(new OmKeyArgs.Builder().setVolumeName(volume) + .setBucketName(bucket).setKeyName(key).build(), null); + } + + /** + * Testing scenario: + * 1) Key k1 is created. + * 2) Snapshot snap1 created. + * 3) Snapshot snap2 is created + * 4) Key k1 is deleted. + * 5) Snapshot snap3 is created. + * 6) Snapdiff b/w snap3 & snap2 taken to assert difference of 1 key + */ + @Test + public void testSnapDiffHandlingReclaimWithLatestUse() throws Exception { + String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5); + String testBucketName = "bucket1"; + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + OzoneBucket bucket = volume.getBucket(testBucketName); + String key1 = "k1"; + key1 = createFileKeyWithPrefix(bucket, key1); + String snap1 = "snap1"; + createSnapshot(testVolumeName, testBucketName, snap1); + String snap2 = "snap2"; + createSnapshot(testVolumeName, testBucketName, snap2); + getOmKeyInfo(testVolumeName, testBucketName, key1); + bucket.deleteKey(key1); + String snap3 = "snap3"; + createSnapshot(testVolumeName, testBucketName, snap3); + SnapshotDiffReportOzone diff = + getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2); + Assert.assertEquals(diff.getDiffList().size(), 0); + diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3); + Assert.assertEquals(diff.getDiffList().size(), 1); + Assert.assertEquals(diff.getDiffList(), Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.DELETE, key1))); + } + + /** + * Testing scenario: + * 1) Key k1 is created. + * 2) Snapshot snap1 created. + * 3) Key k1 is deleted. + * 4) Snapshot snap2 is created. + * 5) Snapshot snap3 is created. + * 6) Snapdiff b/w snap3 & snap1 taken to assert difference of 1 key. + * 7) Snapdiff b/w snap3 & snap2 taken to assert difference of 0 key. + */ + @Test + public void testSnapDiffHandlingReclaimWithPreviousUse() throws Exception { + String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5); + String testBucketName = "bucket1"; + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + OzoneBucket bucket = volume.getBucket(testBucketName); + String key1 = "k1"; + key1 = createFileKeyWithPrefix(bucket, key1); + getOmKeyInfo(testVolumeName, testBucketName, key1); + String snap1 = "snap1"; + createSnapshot(testVolumeName, testBucketName, snap1); + bucket.deleteKey(key1); + String snap2 = "snap2"; + createSnapshot(testVolumeName, testBucketName, snap2); + String snap3 = "snap3"; + createSnapshot(testVolumeName, testBucketName, snap3); + SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, + testBucketName, snap1, snap3); + Assert.assertEquals(diff.getDiffList().size(), 1); + Assert.assertEquals(diff.getDiffList(), Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.DELETE, key1))); + diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3); + Assert.assertEquals(diff.getDiffList().size(), 0); + } + + /** + * Testing scenario: + * 1) Key k1 is created. + * 2) Snapshot snap1 created. + * 3) Key k1 is deleted. + * 4) Key k1 is recreated. + * 5) Snapshot snap2 is created. + * 6) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * of 2 keys. + * 7) Snapdiff b/w snapshot of Active FS & snap2 taken to assert difference + * of 0 key. + * 8) Checking rocks db to ensure the object created shouldn't be reclaimed + * as it is used by snapshot. + * 9) Key k1 is deleted. + * 10) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * of 1 key. + * 11) Snapdiff b/w snapshot of Active FS & snap2 taken to assert difference + * of 1 key. + */ + @Test + public void testSnapDiffReclaimWithKeyRecreation() throws Exception { + String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5); + String testBucketName = "bucket1"; + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + OzoneBucket bucket = volume.getBucket(testBucketName); + String key1 = "k1"; + key1 = createFileKeyWithPrefix(bucket, key1); + getOmKeyInfo(testVolumeName, testBucketName, key1); + String snap1 = "snap1"; + createSnapshot(testVolumeName, testBucketName, snap1); + bucket.deleteKey(key1); + key1 = createFileKey(bucket, key1); + getOmKeyInfo(testVolumeName, testBucketName, key1); + String snap2 = "snap2"; + createSnapshot(testVolumeName, testBucketName, snap2); + String snap3 = "snap3"; + createSnapshot(testVolumeName, testBucketName, snap3); + SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, + testBucketName, snap1, snap3); + Assert.assertEquals(diff.getDiffList().size(), 2); + Assert.assertEquals(diff.getDiffList(), Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.DELETE, key1), + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.CREATE, key1))); + diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3); + Assert.assertEquals(diff.getDiffList().size(), 0); + bucket.deleteKey(key1); + String snap4 = "snap4"; + createSnapshot(testVolumeName, testBucketName, snap4); + diff = getSnapDiffReport(testVolumeName, testBucketName, snap1, snap4); + Assert.assertEquals(diff.getDiffList().size(), 1); + Assert.assertEquals(diff.getDiffList(), Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.DELETE, key1))); + diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap4); + Assert.assertEquals(diff.getDiffList().size(), 1); + Assert.assertEquals(diff.getDiffList(), Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.DELETE, key1))); + } + + /** + * Testing scenario: + * 1) Key k1 is created. + * 2) Snapshot snap1 created. + * 3) Key k1 is renamed to renamed-k1. + * 4) Key renamed-k1 is deleted. + * 5) Snapshot snap2 created. + * 4) Snapdiff b/w snap2 & snap1 taken to assert difference of 1 key. + */ + @Test + public void testSnapDiffReclaimWithKeyRename() throws Exception { + String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5); + String testBucketName = "bucket1"; + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + OzoneBucket bucket = volume.getBucket(testBucketName); + String key1 = "k1"; + key1 = createFileKeyWithPrefix(bucket, key1); + String snap1 = "snap1"; + createSnapshot(testVolumeName, testBucketName, snap1); + String renamedKey = "renamed-" + key1; + bucket.renameKey(key1, renamedKey); + GenericTestUtils.waitFor(() -> { + try { + getOmKeyInfo(testVolumeName, testBucketName, renamedKey); + } catch (IOException e) { + return false; + } + return true; + }, 1000, 10000); + getOmKeyInfo(testVolumeName, testBucketName, renamedKey); + bucket.deleteKey(renamedKey); + String snap2 = "snap2"; + createSnapshot(testVolumeName, testBucketName, snap2); + SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, + testBucketName, snap1, snap2); + Assert.assertEquals(diff.getDiffList().size(), 1); + Assert.assertEquals(diff.getDiffList(), Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.DELETE, key1) + )); + } + + /** + * Testing scenario: + * 1) Key k1 is created. + * 2) Snapshot snap1 created. + * 3) Key k1 is renamed to renamed-k1. + * 4) Key renamed-k1 is renamed to renamed-renamed-k1. + * 5) Key renamed-renamed-k1 is deleted. + * 6) Snapshot snap2 is created. + * 7) Snapdiff b/w Active FS & snap1 taken to assert difference of 1 key. + */ + @Test + public void testSnapDiffWith2RenamesAndDelete() throws Exception { + String testVolumeName = "vol" + counter.incrementAndGet(); + String testBucketName = "bucket" + counter.incrementAndGet(); + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + OzoneBucket bucket = volume.getBucket(testBucketName); + String key1 = "k1"; + key1 = createFileKeyWithPrefix(bucket, key1); + String snap1 = "snap1"; + createSnapshot(testVolumeName, testBucketName, snap1); + String renamedKey = "renamed-" + key1; + bucket.renameKey(key1, renamedKey); + GenericTestUtils.waitFor(() -> { + try { + getOmKeyInfo(testVolumeName, testBucketName, renamedKey); + } catch (IOException e) { + return false; + } + return true; + }, 1000, 120000); + String renamedRenamedKey = "renamed-" + renamedKey; + bucket.renameKey(renamedKey, renamedRenamedKey); + GenericTestUtils.waitFor(() -> { + try { + getOmKeyInfo(testVolumeName, testBucketName, renamedRenamedKey); + } catch (IOException e) { + return false; + } + return true; + }, 1000, 120000); + getOmKeyInfo(testVolumeName, testBucketName, renamedRenamedKey); + bucket.deleteKey(renamedRenamedKey); + String snap2 = "snap2"; + createSnapshot(testVolumeName, testBucketName, snap2); + String snap3 = "snap3"; + createSnapshot(testVolumeName, testBucketName, + snap3); + SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName, + snap1, snap3); + Assert.assertEquals(diff.getDiffList().size(), 1); + Assert.assertEquals(diff.getDiffList(), Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.DELETE, key1) + )); + } + + /** + * Testing scenario: + * 1) Key k1 is created. + * 2) Snapshot snap1 created. + * 3) Key k1 is renamed to renamed-k1. + * 4) Key k1 is recreated. + * 5) Key k1 is deleted. + * 6) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * of 1 key. + */ + @Test + public void testSnapDiffWithKeyRenamesRecreationAndDelete() + throws Exception { + String testVolumeName = "vol" + counter.incrementAndGet(); + String testBucketName = "bucket1"; + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + OzoneBucket bucket = volume.getBucket(testBucketName); + String key1 = "k1"; + key1 = createFileKeyWithPrefix(bucket, key1); + String snap1 = "snap1"; + createSnapshot(testVolumeName, testBucketName, snap1); + String renamedKey = "renamed-" + key1; + bucket.renameKey(key1, renamedKey); + key1 = createFileKey(bucket, key1); + getOmKeyInfo(testVolumeName, testBucketName, key1); + String snap3 = "snap3"; + createSnapshot(testVolumeName, testBucketName, snap3); + getSnapDiffReport(testVolumeName, testBucketName, + snap1, snap3); + bucket.deleteKey(key1); + String activeSnap = "activefs"; + createSnapshot(testVolumeName, testBucketName, activeSnap); + SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName, + snap1, activeSnap); + Assert.assertEquals(diff.getDiffList().size(), 1); + Assert.assertEquals(diff.getDiffList(), Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.RENAME, + key1, renamedKey) + )); + } + + /** + * Testing scenario: + * 1) Snapshot snap1 created. + * 2) Key k1 is created. + * 3) Key k1 is deleted. + * 4) Snapshot s2 is created before key k1 is reclaimed. + * 5) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * of 0 keys. + * 6) Snapdiff b/w snapshot of Active FS & snap2 taken to assert difference + * of 0 keys. + */ + @Test + public void testSnapDiffReclaimWithDeferredKeyDeletion() throws Exception { + String testVolumeName = "vol" + counter.incrementAndGet(); + String testBucketName = "bucket1"; + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + OzoneBucket bucket = volume.getBucket(testBucketName); + String snap1 = "snap1"; + createSnapshot(testVolumeName, testBucketName, snap1); + String key1 = "k1"; + key1 = createFileKeyWithPrefix(bucket, key1); + getOmKeyInfo(testVolumeName, testBucketName, key1); + bucket.deleteKey(key1); + String snap2 = "snap2"; + createSnapshot(testVolumeName, testBucketName, snap2); + String activeSnap = "activefs"; + createSnapshot(testVolumeName, testBucketName, activeSnap); + SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName, + snap1, activeSnap); + Assert.assertEquals(diff.getDiffList().size(), 0); + diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, activeSnap); + Assert.assertEquals(diff.getDiffList().size(), 0); + } + + /** + * Testing scenario: + * 1) Key k1 is created. + * 2) Snapshot snap1 created. + * 3) Key k1 is renamed to key k1_renamed + * 4) Key k1_renamed is renamed to key k1 + * 5) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * of 1 key with 1 Modified entry. + */ + @Test + public void testSnapDiffWithNoEffectiveRename() throws Exception { + String testVolumeName = "vol" + counter.incrementAndGet(); + String testBucketName = "bucket1"; + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + OzoneBucket bucket = volume.getBucket(testBucketName); + String snap1 = "snap1"; + String key1 = "k1"; + key1 = createFileKeyWithPrefix(bucket, key1); + createSnapshot(testVolumeName, testBucketName, snap1); + getOmKeyInfo(bucket.getVolumeName(), bucket.getName(), + key1); + String key1Renamed = key1 + "_renamed"; + + bucket.renameKey(key1, key1Renamed); + bucket.renameKey(key1Renamed, key1); + + + String snap2 = "snap2"; + createSnapshot(testVolumeName, testBucketName, snap2); + SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName, + snap1, snap2); + getOmKeyInfo(bucket.getVolumeName(), bucket.getName(), + key1); + Assert.assertEquals(diff.getDiffList(), Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.MODIFY, key1))); + } + + /** + * Testing scenario: + * 1) Key k1 is created. + * 2) Snapshot snap1 created. + * 3) Dir dir1/dir2 is created. + * 4) Key k1 is renamed to key dir1/dir2/k1_renamed + * 5) Snapdiff b/w snapshot of Active FS & snap1 taken to assert difference + * of 3 key + * with 1 rename entry & 2 dirs create entry. + */ + @Test + public void testSnapDiffWithDirectory() throws Exception { + + String testVolumeName = "vol" + counter.incrementAndGet(); + String testBucketName = "bucket1"; + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + OzoneBucket bucket = volume.getBucket(testBucketName); + String snap1 = "snap1"; + String key1 = "k1"; + key1 = createFileKeyWithPrefix(bucket, key1); + createSnapshot(testVolumeName, testBucketName, snap1); + getOmKeyInfo(bucket.getVolumeName(), bucket.getName(), + key1); + bucket.createDirectory("dir1/dir2"); + String key1Renamed = "dir1/dir2/" + key1 + "_renamed"; + bucket.renameKey(key1, key1Renamed); + + String snap2 = "snap2"; + createSnapshot(testVolumeName, testBucketName, snap2); + SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName, + snap1, snap2); + + List diffEntries; + if (bucketLayout.isFileSystemOptimized()) { + diffEntries = Lists.newArrayList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.RENAME, key1, + "dir1/dir2/" + key1 + "_renamed"), + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.CREATE, "dir1"), + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.CREATE, "dir1/dir2")); + } else { + diffEntries = Lists.newArrayList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.RENAME, + key1, "dir1/dir2/" + key1 + "_renamed"), + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.CREATE, "dir1/dir2"), + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.CREATE, "dir1")); + } + Assert.assertEquals(diff.getDiffList(), diffEntries); + } + + @Test + public void testSnapdiffWithFilesystemCreate() + throws IOException, URISyntaxException, InterruptedException, + TimeoutException { + + Assume.assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths)); + String testVolumeName = "vol" + counter.incrementAndGet(); + String testBucketName = "bucket" + counter.incrementAndGet(); + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + String rootPath = String.format("%s://%s.%s/", + OzoneConsts.OZONE_URI_SCHEME, testBucketName, testVolumeName); + try (FileSystem fs = FileSystem.get(new URI(rootPath), cluster.getConf())) { + String snap1 = "snap1"; + String key = "/dir1/dir2/key1"; + createSnapshot(testVolumeName, testBucketName, snap1); + createFileKey(fs, key); + String snap2 = "snap2"; + createSnapshot(testVolumeName, testBucketName, snap2); + SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, + testBucketName, snap1, snap2); + int idx = bucketLayout.isFileSystemOptimized() ? 0 : + diff.getDiffList().size() - 1; + Path p = new Path("/"); + while (true) { + FileStatus[] fileStatuses = fs.listStatus(p); + Assertions.assertEquals(fileStatuses[0].isDirectory(), + bucketLayout.isFileSystemOptimized() && + idx < diff.getDiffList().size() - 1 || + !bucketLayout.isFileSystemOptimized() && idx > 0); + p = fileStatuses[0].getPath(); + Assertions.assertEquals(diff.getDiffList().get(idx), + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.CREATE, p.toUri().getPath() + .substring(1))); + if (fileStatuses[0].isFile()) { + break; + } + idx += bucketLayout.isFileSystemOptimized() ? 1 : -1; + } + } + } + + @Test + public void testSnapdiffWithFilesystemDirectoryRenameOperation() + throws IOException, URISyntaxException, InterruptedException, + TimeoutException { + Assume.assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths)); + String testVolumeName = "vol" + counter.incrementAndGet(); + String testBucketName = "bucket" + counter.incrementAndGet(); + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + String rootPath = String.format("%s://%s.%s/", + OzoneConsts.OZONE_URI_SCHEME, testBucketName, testVolumeName); + try (FileSystem fs = FileSystem.get(new URI(rootPath), cluster.getConf())) { + String snap1 = "snap1"; + String key = "/dir1/dir2/key1"; + createFileKey(fs, key); + createSnapshot(testVolumeName, testBucketName, snap1); + String snap2 = "snap2"; + fs.rename(new Path("/dir1/dir2"), new Path("/dir1/dir3")); + createSnapshot(testVolumeName, testBucketName, snap2); + SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, + testBucketName, snap1, snap2); + if (bucketLayout.isFileSystemOptimized()) { + Assertions.assertEquals(Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.RENAME, "dir1/dir2", "dir1/dir3"), + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.MODIFY, "dir1")), + diff.getDiffList()); + } else { + Assertions.assertEquals(Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.RENAME, "dir1/dir2/key1", + "dir1/dir3/key1"), + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.RENAME, "dir1/dir2", + "dir1/dir3")), + diff.getDiffList()); + } + + } + } + + @Test + public void testSnapdiffWithFilesystemDirectoryMoveOperation() + throws IOException, URISyntaxException, InterruptedException, + TimeoutException { + Assume.assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths)); + String testVolumeName = "vol" + counter.incrementAndGet(); + String testBucketName = "bucket" + counter.incrementAndGet(); + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + String rootPath = String.format("%s://%s.%s/", + OzoneConsts.OZONE_URI_SCHEME, testBucketName, testVolumeName); + try (FileSystem fs = FileSystem.get(new URI(rootPath), cluster.getConf())) { + String snap1 = "snap1"; + String key = "/dir1/dir2/key1"; + createFileKey(fs, key); + fs.mkdirs(new Path("/dir3")); + createSnapshot(testVolumeName, testBucketName, snap1); + String snap2 = "snap2"; + fs.rename(new Path("/dir1/dir2"), new Path("/dir3/dir2")); + createSnapshot(testVolumeName, testBucketName, snap2); + SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, + testBucketName, snap1, snap2); + if (bucketLayout.isFileSystemOptimized()) { + Assertions.assertEquals(Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.RENAME, "dir1/dir2", "dir3/dir2"), + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.MODIFY, "dir1"), + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.MODIFY, "dir3")), + diff.getDiffList()); + } else { + Assertions.assertEquals(Arrays.asList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.RENAME, "dir1/dir2/key1", + "dir3/dir2/key1"), + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.RENAME, "dir1/dir2", + "dir3/dir2")), + diff.getDiffList()); + } + } + } + @Test public void testBucketDeleteIfSnapshotExists() throws Exception { String volume1 = "vol-" + counter.incrementAndGet(); @@ -726,7 +1332,7 @@ public void testSnapDiffCancel() throws Exception { createSnapshot(volumeName, bucketName, toSnapName); SnapshotDiffResponse response = store.snapshotDiff(volumeName, bucketName, - fromSnapName, toSnapName, null, 0, false); + fromSnapName, toSnapName, null, 0, false, disableNativeDiff); assertEquals(IN_PROGRESS, response.getJobStatus()); @@ -736,7 +1342,7 @@ public void testSnapDiffCancel() throws Exception { assertEquals(CANCEL_SUCCEEDED.getMessage(), cancelResponse.getMessage()); response = store.snapshotDiff(volumeName, bucketName, fromSnapName, - toSnapName, null, 0, false); + toSnapName, null, 0, false, disableNativeDiff); // Job status should be updated to CANCELLED. assertEquals(CANCELLED, response.getJobStatus()); @@ -751,7 +1357,7 @@ public void testSnapDiffCancel() throws Exception { response = store.snapshotDiff(volumeName, bucketName, fromSnapName, toSnapName, - null, 0, false); + null, 0, false, disableNativeDiff); assertEquals(CANCELLED, response.getJobStatus()); String fromSnapshotTableKey = @@ -806,7 +1412,7 @@ public void testSnapDiffCancelFailureResponses() throws Exception { SnapshotDiffResponse response = store.snapshotDiff( volumeName, bucketName, fromSnapName, toSnapName, - null, 0, false); + null, 0, false, disableNativeDiff); Assert.assertEquals(IN_PROGRESS, response.getJobStatus()); @@ -819,7 +1425,7 @@ public void testSnapDiffCancelFailureResponses() throws Exception { response = store.snapshotDiff( volumeName, bucketName, fromSnapName, toSnapName, - null, 0, false); + null, 0, false, disableNativeDiff); Assert.assertEquals(CANCELLED, response.getJobStatus()); @@ -838,7 +1444,8 @@ private SnapshotDiffReportOzone getSnapDiffReport(String volume, SnapshotDiffResponse response; do { response = store.snapshotDiff(volume, bucket, fromSnapshot, - toSnapshot, null, 0, false); + toSnapshot, null, 0, forceFullSnapshotDiff, + disableNativeDiff); Thread.sleep(response.getWaitTimeInMs()); } while (response.getJobStatus() != DONE); @@ -862,19 +1469,20 @@ public void testSnapDiffNoSnapshot() throws Exception { // Destination snapshot is invalid OMException omException = assertThrows(OMException.class, () -> store.snapshotDiff(volume, bucket, snap1, snap2, - null, 0, false)); + null, 0, false, disableNativeDiff)); assertEquals(KEY_NOT_FOUND, omException.getResult()); // From snapshot is invalid omException = assertThrows(OMException.class, () -> store.snapshotDiff(volume, bucket, snap2, snap1, - null, 0, false)); + null, 0, false, disableNativeDiff)); assertEquals(KEY_NOT_FOUND, omException.getResult()); createSnapshot(volume, bucket, snap2); omException = assertThrows(OMException.class, () -> - store.snapshotDiff(volume, bucket, snap2, snap1, null, 0, false) + store.snapshotDiff(volume, bucket, snap2, snap1, null, 0, false, + disableNativeDiff) ); assertEquals(INTERNAL_ERROR, omException.getResult()); @@ -905,17 +1513,52 @@ public void testSnapDiffNonExistentUrl() throws Exception { LambdaTestUtils.intercept(OMException.class, "KEY_NOT_FOUND", () -> store.snapshotDiff(volumea, bucketb, snap1, snap2, - null, 0, false)); + null, 0, forceFullSnapshotDiff, disableNativeDiff)); // Volume is nonexistent LambdaTestUtils.intercept(OMException.class, "KEY_NOT_FOUND", () -> store.snapshotDiff(volumeb, bucketa, snap2, snap1, - null, 0, false)); + null, 0, forceFullSnapshotDiff, + disableNativeDiff)); // Both volume and bucket are nonexistent LambdaTestUtils.intercept(OMException.class, "KEY_NOT_FOUND", () -> store.snapshotDiff(volumeb, bucketb, snap2, snap1, - null, 0, false)); + null, 0, forceFullSnapshotDiff, + disableNativeDiff)); + } + + /** + * Testing scenario: + * 1) Key k1 is created. + * 2) Snapshot snap1 created. + * 3) Key k1 is recreated. + * 4) Snapshot s2 is created. + * 5) Snapdiff b/w Snap1 & Snap2 taken to assert difference of 1 key. + */ + @Test + public void testSnapDiffWithKeyOverwrite() throws Exception { + String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5); + String testBucketName = "bucket1"; + store.createVolume(testVolumeName); + OzoneVolume volume = store.getVolume(testVolumeName); + volume.createBucket(testBucketName); + OzoneBucket bucket = volume.getBucket(testBucketName); + String key1 = "k1"; + key1 = createFileKeyWithPrefix(bucket, key1); + String snap1 = "snap1"; + createSnapshot(testVolumeName, testBucketName, snap1); + getOmKeyInfo(testVolumeName, testBucketName, key1); + createFileKey(bucket, key1); + String snap2 = "snap2"; + createSnapshot(testVolumeName, testBucketName, snap2); + SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName, + testBucketName, snap1, snap2); + getOmKeyInfo(testVolumeName, testBucketName, key1); + Assert.assertEquals(diff.getDiffList().size(), 1); + Assert.assertEquals(diff.getDiffList(), Lists.newArrayList( + SnapshotDiffReportOzone.getDiffReportEntry( + SnapshotDiffReport.DiffType.MODIFY, key1))); } @Test @@ -938,20 +1581,20 @@ public void testSnapDiffMissingMandatoryParams() throws Exception { LambdaTestUtils.intercept(OMException.class, "KEY_NOT_FOUND", () -> store.snapshotDiff(volume, bucket, snap1, nullstr, - null, 0, false)); + null, 0, forceFullSnapshotDiff, disableNativeDiff)); // From snapshot is empty LambdaTestUtils.intercept(OMException.class, "KEY_NOT_FOUND", () -> store.snapshotDiff(volume, bucket, nullstr, snap1, - null, 0, false)); + null, 0, forceFullSnapshotDiff, disableNativeDiff)); // Bucket is empty assertThrows(IllegalArgumentException.class, () -> store.snapshotDiff(volume, nullstr, snap1, snap2, - null, 0, false)); + null, 0, forceFullSnapshotDiff, disableNativeDiff)); // Volume is empty assertThrows(IllegalArgumentException.class, () -> store.snapshotDiff(nullstr, bucket, snap1, snap2, - null, 0, false)); + null, 0, forceFullSnapshotDiff, disableNativeDiff)); } @Test @@ -1021,7 +1664,8 @@ public void testListSnapshotDiffWithInvalidParameters() String snap2 = "snap-2-" + RandomStringUtils.randomNumeric(5); createSnapshot(volume, bucket, snap2); - store.snapshotDiff(volume, bucket, snap1, snap2, null, 0, true); + store.snapshotDiff(volume, bucket, snap1, snap2, null, 0, + forceFullSnapshotDiff, disableNativeDiff); String invalidStatus = "invalid"; String statusErrorMessage = "Invalid job status: " + invalidStatus; @@ -1041,7 +1685,7 @@ public void testListSnapshotDiffWithInvalidParameters() @Ignore //TODO - Fix in HDDS-8005 @Test public void testSnapDiffWithMultipleSSTs() - throws IOException, InterruptedException, TimeoutException { + throws Exception { // Create a volume and 2 buckets String volumeName1 = "vol-" + counter.incrementAndGet(); String bucketName1 = "buck1"; @@ -1076,7 +1720,7 @@ public void testSnapDiffWithMultipleSSTs() Assert.assertEquals(4, getKeyTableSstFiles().size()); SnapshotDiffReportOzone diff1 = store.snapshotDiff(volumeName1, bucketName1, snap1, snap2, - null, 0, false) + null, 0, forceFullSnapshotDiff, disableNativeDiff) .getSnapshotDiffReport(); Assert.assertEquals(1, diff1.getDiffList().size()); } @@ -1203,18 +1847,45 @@ private void deleteKeys(OzoneBucket bucket) throws IOException { } private String createFileKeyWithPrefix(OzoneBucket bucket, String keyPrefix) - throws IOException { + throws Exception { String key = keyPrefix + counter.incrementAndGet(); - createFileKey(bucket, key); - return key; + return createFileKey(bucket, key); } - private void createFileKey(OzoneBucket bucket, String key) - throws IOException { + private String createFileKey(OzoneBucket bucket, String key) + throws Exception { byte[] value = RandomStringUtils.randomAscii(10240).getBytes(UTF_8); OzoneOutputStream fileKey = bucket.createKey(key, value.length); fileKey.write(value); fileKey.close(); + GenericTestUtils.waitFor(() -> { + try { + getOmKeyInfo(bucket.getVolumeName(), bucket.getName(), key); + } catch (IOException e) { + return false; + } + return true; + }, 300, 10000); + return key; + } + + private String createFileKey(FileSystem fs, + String path) + throws IOException, InterruptedException, TimeoutException { + byte[] value = RandomStringUtils.randomAscii(10240).getBytes(UTF_8); + Path pathVal = new Path(path); + FSDataOutputStream fileKey = fs.create(pathVal); + fileKey.write(value); + fileKey.close(); + GenericTestUtils.waitFor(() -> { + try { + fs.getFileStatus(pathVal); + } catch (IOException e) { + return false; + } + return true; + }, 1000, 30000); + return path; } @Test @@ -1235,14 +1906,14 @@ public void testSnapshotOpensWithDisabledAutoCompaction() throws Exception { // Test snapshot diff when OM restarts in non-HA OM env and diff job is // in_progress when it restarts. @Test - public void testSnapshotDiffWhenOmRestart() - throws IOException, InterruptedException { + public void testSnapshotDiffWhenOmRestart() throws Exception { String snapshot1 = "snap-" + RandomStringUtils.randomNumeric(5); String snapshot2 = "snap-" + RandomStringUtils.randomNumeric(5); createSnapshots(snapshot1, snapshot2); SnapshotDiffResponse response = store.snapshotDiff(volumeName, bucketName, - snapshot1, snapshot2, null, 0, false); + snapshot1, snapshot2, null, 0, forceFullSnapshotDiff, + disableNativeDiff); assertEquals(IN_PROGRESS, response.getJobStatus()); @@ -1253,7 +1924,8 @@ public void testSnapshotDiffWhenOmRestart() until(() -> cluster.getOzoneManager().isRunning()); response = store.snapshotDiff(volumeName, bucketName, - snapshot1, snapshot2, null, 0, false); + snapshot1, snapshot2, null, 0, forceFullSnapshotDiff, + disableNativeDiff); // If job was IN_PROGRESS or DONE state when OM restarted, it should be // DONE by this time. @@ -1277,7 +1949,7 @@ public void testSnapshotDiffWhenOmRestart() // partially received. @Test public void testSnapshotDiffWhenOmRestartAndReportIsPartiallyFetched() - throws IOException, InterruptedException { + throws Exception { int pageSize = 10; String snapshot1 = "snap-" + RandomStringUtils.randomNumeric(5); String snapshot2 = "snap-" + RandomStringUtils.randomNumeric(5); @@ -1311,7 +1983,8 @@ private SnapshotDiffReportOzone fetchReportPage(String fromSnapshot, while (true) { SnapshotDiffResponse response = store.snapshotDiff(volumeName, bucketName, - fromSnapshot, toSnapshot, token, pageSize, false); + fromSnapshot, toSnapshot, token, pageSize, forceFullSnapshotDiff, + disableNativeDiff); if (response.getJobStatus() == IN_PROGRESS) { Thread.sleep(response.getWaitTimeInMs()); } else if (response.getJobStatus() == DONE) { @@ -1323,7 +1996,7 @@ private SnapshotDiffReportOzone fetchReportPage(String fromSnapshot, } private void createSnapshots(String snapshot1, - String snapshot2) throws IOException { + String snapshot2) throws Exception { createFileKeyWithPrefix(ozoneBucket, "key"); store.createSnapshot(volumeName, bucketName, snapshot1); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHASnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHASnapshot.java index b294707e2f8e..7f071f11373b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHASnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHASnapshot.java @@ -113,7 +113,7 @@ public void testSnapshotDiffWhenOmLeaderRestart() SnapshotDiffResponse response = store.snapshotDiff(volumeName, bucketName, - snapshot1, snapshot2, null, 0, false); + snapshot1, snapshot2, null, 0, false, false); assertEquals(IN_PROGRESS, response.getJobStatus()); @@ -131,18 +131,18 @@ public void testSnapshotDiffWhenOmLeaderRestart() if (Objects.equals(oldLeader, newLeader)) { // If old leader becomes leader again. Job should be done by this time. response = store.snapshotDiff(volumeName, bucketName, - snapshot1, snapshot2, null, 0, false); + snapshot1, snapshot2, null, 0, false, false); assertEquals(DONE, response.getJobStatus()); assertEquals(100, response.getSnapshotDiffReport().getDiffList().size()); } else { // If new leader is different from old leader. SnapDiff request will be // new to OM, and job status should be IN_PROGRESS. response = store.snapshotDiff(volumeName, bucketName, snapshot1, - snapshot2, null, 0, false); + snapshot2, null, 0, false, false); assertEquals(IN_PROGRESS, response.getJobStatus()); while (true) { response = store.snapshotDiff(volumeName, bucketName, snapshot1, - snapshot2, null, 0, false); + snapshot2, null, 0, false, false); if (DONE == response.getJobStatus()) { assertEquals(100, response.getSnapshotDiffReport().getDiffList().size()); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index f38873a9e009..2655c30c1155 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -829,6 +829,7 @@ message SnapshotDiffJobProto { optional bool forceFullDiff = 8; optional uint64 totalDiffEntries = 9; optional string message = 10; + optional bool disableNativeDiff = 11; } message OzoneObj { @@ -1753,6 +1754,7 @@ message SnapshotDiffRequest { optional string token = 5; optional uint32 pageSize = 6; optional bool forceFullDiff = 7; + optional bool disableNativeDiff = 8; } message CancelSnapshotDiffRequest { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 79ba34e1cdd4..7f6b8050a95b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -734,13 +734,15 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff( toSnapshot); } + @SuppressWarnings("parameternumber") public SnapshotDiffResponse getSnapshotDiffReport(final String volume, final String bucket, final String fromSnapshot, final String toSnapshot, final String token, int pageSize, - boolean forceFullDiff) + boolean forceFullDiff, + boolean disableNativeDiff) throws IOException { validateSnapshotsExistAndActive(volume, bucket, fromSnapshot, toSnapshot); @@ -760,7 +762,8 @@ public SnapshotDiffResponse getSnapshotDiffReport(final String volume, SnapshotDiffResponse snapshotDiffReport = snapshotDiffManager.getSnapshotDiffReport(volume, bucket, - fromSnapshot, toSnapshot, index, pageSize, forceFullDiff); + fromSnapshot, toSnapshot, index, pageSize, forceFullDiff, + disableNativeDiff); // Check again to make sure that from and to snapshots are still active and // were not deleted in between response generation. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index ccbd673414fe..92a0d4adafe2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -4615,16 +4615,19 @@ IOmMetadataReader, SnapshotCache> getReader(OzoneObj ozoneObj) false); } + @SuppressWarnings("parameternumber") public SnapshotDiffResponse snapshotDiff(String volume, String bucket, String fromSnapshot, String toSnapshot, String token, int pageSize, - boolean forceFullDiff) + boolean forceFullDiff, + boolean disableNativeDiff) throws IOException { return omSnapshotManager.getSnapshotDiffReport(volume, bucket, - fromSnapshot, toSnapshot, token, pageSize, forceFullDiff); + fromSnapshot, toSnapshot, token, pageSize, forceFullDiff, + disableNativeDiff); } public CancelSnapshotDiffResponse cancelSnapshotDiff(String volume, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 19f2976978e8..59f9e0cc7a53 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -112,6 +112,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_THREAD_POOL_SIZE_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS_DEFAULT; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.FILE_TABLE; import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; @@ -183,6 +185,9 @@ public class SnapshotDiffManager implements AutoCloseable { private final String sstBackupDirForSnapDiffJobs; private final boolean snapshotForceFullDiff; + + private final boolean diffDisableNativeLibs; + private final Optional sstDumpTool; private Optional sstDumpToolExecService; @@ -218,6 +223,10 @@ public SnapshotDiffManager(ManagedRocksDB db, OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF, OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT); + this.diffDisableNativeLibs = ozoneManager.getConfiguration().getBoolean( + OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS, + OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS_DEFAULT); + this.maxAllowedKeyChangesForASnapDiff = ozoneManager.getConfiguration() .getLong( OZONE_OM_SNAPSHOT_DIFF_MAX_ALLOWED_KEYS_CHANGED_PER_DIFF_JOB, @@ -493,6 +502,7 @@ private JobStatus getJobStatus(String jobStatus) } } + @SuppressWarnings("parameternumber") public SnapshotDiffResponse getSnapshotDiffReport( final String volumeName, final String bucketName, @@ -500,7 +510,8 @@ public SnapshotDiffResponse getSnapshotDiffReport( final String toSnapshotName, final int index, final int pageSize, - final boolean forceFullDiff + final boolean forceFullDiff, + final boolean disableNativeDiff ) throws IOException { SnapshotInfo fsInfo = getSnapshotInfo(ozoneManager, @@ -512,14 +523,15 @@ public SnapshotDiffResponse getSnapshotDiffReport( SnapshotDiffJob snapDiffJob = getSnapDiffReportStatus(snapDiffJobKey, volumeName, bucketName, fromSnapshotName, toSnapshotName, - forceFullDiff); + forceFullDiff, disableNativeDiff); OFSPath snapshotRoot = getSnapshotRootPath(volumeName, bucketName); switch (snapDiffJob.getStatus()) { case QUEUED: return submitSnapDiffJob(snapDiffJobKey, volumeName, bucketName, - fromSnapshotName, toSnapshotName, index, pageSize, forceFullDiff); + fromSnapshotName, toSnapshotName, index, pageSize, forceFullDiff, + disableNativeDiff); case IN_PROGRESS: return new SnapshotDiffResponse( new SnapshotDiffReportOzone(snapshotRoot.toString(), volumeName, @@ -664,7 +676,8 @@ private synchronized SnapshotDiffResponse submitSnapDiffJob( final String toSnapshot, final int index, final int pageSize, - final boolean forceFullDiff + final boolean forceFullDiff, + final boolean disableNativeDiff ) throws IOException { SnapshotDiffJob snapDiffJob = snapDiffJobTable.get(jobKey); @@ -702,9 +715,10 @@ private synchronized SnapshotDiffResponse submitSnapDiffJob( } return submitSnapDiffJob(jobKey, snapDiffJob.getJobId(), volume, bucket, - fromSnapshot, toSnapshot, forceFullDiff); + fromSnapshot, toSnapshot, forceFullDiff, disableNativeDiff); } + @SuppressWarnings("parameternumber") private synchronized SnapshotDiffResponse submitSnapDiffJob( final String jobKey, final String jobId, @@ -712,8 +726,8 @@ private synchronized SnapshotDiffResponse submitSnapDiffJob( final String bucketName, final String fromSnapshotName, final String toSnapshotName, - final boolean forceFullDiff - ) { + final boolean forceFullDiff, + final boolean disableNativeDiff) { LOG.info("Submitting snap diff report generation request for" + " volume: {}, bucket: {}, fromSnapshot: {} and toSnapshot: {}", @@ -727,7 +741,7 @@ private synchronized SnapshotDiffResponse submitSnapDiffJob( try { snapDiffExecutor.execute(() -> generateSnapshotDiffReport(jobKey, jobId, volumeName, bucketName, fromSnapshotName, toSnapshotName, - forceFullDiff)); + forceFullDiff, disableNativeDiff)); updateJobStatus(jobKey, QUEUED, IN_PROGRESS); return new SnapshotDiffResponse( new SnapshotDiffReportOzone(snapshotRoot.toString(), volumeName, @@ -772,15 +786,15 @@ private synchronized SnapshotDiffJob getSnapDiffReportStatus( String bucketName, String fromSnapshotName, String toSnapshotName, - boolean forceFullDiff - ) { + boolean forceFullDiff, + boolean disableNativeDiff) { SnapshotDiffJob snapDiffJob = snapDiffJobTable.get(jobKey); if (snapDiffJob == null) { String jobId = UUID.randomUUID().toString(); snapDiffJob = new SnapshotDiffJob(System.currentTimeMillis(), jobId, QUEUED, volumeName, bucketName, fromSnapshotName, toSnapshotName, - forceFullDiff, 0L); + forceFullDiff, disableNativeDiff, 0L); snapDiffJobTable.put(jobKey, snapDiffJob); } @@ -808,7 +822,7 @@ boolean areDiffJobAndSnapshotsActive( return true; } - @SuppressWarnings("methodlength") + @SuppressWarnings({"methodlength", "parameternumber"}) @VisibleForTesting void generateSnapshotDiffReport(final String jobKey, final String jobId, @@ -816,7 +830,8 @@ void generateSnapshotDiffReport(final String jobKey, final String bucketName, final String fromSnapshotName, final String toSnapshotName, - final boolean forceFullDiff) { + final boolean forceFullDiff, + final boolean disableNativeDiff) { LOG.info("Started snap diff report generation for volume: '{}', " + "bucket: '{}', fromSnapshot: '{}', toSnapshot: '{}'", volumeName, bucketName, fromSnapshotName, toSnapshotName); @@ -890,6 +905,7 @@ void generateSnapshotDiffReport(final String jobKey, bucketName); boolean useFullDiff = snapshotForceFullDiff || forceFullDiff; + boolean performNonNativeDiff = diffDisableNativeLibs || disableNativeDiff; if (!areDiffJobAndSnapshotsActive(volumeName, bucketName, fromSnapshotName, toSnapshotName)) { @@ -932,7 +948,8 @@ void generateSnapshotDiffReport(final String jobKey, () -> { getDeltaFilesAndDiffKeysToObjectIdToKeyMap(fsKeyTable, tsKeyTable, fromSnapshot, toSnapshot, fsInfo, tsInfo, useFullDiff, - tablePrefixes, objectIdToKeyNameMapForFromSnapshot, + performNonNativeDiff, tablePrefixes, + objectIdToKeyNameMapForFromSnapshot, objectIdToKeyNameMapForToSnapshot, objectIdToDiffObject, oldParentIds, newParentIds, path.toString()); return null; @@ -941,7 +958,8 @@ void generateSnapshotDiffReport(final String jobKey, if (bucketLayout.isFileSystemOptimized()) { getDeltaFilesAndDiffKeysToObjectIdToKeyMap(fsDirTable, tsDirTable, fromSnapshot, toSnapshot, fsInfo, tsInfo, useFullDiff, - tablePrefixes, objectIdToKeyNameMapForFromSnapshot, + performNonNativeDiff, tablePrefixes, + objectIdToKeyNameMapForFromSnapshot, objectIdToKeyNameMapForToSnapshot, objectIdToDiffObject, oldParentIds, newParentIds, path.toString()); } @@ -1035,7 +1053,8 @@ private void getDeltaFilesAndDiffKeysToObjectIdToKeyMap( final Table tsTable, final OmSnapshot fromSnapshot, final OmSnapshot toSnapshot, final SnapshotInfo fsInfo, final SnapshotInfo tsInfo, - final boolean useFullDiff, final Map tablePrefixes, + final boolean useFullDiff, final boolean skipNativeDiff, + final Map tablePrefixes, final PersistentMap oldObjIdToKeyMap, final PersistentMap newObjIdToKeyMap, final PersistentMap objectIdToDiffObject, @@ -1051,7 +1070,7 @@ private void getDeltaFilesAndDiffKeysToObjectIdToKeyMap( // Workaround to handle deletes if native rocksDb tool for reading // tombstone is not loaded. // TODO: [SNAPSHOT] Update Rocksdb SSTFileIterator to read tombstone - if (!sstDumpTool.isPresent()) { + if (skipNativeDiff || !sstDumpTool.isPresent()) { deltaFiles.addAll(getSSTFileListForSnapshot(fromSnapshot, tablesToLookUp)); } @@ -1060,7 +1079,7 @@ private void getDeltaFilesAndDiffKeysToObjectIdToKeyMap( addToObjectIdMap(fsTable, tsTable, deltaFiles, - sstDumpTool.isPresent(), + !skipNativeDiff && sstDumpTool.isPresent(), oldObjIdToKeyMap, newObjIdToKeyMap, objectIdToDiffObject, @@ -1679,7 +1698,8 @@ void loadJobsOnStartUp() { snapshotDiffJob.getBucket(), snapshotDiffJob.getFromSnapshot(), snapshotDiffJob.getToSnapshot(), - snapshotDiffJob.isForceFullDiff()); + snapshotDiffJob.isForceFullDiff(), + snapshotDiffJob.isNativeDiffDisabled()); } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index e6023d23a17e..3177c3f4c41f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -1268,7 +1268,8 @@ private SnapshotDiffResponse snapshotDiff( snapshotDiffRequest.getToSnapshot(), snapshotDiffRequest.getToken(), snapshotDiffRequest.getPageSize(), - snapshotDiffRequest.getForceFullDiff()); + snapshotDiffRequest.getForceFullDiff(), + snapshotDiffRequest.getDisableNativeDiff()); SnapshotDiffResponse.Builder builder = SnapshotDiffResponse.newBuilder() .setJobStatus(response.getJobStatus().toProtobuf()) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java index a2d6b6885b46..8c9507b20ff5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java @@ -299,7 +299,7 @@ private SnapshotDiffJob addJobAndReport(JobStatus jobStatus, String jobKey = fromSnapshot + DELIMITER + toSnapshot; SnapshotDiffJob job = new SnapshotDiffJob(creationTime, jobId, jobStatus, - volume, bucket, fromSnapshot, toSnapshot, false, noOfEntries); + volume, bucket, fromSnapshot, toSnapshot, false, false, noOfEntries); db.get().put(jobTableCfh, codecRegistry.asRawData(jobKey), codecRegistry.asRawData(job)); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 73714951130a..0bf739057666 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -305,7 +305,7 @@ public void init() throws RocksDBException, IOException, ExecutionException { SnapshotDiffJob diffJob = new SnapshotDiffJob(System.currentTimeMillis(), UUID.randomUUID().toString(), jobStatus, VOLUME_NAME, BUCKET_NAME, - baseSnapshotName, targetSnapshotName, false, 0); + baseSnapshotName, targetSnapshotName, false, false, 0); snapshotNames.add(targetSnapshotName); snapshotInfoList.add(targetSnapshot); @@ -828,7 +828,7 @@ public void testGenerateDiffReport() throws IOException { assertEquals(100, totalDiffEntries); SnapshotDiffJob snapshotDiffJob = new SnapshotDiffJob(0, "jobId", - JobStatus.DONE, "vol", "buck", "fs", "ts", + JobStatus.DONE, "vol", "buck", "fs", "ts", false, true, diffMap.size()); SnapshotDiffReportOzone snapshotDiffReportOzone = snapshotDiffManager.createPageResponse(snapshotDiffJob, "vol", @@ -889,11 +889,11 @@ public void testCreatePageResponse(int startIdx, }); SnapshotDiffJob snapshotDiffJob = new SnapshotDiffJob(0, testJobId, - JobStatus.DONE, "vol", "buck", "fs", "ts", + JobStatus.DONE, "vol", "buck", "fs", "ts", false, true, totalNumberOfRecords); SnapshotDiffJob snapshotDiffJob2 = new SnapshotDiffJob(0, testJobId2, - JobStatus.DONE, "vol", "buck", "fs", "ts", + JobStatus.DONE, "vol", "buck", "fs", "ts", false, true, totalNumberOfRecords); db.get().put(snapDiffJobTable, @@ -959,12 +959,12 @@ public void testGetSnapshotDiffReportForCancelledJob() throws IOException { SnapshotDiffManager spy = spy(snapshotDiffManager); doNothing().when(spy).generateSnapshotDiffReport(eq(diffJobKey), anyString(), eq(volumeName), eq(bucketName), eq(fromSnapshotName), - eq(toSnapshotName), eq(false)); + eq(toSnapshotName), eq(false), eq(false)); // Submit a new job. SnapshotDiffResponse snapshotDiffResponse = spy.getSnapshotDiffReport(volumeName, bucketName, fromSnapshotName, - toSnapshotName, 0, 0, false); + toSnapshotName, 0, 0, false, false); assertEquals(JobStatus.IN_PROGRESS, snapshotDiffResponse.getJobStatus()); @@ -976,7 +976,7 @@ public void testGetSnapshotDiffReportForCancelledJob() throws IOException { // Job status should be cancelled until the cleanup // service removes the job from the table. snapshotDiffResponse = spy.getSnapshotDiffReport(volumeName, bucketName, - fromSnapshotName, toSnapshotName, 0, 0, false); + fromSnapshotName, toSnapshotName, 0, 0, false, false); assertEquals(JobStatus.CANCELLED, snapshotDiffResponse.getJobStatus()); @@ -989,7 +989,7 @@ public void testGetSnapshotDiffReportForCancelledJob() throws IOException { // Response should still be cancelled. snapshotDiffResponse = spy.getSnapshotDiffReport(volumeName, bucketName, - fromSnapshotName, toSnapshotName, 0, 0, false); + fromSnapshotName, toSnapshotName, 0, 0, false, false); assertEquals(JobStatus.CANCELLED, snapshotDiffResponse.getJobStatus()); @@ -1039,7 +1039,7 @@ public void testSnapshotDiffCancelFailure(JobStatus jobStatus, String jobId = UUID.randomUUID().toString(); SnapshotDiffJob snapshotDiffJob = new SnapshotDiffJob(0L, jobId, jobStatus, volumeName, bucketName, - fromSnapshotName, toSnapshotName, true, 10); + fromSnapshotName, toSnapshotName, true, false, 10); snapDiffJobMap.put(diffJobKey, snapshotDiffJob); @@ -1123,12 +1123,12 @@ public void testListSnapshotDiffJobs(String jobStatus, SnapshotDiffManager spy = spy(snapshotDiffManager); doNothing().when(spy).generateSnapshotDiffReport(eq(diffJobKey), anyString(), eq(volumeName), eq(bucketName), eq(fromSnapshotName), - eq(toSnapshotName), eq(false)); + eq(toSnapshotName), eq(false), eq(false)); // SnapshotDiffReport SnapshotDiffResponse snapshotDiffResponse = spy.getSnapshotDiffReport(volumeName, bucketName, fromSnapshotName, - toSnapshotName, 0, 0, false); + toSnapshotName, 0, 0, false, false); assertEquals(SnapshotDiffResponse.JobStatus.IN_PROGRESS, snapshotDiffResponse.getJobStatus()); @@ -1170,10 +1170,10 @@ public void testListSnapDiffWithInvalidStatus() throws IOException { doNothing().when(spy).generateSnapshotDiffReport(eq(diffJobKey), anyString(), eq(volumeName), eq(bucketName), eq(fromSnapshotName), - eq(toSnapshotName), eq(false)); + eq(toSnapshotName), eq(false), eq(false)); spy.getSnapshotDiffReport(volumeName, bucketName, fromSnapshotName, - toSnapshotName, 0, 0, false); + toSnapshotName, 0, 0, false, false); // Invalid status, without listAll true, results in an exception. assertThrows(IOException.class, () -> snapshotDiffManager @@ -1275,7 +1275,7 @@ public void testLoadJobsOnStartUp() throws Exception { } ).when(spy).generateSnapshotDiffReport(anyString(), anyString(), eq(VOLUME_NAME), eq(BUCKET_NAME), eq(snapshotInfo.getName()), - eq(snapshotInfoList.get(1).getName()), eq(false)); + eq(snapshotInfoList.get(1).getName()), eq(false), eq(false)); spy.loadJobsOnStartUp(); @@ -1361,7 +1361,7 @@ public void testThreadPoolIsFull(String description, return null; }).when(spy).generateSnapshotDiffReport(anyString(), anyString(), eq(VOLUME_NAME), eq(BUCKET_NAME), eq(fromSnapshotName), - eq(toSnapshotName), eq(false)); + eq(toSnapshotName), eq(false), eq(false)); } } @@ -1419,7 +1419,7 @@ private SnapshotDiffResponse submitJob(SnapshotDiffManager diffManager, String toSnapshotName) { try { return diffManager.getSnapshotDiffReport(VOLUME_NAME, BUCKET_NAME, - fromSnapshotName, toSnapshotName, 0, 1000, false); + fromSnapshotName, toSnapshotName, 0, 1000, false, false); } catch (IOException exception) { throw new RuntimeException(exception); } @@ -1513,11 +1513,11 @@ public void testGetSnapshotDiffReportHappyCase() throws Exception { omMetadataManager); spy.getSnapshotDiffReport(VOLUME_NAME, BUCKET_NAME, fromSnapInfo.getName(), - toSnapInfo.getName(), 0, 1000, false); + toSnapInfo.getName(), 0, 1000, false, false); Thread.sleep(1000L); spy.getSnapshotDiffReport(VOLUME_NAME, BUCKET_NAME, fromSnapInfo.getName(), - toSnapInfo.getName(), 0, 1000, false); + toSnapInfo.getName(), 0, 1000, false, false); SnapshotDiffJob snapDiffJob = getSnapshotDiffJobFromDb(fromSnapInfo, toSnapInfo); @@ -1551,14 +1551,13 @@ public void testGetSnapshotDiffReportJob() throws Exception { } ).when(spy).generateSnapshotDiffReport(anyString(), anyString(), eq(VOLUME_NAME), eq(BUCKET_NAME), eq(snapshotInfo.getName()), - eq(snapshotInfoList.get(0).getName()), eq(false)); + eq(snapshotInfoList.get(0).getName()), eq(false), eq(false)); for (int i = 0; i < snapshotInfoList.size(); i++) { SnapshotDiffResponse snapshotDiffReport = spy.getSnapshotDiffReport(VOLUME_NAME, BUCKET_NAME, snapshotInfo.getName(), snapshotInfoList.get(i).getName(), 0, - 1000, - false); + 1000, false, false); SnapshotDiffJob diffJob = snapDiffJobs.get(i); if (diffJob.getStatus() == QUEUED) { assertEquals(IN_PROGRESS, snapshotDiffReport.getJobStatus()); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index 443ba17d5968..4dc97d3a9010 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -708,7 +708,7 @@ private SnapshotDiffReportOzone getSnapshotDiffReportOnceComplete( while (true) { snapshotDiffResponse = objectStore.snapshotDiff(volume.getName(), bucket.getName(), - fromSnapshot, toSnapshot, token, -1, false); + fromSnapshot, toSnapshot, token, -1, false, false); if (snapshotDiffResponse.getJobStatus() == DONE) { break; } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 6c6be7368014..1ad2dc34fd44 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -1362,7 +1362,7 @@ private SnapshotDiffReportOzone getSnapshotDiffReportOnceComplete( while (true) { snapshotDiffResponse = objectStore.snapshotDiff(volume, bucket, fromSnapshot, toSnapshot, - token, -1, false); + token, -1, false, false); if (snapshotDiffResponse.getJobStatus() == DONE) { break; } diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml index a7e6d34a85ec..430cb03f6304 100644 --- a/hadoop-ozone/s3gateway/pom.xml +++ b/hadoop-ozone/s3gateway/pom.xml @@ -178,7 +178,6 @@ commons-lang3 - diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index e48735afb5a4..ea9d128877ae 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -638,7 +638,8 @@ public SnapshotDiffResponse snapshotDiff(String volumeName, String toSnapshot, String token, int pageSize, - boolean forceFullDiff) + boolean forceFullDiff, + boolean disableNativeDiff) throws IOException { return null; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java index 619af13a60a8..88792061abe4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java @@ -68,6 +68,12 @@ public class SnapshotDiffHandler extends Handler { defaultValue = "false") private boolean cancel; + @CommandLine.Option(names = {"--dnld", "--disable-native-libs-diff"}, + description = "perform diff of snapshot without using" + + " native libs (optional)", + hidden = true) + private boolean diffDisableNativeLibs; + @Override protected OzoneAddress getAddress() { return snapshotPath.getValue(); @@ -93,7 +99,7 @@ private void getSnapshotDiff(ObjectStore store, String volumeName, String bucketName) throws IOException { try (PrintStream stream = out()) { stream.print(store.snapshotDiff(volumeName, bucketName, fromSnapshot, - toSnapshot, token, pageSize, forceFullDiff)); + toSnapshot, token, pageSize, forceFullDiff, diffDisableNativeLibs)); } }