diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 4a109e8eaab4..b3c522eb61a7 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -191,7 +191,7 @@
import org.apache.hadoop.ozone.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
import org.apache.hadoop.ozone.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
import org.apache.hadoop.ozone.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
@@ -1228,8 +1228,8 @@ public SnapshotDiffResponse snapshotDiff(String volumeName,
OzoneManagerProtocolProtos.SnapshotDiffResponse diffResponse =
omResponse.getSnapshotDiffResponse();
- return new SnapshotDiffResponse(
- SnapshotDiffReport.fromProtobuf(diffResponse.getSnapshotDiffReport()),
+ return new SnapshotDiffResponse(SnapshotDiffReportOzone.fromProtobuf(
+ diffResponse.getSnapshotDiffReport()),
JobStatus.fromProtobuf(diffResponse.getJobStatus()),
diffResponse.getWaitTimeInMs());
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java
deleted file mode 100644
index 04732fa1a434..000000000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.snapshot;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffReportProto;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DiffReportEntryProto;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DiffReportEntryProto.DiffTypeProto;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Snapshot diff report.
- */
-public class SnapshotDiffReport {
-
- private static final String LINE_SEPARATOR = System.getProperty(
- "line.separator", "\n");
-
- /**
- * Types of the difference, which include CREATE, MODIFY, DELETE, and RENAME.
- * Each type has a label for representation:
- * + CREATE
- * M MODIFY
- * - DELETE
- * R RENAME
- */
- public enum DiffType {
- CREATE("+"),
- MODIFY("M"),
- DELETE("-"),
- RENAME("R");
-
- private final String label;
-
- DiffType(String label) {
- this.label = label;
- }
-
- public String getLabel() {
- return label;
- }
-
- public DiffTypeProto toProtobuf() {
- return DiffTypeProto.valueOf(this.name());
- }
-
- public static DiffType fromProtobuf(final DiffTypeProto type) {
- return DiffType.valueOf(type.name());
- }
- }
-
- /**
- * Snapshot diff report entry.
- */
- public static final class DiffReportEntry {
-
- /**
- * The type of diff.
- */
- private final DiffType type;
-
- /**
- * Source File/Object path.
- */
- private final String sourcePath;
-
- /**
- * Destination File/Object path, if this is a re-name operation.
- */
- private final String targetPath;
-
- private DiffReportEntry(final DiffType type, final String sourcePath,
- final String targetPath) {
- this.type = type;
- this.sourcePath = sourcePath;
- this.targetPath = targetPath;
- }
-
- public static DiffReportEntry of(final DiffType type,
- final String sourcePath) {
- return of(type, sourcePath, null);
- }
-
- public static DiffReportEntry of(final DiffType type,
- final String sourcePath,
- final String targetPath) {
- return new DiffReportEntry(type, sourcePath, targetPath);
-
- }
-
- @Override
- public String toString() {
- String str = type.getLabel() + "\t" + sourcePath;
- if (type == DiffType.RENAME) {
- str += " -> " + targetPath;
- }
- return str;
- }
-
- public DiffType getType() {
- return type;
- }
-
- @Override
- public boolean equals(Object other) {
- if (this == other) {
- return true;
- }
- if (other instanceof DiffReportEntry) {
- DiffReportEntry entry = (DiffReportEntry) other;
- return this.type.equals(entry.getType()) && this.sourcePath
- .equals(entry.sourcePath) && (this.targetPath != null ?
- this.targetPath.equals(entry.targetPath) : true);
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return toString().hashCode();
- }
-
- public DiffReportEntryProto toProtobuf() {
- final DiffReportEntryProto.Builder builder = DiffReportEntryProto
- .newBuilder();
- builder.setDiffType(type.toProtobuf()).setSourcePath(sourcePath);
- if (targetPath != null) {
- builder.setTargetPath(targetPath);
- }
- return builder.build();
- }
-
- public static DiffReportEntry fromProtobuf(
- final DiffReportEntryProto entry) {
- return of(DiffType.fromProtobuf(entry.getDiffType()),
- entry.getSourcePath(),
- entry.hasTargetPath() ? entry.getTargetPath() : null);
- }
-
- }
-
-
- /**
- * Volume name to which the snapshot bucket belongs.
- */
- private final String volumeName;
-
- /**
- * Bucket name to which the snapshot belongs.
- */
- private final String bucketName;
- /**
- * start point of the diff.
- */
- private final String fromSnapshot;
-
- /**
- * end point of the diff.
- */
- private final String toSnapshot;
-
- /**
- * list of diff.
- */
- private final List diffList;
-
- /**
- * subsequent token for the diff report.
- */
- private final String token;
-
- public SnapshotDiffReport(final String volumeName,
- final String bucketName,
- final String fromSnapshot,
- final String toSnapshot,
- final List entryList,
- final String token) {
- this.volumeName = volumeName;
- this.bucketName = bucketName;
- this.fromSnapshot = fromSnapshot;
- this.toSnapshot = toSnapshot;
- this.diffList = entryList != null ? entryList : Collections.emptyList();
- this.token = token;
- }
-
- public List getDiffList() {
- return diffList;
- }
-
- @Override
- public String toString() {
- StringBuilder str = new StringBuilder();
- str.append("Difference between snapshot: ")
- .append(fromSnapshot)
- .append(" and snapshot: ")
- .append(toSnapshot)
- .append(LINE_SEPARATOR);
- for (DiffReportEntry entry : diffList) {
- str.append(entry.toString()).append(LINE_SEPARATOR);
- }
- if (StringUtils.isNotEmpty(token)) {
- str.append("Next token: ")
- .append(token)
- .append(LINE_SEPARATOR);
- }
- return str.toString();
- }
-
- public SnapshotDiffReportProto toProtobuf() {
- final SnapshotDiffReportProto.Builder builder = SnapshotDiffReportProto
- .newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setFromSnapshot(fromSnapshot)
- .setToSnapshot(toSnapshot);
- builder.addAllDiffList(diffList.stream().map(DiffReportEntry::toProtobuf)
- .collect(Collectors.toList()));
- if (StringUtils.isNotEmpty(token)) {
- builder.setToken(token);
- }
- return builder.build();
- }
-
- public static SnapshotDiffReport fromProtobuf(
- final SnapshotDiffReportProto report) {
- return new SnapshotDiffReport(report.getVolumeName(),
- report.getBucketName(),
- report.getFromSnapshot(),
- report.getToSnapshot(),
- report.getDiffListList().stream()
- .map(DiffReportEntry::fromProtobuf)
- .collect(Collectors.toList()),
- report.getToken());
- }
-
-}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
new file mode 100644
index 000000000000..f52ca5c4889f
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.snapshot;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OFSPath;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffReportProto;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+
+/**
+ * Snapshot diff report.
+ */
+public class SnapshotDiffReportOzone
+ extends org.apache.hadoop.hdfs.protocol.SnapshotDiffReport {
+
+ private static final String LINE_SEPARATOR = System.getProperty(
+ "line.separator", "\n");
+
+ /**
+ * Volume name to which the snapshot bucket belongs.
+ */
+ private final String volumeName;
+
+ /**
+ * Bucket name to which the snapshot belongs.
+ */
+ private final String bucketName;
+
+
+ /**
+ * subsequent token for the diff report.
+ */
+ private final String token;
+
+
+ public SnapshotDiffReportOzone(final String snapshotRoot,
+ final String volumeName,
+ final String bucketName,
+ final String fromSnapshot,
+ final String toSnapshot,
+ final List entryList, final String token) {
+ super(snapshotRoot, fromSnapshot, toSnapshot, entryList);
+ this.volumeName = volumeName;
+ this.bucketName = bucketName;
+ this.token = token;
+ }
+
+ public List getDiffList() {
+ return super.getDiffList();
+ }
+
+ public String getToken() {
+ return token;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder str = new StringBuilder();
+ str.append("Difference between snapshot: ")
+ .append(getFromSnapshot())
+ .append(" and snapshot: ")
+ .append(getLaterSnapshotName())
+ .append(LINE_SEPARATOR);
+ for (DiffReportEntry entry : getDiffList()) {
+ str.append(entry.toString()).append(LINE_SEPARATOR);
+ }
+ if (StringUtils.isNotEmpty(token)) {
+ str.append("Next token: ")
+ .append(token)
+ .append(LINE_SEPARATOR);
+ }
+ return str.toString();
+ }
+
+ public SnapshotDiffReportProto toProtobuf() {
+ final SnapshotDiffReportProto.Builder builder = SnapshotDiffReportProto
+ .newBuilder();
+ builder.setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setFromSnapshot(getFromSnapshot())
+ .setToSnapshot(getLaterSnapshotName());
+ builder.addAllDiffList(getDiffList().stream().map(
+ SnapshotDiffReportOzone::toProtobufDiffReportEntry)
+ .collect(Collectors.toList()));
+ if (StringUtils.isNotEmpty(token)) {
+ builder.setToken(token);
+ }
+ return builder.build();
+ }
+
+ public static SnapshotDiffReportOzone fromProtobuf(
+ final SnapshotDiffReportProto report) {
+ Path bucketPath = new Path(
+ OZONE_URI_DELIMITER + report.getVolumeName()
+ + OZONE_URI_DELIMITER + report.getBucketName());
+ OFSPath path = new OFSPath(bucketPath, new OzoneConfiguration());
+ return new SnapshotDiffReportOzone(path.toString(),
+ report.getVolumeName(),
+ report.getBucketName(),
+ report.getFromSnapshot(),
+ report.getToSnapshot(),
+ report.getDiffListList().stream()
+ .map(SnapshotDiffReportOzone::fromProtobufDiffReportEntry)
+ .collect(Collectors.toList()),
+ report.getToken());
+ }
+
+ public static DiffType fromProtobufDiffType(
+ final OzoneManagerProtocolProtos.DiffReportEntryProto
+ .DiffTypeProto type) {
+ return DiffType.valueOf(type.name());
+ }
+
+ public static OzoneManagerProtocolProtos.DiffReportEntryProto
+ .DiffTypeProto toProtobufDiffType(DiffType type) {
+ return OzoneManagerProtocolProtos.DiffReportEntryProto
+ .DiffTypeProto.valueOf(type.name());
+ }
+
+ public static DiffReportEntry fromProtobufDiffReportEntry(
+ final OzoneManagerProtocolProtos.DiffReportEntryProto entry) {
+ if (entry == null) {
+ return null;
+ }
+ DiffType type = fromProtobufDiffType(entry.getDiffType());
+ return type == null ? null : new DiffReportEntry(type,
+ entry.getSourcePath().getBytes(StandardCharsets.UTF_8),
+ entry.hasTargetPath() ?
+ entry.getTargetPath().getBytes(StandardCharsets.UTF_8) : null);
+ }
+
+ public static OzoneManagerProtocolProtos
+ .DiffReportEntryProto toProtobufDiffReportEntry(DiffReportEntry entry) {
+ final OzoneManagerProtocolProtos.DiffReportEntryProto.Builder builder =
+ OzoneManagerProtocolProtos.DiffReportEntryProto.newBuilder();
+ builder.setDiffType(toProtobufDiffType(entry.getType())).setSourcePath(
+ new String(entry.getSourcePath(), StandardCharsets.UTF_8));
+ if (entry.getTargetPath() != null) {
+ String targetPath =
+ new String(entry.getTargetPath(), StandardCharsets.UTF_8);
+ builder.setTargetPath(targetPath);
+ }
+ return builder.build();
+ }
+
+
+ public static DiffReportEntry getDiffReportEntry(final DiffType type,
+ final String sourcePath) {
+ return getDiffReportEntry(type, sourcePath, null);
+ }
+
+ public static DiffReportEntry getDiffReportEntry(final DiffType type,
+ final String sourcePath, final String targetPath) {
+ return new DiffReportEntry(type,
+ sourcePath.getBytes(StandardCharsets.UTF_8),
+ targetPath != null ? targetPath.getBytes(StandardCharsets.UTF_8) :
+ null);
+ }
+
+ /**
+ * @param diffReport
+ * Add the diffReportEntries from given diffReport to the report.
+ * Used while aggregating paged response of snapdiff.
+ */
+ public void aggregate(SnapshotDiffReportOzone diffReport) {
+ this.getDiffList().addAll(diffReport.getDiffList());
+ }
+
+
+}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java
index 8175ae86ebda..6cd9bff03922 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java
@@ -43,11 +43,11 @@ public static JobStatus fromProtobuf(JobStatusProto jobStatusProto) {
}
}
- private final SnapshotDiffReport snapshotDiffReport;
+ private final SnapshotDiffReportOzone snapshotDiffReport;
private final JobStatus jobStatus;
private final long waitTimeInMs;
- public SnapshotDiffResponse(final SnapshotDiffReport snapshotDiffReport,
+ public SnapshotDiffResponse(final SnapshotDiffReportOzone snapshotDiffReport,
final JobStatus jobStatus,
final long waitTimeInMs) {
this.snapshotDiffReport = snapshotDiffReport;
@@ -55,7 +55,7 @@ public SnapshotDiffResponse(final SnapshotDiffReport snapshotDiffReport,
this.waitTimeInMs = waitTimeInMs;
}
- public SnapshotDiffReport getSnapshotDiffReport() {
+ public SnapshotDiffReportOzone getSnapshotDiffReport() {
return snapshotDiffReport;
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index d1a556f7f9bd..56f6db0e6eec 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.ozone.OzoneAcl;
@@ -85,6 +86,8 @@
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
@@ -113,6 +116,7 @@
import static org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec.RS;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPDIFF_MAX_PAGE_SIZE;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_OFS_SHARED_TMP_DIR;
@@ -273,6 +277,7 @@ public static void initClusterAndEnv() throws IOException,
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
// Set the number of keys to be processed during batch operate.
conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5);
+ conf.setInt(OZONE_OM_SNAPDIFF_MAX_PAGE_SIZE, 4);
// fs.ofs.impl would be loaded from META-INF, no need to manually set it
fs = FileSystem.get(conf);
trash = new Trash(conf);
@@ -2261,4 +2266,64 @@ public void testFileSystemDeclaresCapability() throws Throwable {
assertHasPathCapabilities(fs, getBucketPath(), FS_ACLS);
assertHasPathCapabilities(fs, getBucketPath(), FS_CHECKSUMS);
}
+
+
+ @Test
+ public void testSnapshotDiff() throws Exception {
+ OzoneBucket bucket1 =
+ TestDataUtil.createVolumeAndBucket(client, bucketLayout);
+ Path volumePath1 = new Path(OZONE_URI_DELIMITER, bucket1.getVolumeName());
+ Path bucketPath1 = new Path(volumePath1, bucket1.getName());
+ Path snap1 = fs.createSnapshot(bucketPath1);
+ Path file1 = new Path(bucketPath1, "key1");
+ Path file2 = new Path(bucketPath1, "key2");
+ ContractTestUtils.touch(fs, file1);
+ ContractTestUtils.touch(fs, file2);
+ Path snap2 = fs.createSnapshot(bucketPath1);
+ java.nio.file.Path fromSnapPath = Paths.get(snap1.toString()).getFileName();
+ java.nio.file.Path toSnapPath = Paths.get(snap2.toString()).getFileName();
+ String fromSnap = fromSnapPath != null ? fromSnapPath.toString() : null;
+ String toSnap = toSnapPath != null ? toSnapPath.toString() : null;
+ SnapshotDiffReport diff =
+ ofs.getSnapshotDiffReport(bucketPath1, fromSnap, toSnap);
+ Assert.assertEquals(2, diff.getDiffList().size());
+ Assert.assertEquals(SnapshotDiffReport.DiffType.CREATE,
+ diff.getDiffList().get(0).getType());
+ Assert.assertEquals(SnapshotDiffReport.DiffType.CREATE,
+ diff.getDiffList().get(1).getType());
+ Assert.assertArrayEquals("key1".getBytes(StandardCharsets.UTF_8),
+ diff.getDiffList().get(0).getSourcePath());
+ Assert.assertArrayEquals("key2".getBytes(StandardCharsets.UTF_8),
+ diff.getDiffList().get(1).getSourcePath());
+
+ // test whether snapdiff returns aggregated response as
+ // page size is 4.
+ for (int fileCount = 0; fileCount < 10; fileCount++) {
+ Path file =
+ new Path(bucketPath1, "key" + RandomStringUtils.randomAlphabetic(5));
+ ContractTestUtils.touch(fs, file);
+ }
+ Path snap3 = fs.createSnapshot(bucketPath1);
+ fromSnapPath = toSnapPath;
+ toSnapPath = Paths.get(snap3.toString()).getFileName();
+ fromSnap = fromSnapPath != null ? fromSnapPath.toString() : null;
+ toSnap = toSnapPath != null ? toSnapPath.toString() : null;
+ diff = ofs.getSnapshotDiffReport(bucketPath1, fromSnap, toSnap);
+ Assert.assertEquals(10, diff.getDiffList().size());
+
+ Path file =
+ new Path(bucketPath1, "key" + RandomStringUtils.randomAlphabetic(5));
+ ContractTestUtils.touch(fs, file);
+ diff = ofs.getSnapshotDiffReport(bucketPath1, toSnap, ".");
+ Assert.assertEquals(1, diff.getDiffList().size());
+
+
+ // try snapDiff between non-bucket paths
+ String errorMsg = "Path is not a bucket";
+ String finalFromSnap = fromSnap;
+ String finalToSnap = toSnap;
+ LambdaTestUtils.intercept(IllegalArgumentException.class, errorMsg,
+ () -> ofs.getSnapshotDiffReport(volumePath1, finalFromSnap,
+ finalToSnap));
+ }
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index ea4708232666..c70fa0562fe9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -44,7 +44,7 @@
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -498,7 +498,8 @@ public void testSnapDiff() throws Exception {
String snap2 = "snap" + RandomStringUtils.randomNumeric(5);
createSnapshot(volume, bucket, snap2);
- SnapshotDiffReport diff1 = getSnapDiffReport(volume, bucket, snap1, snap2);
+ SnapshotDiffReportOzone
+ diff1 = getSnapDiffReport(volume, bucket, snap1, snap2);
Assert.assertTrue(diff1.getDiffList().isEmpty());
// Create Key2 and delete Key1, take snapshot
String key2 = "key-2-";
@@ -508,14 +509,15 @@ public void testSnapDiff() throws Exception {
createSnapshot(volume, bucket, snap3);
// Diff should have 2 entries
- SnapshotDiffReport diff2 = getSnapDiffReport(volume, bucket, snap2, snap3);
+ SnapshotDiffReportOzone
+ diff2 = getSnapDiffReport(volume, bucket, snap2, snap3);
Assert.assertEquals(2, diff2.getDiffList().size());
Assert.assertTrue(diff2.getDiffList().contains(
- SnapshotDiffReport.DiffReportEntry
- .of(SnapshotDiffReport.DiffType.CREATE, key2)));
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReportOzone.DiffType.CREATE, key2)));
Assert.assertTrue(diff2.getDiffList().contains(
- SnapshotDiffReport.DiffReportEntry
- .of(SnapshotDiffReport.DiffType.DELETE, key1)));
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReportOzone.DiffType.DELETE, key1)));
// Rename Key2
String key2Renamed = key2 + "_renamed";
@@ -523,11 +525,12 @@ public void testSnapDiff() throws Exception {
String snap4 = "snap" + RandomStringUtils.randomNumeric(5);
createSnapshot(volume, bucket, snap4);
- SnapshotDiffReport diff3 = getSnapDiffReport(volume, bucket, snap3, snap4);
+ SnapshotDiffReportOzone
+ diff3 = getSnapDiffReport(volume, bucket, snap3, snap4);
Assert.assertEquals(1, diff3.getDiffList().size());
Assert.assertTrue(diff3.getDiffList().contains(
- SnapshotDiffReport.DiffReportEntry
- .of(SnapshotDiffReport.DiffType.RENAME, key2, key2Renamed)));
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReportOzone.DiffType.RENAME, key2, key2Renamed)));
// Create a directory
@@ -535,7 +538,8 @@ public void testSnapDiff() throws Exception {
bucket1.createDirectory(dir1);
String snap5 = "snap" + RandomStringUtils.randomNumeric(5);
createSnapshot(volume, bucket, snap5);
- SnapshotDiffReport diff4 = getSnapDiffReport(volume, bucket, snap4, snap5);
+ SnapshotDiffReportOzone
+ diff4 = getSnapDiffReport(volume, bucket, snap4, snap5);
Assert.assertEquals(1, diff4.getDiffList().size());
// for non-fso, directories are a special type of key with "/" appended
// at the end.
@@ -543,12 +547,12 @@ public void testSnapDiff() throws Exception {
dir1 = dir1 + OM_KEY_PREFIX;
}
Assert.assertTrue(diff4.getDiffList().contains(
- SnapshotDiffReport.DiffReportEntry
- .of(SnapshotDiffReport.DiffType.CREATE, dir1)));
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReportOzone.DiffType.CREATE, dir1)));
}
- private SnapshotDiffReport getSnapDiffReport(String volume,
+ private SnapshotDiffReportOzone getSnapDiffReport(String volume,
String bucket,
String fromSnapshot,
String toSnapshot)
@@ -683,7 +687,7 @@ public void testSnapDiffMultipleBuckets() throws Exception {
createFileKey(bucket2, key1);
String snap2 = "snap" + RandomStringUtils.randomNumeric(5);
createSnapshot(volume, bucketName1, snap2);
- SnapshotDiffReport diff1 =
+ SnapshotDiffReportOzone diff1 =
getSnapDiffReport(volume, bucketName1, snap1, snap2);
Assert.assertEquals(1, diff1.getDiffList().size());
}
@@ -730,7 +734,7 @@ public void testSnapDiffWithMultipleSSTs()
String snap2 = "snap" + RandomStringUtils.randomNumeric(5);
createSnapshot(volumeName1, bucketName1, snap2); // 1.sst 2.sst 3.sst 4.sst
Assert.assertEquals(4, getKeyTableSstFiles().size());
- SnapshotDiffReport diff1 =
+ SnapshotDiffReportOzone diff1 =
store.snapshotDiff(volumeName1, bucketName1, snap1, snap2,
null, 0, false)
.getSnapshotDiffReport();
diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmDBDiffReportEntryCodec.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmDBDiffReportEntryCodec.java
index 06ac8452a3a1..455a730f8014 100644
--- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmDBDiffReportEntryCodec.java
+++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmDBDiffReportEntryCodec.java
@@ -20,33 +20,41 @@
import java.io.IOException;
import org.apache.hadoop.hdds.utils.db.Codec;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import static com.google.common.base.Preconditions.checkNotNull;
/**
* Codec to encode DiffReportEntry as byte array.
*/
-public class OmDBDiffReportEntryCodec implements Codec {
+public class OmDBDiffReportEntryCodec implements
+ Codec {
@Override
- public byte[] toPersistedFormat(DiffReportEntry object)
+ public byte[] toPersistedFormat(
+ org.apache.hadoop.hdfs.protocol
+ .SnapshotDiffReport.DiffReportEntry object)
throws IOException {
checkNotNull(object, "Null object can't be converted to byte array.");
- return object.toProtobuf().toByteArray();
+ return SnapshotDiffReportOzone.toProtobufDiffReportEntry(object)
+ .toByteArray();
}
@Override
- public DiffReportEntry fromPersistedFormat(byte[] rawData)
- throws IOException {
- checkNotNull(rawData, "Null byte array can't be converted to " +
- "real object.");
- return DiffReportEntry.fromProtobuf(
+ public org.apache.hadoop.hdfs.protocol
+ .SnapshotDiffReport.DiffReportEntry fromPersistedFormat(
+ byte[] rawData) throws IOException {
+ checkNotNull(rawData,
+ "Null byte array can't be converted to " + "real object.");
+ return SnapshotDiffReportOzone.fromProtobufDiffReportEntry(
OzoneManagerProtocolProtos.DiffReportEntryProto.parseFrom(rawData));
}
@Override
- public DiffReportEntry copyObject(DiffReportEntry object) {
+ public org.apache.hadoop.hdfs.protocol
+ .SnapshotDiffReport.DiffReportEntry copyObject(
+ org.apache.hadoop.hdfs.protocol
+ .SnapshotDiffReport.DiffReportEntry object) {
// Note: Not really a "copy". from OmDBDiffReportEntryCodec
return object;
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index a41d8ecd076e..a97ed425b532 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -23,6 +23,7 @@
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.cache.RemovalListener;
+
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
@@ -62,7 +63,7 @@
import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffJob;
import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager;
import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer;
import org.rocksdb.ColumnFamilyDescriptor;
@@ -76,6 +77,8 @@
import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.hadoop.hdds.utils.db.DBStoreBuilder.DEFAULT_COLUMN_FAMILY_NAME;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPDIFF_MAX_PAGE_SIZE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OM_SNAPDIFF_MAX_PAGE_SIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DB_NAME;
@@ -147,9 +150,6 @@ public final class OmSnapshotManager implements AutoCloseable {
private final CodecRegistry codecRegistry;
private final SnapshotDiffCleanupService snapshotDiffCleanupService;
- // TODO: [SNAPSHOT] create config for max allowed page size.
- private final int maxPageSize = 1000;
-
public OmSnapshotManager(OzoneManager ozoneManager) {
this.options = new ManagedDBOptions();
this.options.setCreateIfMissing(true);
@@ -318,7 +318,7 @@ private CodecRegistry createCodecRegistryForSnapDiff() {
// Integers are used for indexing persistent list.
registry.addCodec(Integer.class, new IntegerCodec());
// DiffReportEntry codec for Diff Report.
- registry.addCodec(SnapshotDiffReport.DiffReportEntry.class,
+ registry.addCodec(SnapshotDiffReportOzone.DiffReportEntry.class,
new OmDBDiffReportEntryCodec());
registry.addCodec(SnapshotDiffJob.class,
new SnapshotDiffJob.SnapshotDiffJobCodec());
@@ -548,6 +548,9 @@ public SnapshotDiffResponse getSnapshotDiffReport(final String volume,
verifySnapshotInfoForSnapDiff(fsInfo, tsInfo);
int index = getIndexFromToken(token);
+ int maxPageSize = ozoneManager.getConfiguration()
+ .getInt(OZONE_OM_SNAPDIFF_MAX_PAGE_SIZE,
+ OZONE_OM_SNAPDIFF_MAX_PAGE_SIZE_DEFAULT);
if (pageSize <= 0 || pageSize > maxPageSize) {
pageSize = maxPageSize;
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
index 7c5a8e355827..ca6c49d3ac2b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
@@ -45,9 +45,12 @@
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
+
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.utils.db.CodecRegistry;
import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions;
@@ -62,12 +65,10 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.WithObjectID;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport.DiffReportEntry;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.util.ClosableIterator;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus;
-import org.apache.hadoop.util.ClosableIterator;
import org.apache.ozone.rocksdb.util.ManagedSstFileReader;
import org.apache.ozone.rocksdb.util.RdbUtil;
import org.apache.ozone.rocksdiff.DifferSnapshotInfo;
@@ -85,6 +86,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER;
import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.dropColumnFamilyHandle;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.getSnapshotInfo;
import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.FAILED;
@@ -92,6 +94,9 @@
import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.QUEUED;
import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.REJECTED;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+
/**
* Class to generate snapshot diff.
*/
@@ -282,21 +287,26 @@ public SnapshotDiffResponse getSnapshotDiffReport(
volume, bucket, fromSnapshot.getName(), toSnapshot.getName(),
forceFullDiff);
+ OFSPath snapshotRoot = getSnapshotRootPath(volume, bucket);
+
switch (snapDiffJob.getStatus()) {
case QUEUED:
return submitSnapDiffJob(snapDiffJobKey, volume, bucket, fromSnapshot,
toSnapshot, fsInfo, tsInfo, index, pageSize, forceFullDiff);
case IN_PROGRESS:
- return new SnapshotDiffResponse(new SnapshotDiffReport(volume, bucket,
- fromSnapshot.getName(), toSnapshot.getName(), new ArrayList<>(),
- null), IN_PROGRESS, DEFAULT_WAIT_TIME.toMillis());
+ return new SnapshotDiffResponse(
+ new SnapshotDiffReportOzone(snapshotRoot.toString(), volume, bucket,
+ fromSnapshot.getName(), toSnapshot.getName(), new ArrayList<>(),
+ null), IN_PROGRESS, DEFAULT_WAIT_TIME.toMillis());
case FAILED:
- return new SnapshotDiffResponse(new SnapshotDiffReport(volume, bucket,
- fromSnapshot.getName(), toSnapshot.getName(), new ArrayList<>(),
- null), FAILED, DEFAULT_WAIT_TIME.toMillis());
+ return new SnapshotDiffResponse(
+ new SnapshotDiffReportOzone(snapshotRoot.toString(), volume, bucket,
+ fromSnapshot.getName(), toSnapshot.getName(), new ArrayList<>(),
+ null), FAILED, DEFAULT_WAIT_TIME.toMillis());
case DONE:
- SnapshotDiffReport report = createPageResponse(snapDiffJob.getJobId(),
- volume, bucket, fromSnapshot, toSnapshot, index, pageSize);
+ SnapshotDiffReportOzone report =
+ createPageResponse(snapDiffJob.getJobId(), volume, bucket,
+ fromSnapshot, toSnapshot, index, pageSize);
return new SnapshotDiffResponse(report, DONE, 0L);
default:
throw new IllegalStateException("Unknown snapshot job status: " +
@@ -304,7 +314,15 @@ public SnapshotDiffResponse getSnapshotDiffReport(
}
}
- private SnapshotDiffReport createPageResponse(final String jobId,
+ @NotNull
+ private static OFSPath getSnapshotRootPath(String volume, String bucket) {
+ Path bucketPath = new Path(
+ OZONE_URI_DELIMITER + volume + OZONE_URI_DELIMITER + bucket);
+ OFSPath path = new OFSPath(bucketPath, new OzoneConfiguration());
+ return path;
+ }
+
+ private SnapshotDiffReportOzone createPageResponse(final String jobId,
final String volume,
final String bucket,
final OmSnapshot fromSnapshot,
@@ -314,6 +332,8 @@ private SnapshotDiffReport createPageResponse(final String jobId,
throws IOException {
List diffReportList = new ArrayList<>();
+ OFSPath path = getSnapshotRootPath(volume, bucket);
+
boolean hasMoreEntries = true;
for (int idx = index; idx - index < pageSize; idx++) {
@@ -329,8 +349,9 @@ private SnapshotDiffReport createPageResponse(final String jobId,
String tokenString = hasMoreEntries ?
String.valueOf(index + pageSize) : null;
- return new SnapshotDiffReport(volume, bucket, fromSnapshot.getName(),
- toSnapshot.getName(), diffReportList, tokenString);
+ return new SnapshotDiffReportOzone(path.toString(), volume, bucket,
+ fromSnapshot.getName(), toSnapshot.getName(), diffReportList,
+ tokenString);
}
@SuppressWarnings("parameternumber")
@@ -349,6 +370,8 @@ private synchronized SnapshotDiffResponse submitSnapDiffJob(
SnapshotDiffJob snapDiffJob = snapDiffJobTable.get(jobKey);
+ OFSPath snapshotRoot = getSnapshotRootPath(volume, bucket);
+
// This is only possible if another thread tired to submit the request,
// and it got rejected. In this scenario, return the Rejected job status
// with wait time.
@@ -356,9 +379,10 @@ private synchronized SnapshotDiffResponse submitSnapDiffJob(
LOG.info("Snap diff job has been removed for for volume: {}, " +
"bucket: {}, fromSnapshot: {} and toSnapshot: {}.",
volume, bucket, fromSnapshot.getName(), toSnapshot.getName());
- return new SnapshotDiffResponse(new SnapshotDiffReport(volume, bucket,
- fromSnapshot.getName(), toSnapshot.getName(), new ArrayList<>(),
- null), REJECTED, DEFAULT_WAIT_TIME.toMillis());
+ return new SnapshotDiffResponse(
+ new SnapshotDiffReportOzone(snapshotRoot.toString(),
+ volume, bucket, fromSnapshot.getName(), toSnapshot.getName(),
+ new ArrayList<>(), null), REJECTED, DEFAULT_WAIT_TIME.toMillis());
}
// Check again that request is still in queued status. If it is not queued,
@@ -366,14 +390,16 @@ private synchronized SnapshotDiffResponse submitSnapDiffJob(
if (snapDiffJob.getStatus() != QUEUED) {
// Same request is submitted by another thread and already completed.
if (snapDiffJob.getStatus() == DONE) {
- SnapshotDiffReport report = createPageResponse(snapDiffJob.getJobId(),
- volume, bucket, fromSnapshot, toSnapshot, index, pageSize);
+ SnapshotDiffReportOzone report =
+ createPageResponse(snapDiffJob.getJobId(), volume, bucket,
+ fromSnapshot, toSnapshot, index, pageSize);
return new SnapshotDiffResponse(report, DONE, 0L);
} else {
// Otherwise, return the same status as in DB with wait time.
- return new SnapshotDiffResponse(new SnapshotDiffReport(volume, bucket,
- fromSnapshot.getName(), toSnapshot.getName(), new ArrayList<>(),
- null), snapDiffJob.getStatus(), DEFAULT_WAIT_TIME.toMillis());
+ return new SnapshotDiffResponse(
+ new SnapshotDiffReportOzone(snapshotRoot.toString(), volume, bucket,
+ fromSnapshot.getName(), toSnapshot.getName(), new ArrayList<>(),
+ null), snapDiffJob.getStatus(), DEFAULT_WAIT_TIME.toMillis());
}
}
@@ -398,6 +424,8 @@ private synchronized SnapshotDiffResponse submitSnapDiffJob(
" volume: {}, bucket: {}, fromSnapshot: {} and toSnapshot: {}",
volume, bucket, fromSnapshot.getName(), toSnapshot.getName());
+ OFSPath snapshotRoot = getSnapshotRootPath(volume, bucket);
+
// Submit the request to the executor if job is still in queued status.
// If executor cannot take any more job, remove the job form DB and return
// the Rejected Job status with wait time.
@@ -406,10 +434,10 @@ private synchronized SnapshotDiffResponse submitSnapDiffJob(
volume, bucket, fromSnapshot, toSnapshot, fsInfo, tsInfo,
forceFullDiff));
updateJobStatus(jobKey, QUEUED, IN_PROGRESS);
- return new SnapshotDiffResponse(new SnapshotDiffReport(volume, bucket,
- fromSnapshot.getName(), toSnapshot.getName(), new ArrayList<>(),
- null),
- IN_PROGRESS, DEFAULT_WAIT_TIME.toMillis());
+ return new SnapshotDiffResponse(
+ new SnapshotDiffReportOzone(snapshotRoot.toString(), volume, bucket,
+ fromSnapshot.getName(), toSnapshot.getName(), new ArrayList<>(),
+ null), IN_PROGRESS, DEFAULT_WAIT_TIME.toMillis());
} catch (RejectedExecutionException exception) {
// Remove the entry from job table so that client can retry.
// If entry is not removed, client has to wait till cleanup service
@@ -418,9 +446,10 @@ private synchronized SnapshotDiffResponse submitSnapDiffJob(
snapDiffJobTable.remove(jobKey);
LOG.info("Exceeded the snapDiff parallel requests progressing " +
"limit. Please retry after {}.", DEFAULT_WAIT_TIME);
- return new SnapshotDiffResponse(new SnapshotDiffReport(volume, bucket,
- fromSnapshot.getName(), toSnapshot.getName(), new ArrayList<>(),
- null), REJECTED, DEFAULT_WAIT_TIME.toMillis());
+ return new SnapshotDiffResponse(
+ new SnapshotDiffReportOzone(snapshotRoot.toString(), volume, bucket,
+ fromSnapshot.getName(), toSnapshot.getName(), new ArrayList<>(),
+ null), REJECTED, DEFAULT_WAIT_TIME.toMillis());
}
}
@@ -781,21 +810,28 @@ private long generateDiffReport(
"Old and new key name both are null");
} else if (oldKeyName == null) { // Key Created.
String key = codecRegistry.asObject(newKeyName, String.class);
- DiffReportEntry entry = DiffReportEntry.of(DiffType.CREATE, key);
+ DiffReportEntry entry =
+ SnapshotDiffReportOzone.getDiffReportEntry(DiffType.CREATE,
+ key);
createDiffs.add(codecRegistry.asRawData(entry));
} else if (newKeyName == null) { // Key Deleted.
String key = codecRegistry.asObject(oldKeyName, String.class);
- DiffReportEntry entry = DiffReportEntry.of(DiffType.DELETE, key);
+ DiffReportEntry entry =
+ SnapshotDiffReportOzone.getDiffReportEntry(DiffType.DELETE,
+ key);
deleteDiffs.add(codecRegistry.asRawData(entry));
} else if (Arrays.equals(oldKeyName, newKeyName)) { // Key modified.
String key = codecRegistry.asObject(newKeyName, String.class);
- DiffReportEntry entry = DiffReportEntry.of(DiffType.MODIFY, key);
+ DiffReportEntry entry =
+ SnapshotDiffReportOzone.getDiffReportEntry(DiffType.MODIFY,
+ key);
modifyDiffs.add(codecRegistry.asRawData(entry));
} else { // Key Renamed.
String oldKey = codecRegistry.asObject(oldKeyName, String.class);
String newKey = codecRegistry.asObject(newKeyName, String.class);
renameDiffs.add(codecRegistry.asRawData(
- DiffReportEntry.of(DiffType.RENAME, oldKey, newKey)));
+ SnapshotDiffReportOzone.getDiffReportEntry(DiffType.RENAME,
+ oldKey, newKey)));
}
}
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java
index f0a1cfe59b39..6af8b0733ebd 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java
@@ -36,7 +36,7 @@
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.codec.OmDBDiffReportEntryCodec;
import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffJob;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterAll;
@@ -148,7 +148,7 @@ public void init() throws RocksDBException, IOException {
// Integers are used for indexing persistent list.
codecRegistry.addCodec(Integer.class, new IntegerCodec());
// DiffReportEntry codec for Diff Report.
- codecRegistry.addCodec(SnapshotDiffReport.DiffReportEntry.class,
+ codecRegistry.addCodec(SnapshotDiffReportOzone.DiffReportEntry.class,
new OmDBDiffReportEntryCodec());
codecRegistry.addCodec(SnapshotDiffJob.class,
new SnapshotDiffJob.SnapshotDiffJobCodec());
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
index f36dad9a9603..3eadb44f0b24 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -44,6 +44,7 @@
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.ozone.OmUtils;
@@ -62,15 +63,20 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
-import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
+import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -85,6 +91,7 @@ public class BasicOzoneClientAdapterImpl implements OzoneClientAdapter {
static final Logger LOG =
LoggerFactory.getLogger(BasicOzoneClientAdapterImpl.class);
+ public static final String ACTIVE_FS_SNAPSHOT_NAME = ".";
private OzoneClient ozoneClient;
private ObjectStore objectStore;
@@ -646,4 +653,56 @@ public String createSnapshot(String pathStr, String snapshotName)
ofsPath.getBucketName(),
snapshotName);
}
+
+ @Override
+ public SnapshotDiffReport getSnapshotDiffReport(Path snapshotDir,
+ String fromSnapshot, String toSnapshot)
+ throws IOException, InterruptedException {
+ boolean takeTemporarySnapshot = false;
+ if (toSnapshot.equals(ACTIVE_FS_SNAPSHOT_NAME)) {
+ takeTemporarySnapshot = true;
+ toSnapshot = createSnapshot(snapshotDir.toString(),
+ "temp" + SnapshotInfo.generateName(Time.now()));
+ }
+ try {
+ SnapshotDiffReportOzone aggregated;
+ SnapshotDiffReportOzone report =
+ getSnapshotDiffReportOnceComplete(fromSnapshot, toSnapshot, "");
+ aggregated = report;
+ while (!report.getToken().isEmpty()) {
+ LOG.info(
+ "Total Snapshot Diff length between snapshot {} and {} exceeds"
+ + " max page size, Performing another " +
+ "snapdiff with index at {}",
+ fromSnapshot, toSnapshot, report.getToken());
+ report = getSnapshotDiffReportOnceComplete(fromSnapshot, toSnapshot,
+ report.getToken());
+ aggregated.aggregate(report);
+ }
+ return aggregated;
+ } finally {
+ // delete the temp snapshot
+ if (takeTemporarySnapshot) {
+ OFSPath snapPath = new OFSPath(snapshotDir.toString(), config);
+ ozoneClient.getObjectStore()
+ .deleteSnapshot(snapPath.getVolumeName(), snapPath.getBucketName(),
+ toSnapshot);
+ }
+ }
+ }
+
+ private SnapshotDiffReportOzone getSnapshotDiffReportOnceComplete(
+ String fromSnapshot, String toSnapshot, String token)
+ throws IOException, InterruptedException {
+ SnapshotDiffResponse snapshotDiffResponse = null;
+ do {
+ snapshotDiffResponse =
+ objectStore.snapshotDiff(volume.getName(), bucket.getName(),
+ fromSnapshot, toSnapshot, token, -1, false);
+ Thread.sleep(snapshotDiffResponse.getWaitTimeInMs());
+ } while (snapshotDiffResponse.getJobStatus() != DONE);
+ SnapshotDiffReportOzone report =
+ snapshotDiffResponse.getSnapshotDiffReport();
+ return report;
+ }
}
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 5c953f56b13c..be0f46bfc250 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -51,6 +51,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.ozone.OmUtils;
@@ -73,12 +74,16 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -88,6 +93,7 @@
.BUCKET_ALREADY_EXISTS;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
.VOLUME_ALREADY_EXISTS;
+import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
/**
* Basic Implementation of the RootedOzoneFileSystem calls.
@@ -111,6 +117,8 @@ public class BasicRootedOzoneClientAdapterImpl
private BucketLayout defaultOFSBucketLayout;
private OzoneConfiguration config;
+ public static final String ACTIVE_FS_SNAPSHOT_NAME = ".";
+
/**
* Create new OzoneClientAdapter implementation.
*
@@ -1283,6 +1291,62 @@ public String createSnapshot(String pathStr, String snapshotName)
snapshotName);
}
+ @Override
+ public SnapshotDiffReport getSnapshotDiffReport(Path snapshotDir,
+ String fromSnapshot, String toSnapshot)
+ throws IOException, InterruptedException {
+ boolean takeTemporarySnapshot = false;
+ if (toSnapshot.equals(ACTIVE_FS_SNAPSHOT_NAME)) {
+ takeTemporarySnapshot = true;
+ toSnapshot = createSnapshot(snapshotDir.toString(),
+ "temp" + SnapshotInfo.generateName(Time.now()));
+ }
+ OFSPath ofsPath = new OFSPath(snapshotDir, config);
+ String volume = ofsPath.getVolumeName();
+ String bucket = ofsPath.getBucketName();
+ try {
+ SnapshotDiffReportOzone aggregated;
+ SnapshotDiffReportOzone report =
+ getSnapshotDiffReportOnceComplete(fromSnapshot, toSnapshot, volume,
+ bucket, "");
+ aggregated = report;
+ while (!report.getToken().isEmpty()) {
+ LOG.info(
+ "Total Snapshot Diff length between snapshot {} and {} exceeds"
+ + " max page size, Performing another" +
+ " snapdiff with index at {}",
+ fromSnapshot, toSnapshot, report.getToken());
+ report =
+ getSnapshotDiffReportOnceComplete(fromSnapshot, toSnapshot, volume,
+ bucket, report.getToken());
+ aggregated.aggregate(report);
+ }
+ return aggregated;
+ } finally {
+ // delete the temp snapshot
+ if (takeTemporarySnapshot) {
+ ozoneClient.getObjectStore()
+ .deleteSnapshot(ofsPath.getVolumeName(), ofsPath.getBucketName(),
+ toSnapshot);
+ }
+ }
+ }
+
+ private SnapshotDiffReportOzone getSnapshotDiffReportOnceComplete(
+ String fromSnapshot, String toSnapshot, String volume, String bucket,
+ String token) throws IOException, InterruptedException {
+ SnapshotDiffResponse snapshotDiffResponse = null;
+ do {
+ snapshotDiffResponse =
+ objectStore.snapshotDiff(volume, bucket, fromSnapshot, toSnapshot,
+ token, -1, false);
+ Thread.sleep(snapshotDiffResponse.getWaitTimeInMs());
+ } while (snapshotDiffResponse.getJobStatus() != DONE);
+ SnapshotDiffReportOzone report =
+ snapshotDiffResponse.getSnapshotDiffReport();
+ return report;
+ }
+
public boolean recoverLease(final Path f) throws IOException {
OFSPath ofsPath = new OFSPath(f, config);
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 3c3610db02b2..9a21be4ed84e 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -41,6 +41,7 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.ozone.OFSPath;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -1427,6 +1428,20 @@ public ContentSummary getContentSummary(Path f) throws IOException {
spaceConsumed(summary[1]).build();
}
+ public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir,
+ final String fromSnapshot, final String toSnapshot)
+ throws IOException, InterruptedException {
+ OFSPath ofsPath =
+ new OFSPath(snapshotDir, OzoneConfiguration.of(getConf()));
+ Preconditions.checkArgument(ofsPath.isBucket(),
+ "Unsupported : Path is not a bucket");
+ // TODO:HDDS-7681 support snapdiff when toSnapshot="." referring to
+ // current state of the bucket, This can be achieved by calling
+ // createSnapshot and then doing the diff.
+ return adapter.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot);
+ }
+
+
/**
* Start the lease recovery of a file.
*
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
index 0b4ff1546920..50fe9a71c105 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.token.Token;
@@ -86,4 +87,8 @@ FileStatusAdapter getFileStatus(String key, URI uri,
FileChecksum getFileChecksum(String keyName, long length) throws IOException;
String createSnapshot(String pathStr, String snapshotName) throws IOException;
+
+ SnapshotDiffReport getSnapshotDiffReport(Path snapshotDir,
+ String fromSnapshot, String toSnapshot)
+ throws IOException, InterruptedException;
}