HDDS-7916. [Snapshot] Support for Distcp with Ozone Snapshots.

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/PersistentList.java → hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PersistentList.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone.om.snapshot;
+package org.apache.hadoop.ozone.util;
 
 import java.util.Iterator;
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java

@@ -671,6 +671,18 @@ public void writeLock() {
     this.lock.writeLock().lock();
   }
 
+  @Override
+  public void readUnlock(String opName) {
+    // not used anywhere, implemented as part of interface.
+    this.lock.writeLock().lock();
+  }
+
+  @Override
+  public void writeUnlock(String opName) {
+    // not used anywhere, implemented as part of interface.
+    this.lock.writeLock().unlock();
+  }
+
   /**
    * Release write lock.
    */

hadoop-ozone/common/pom.xml

@@ -107,6 +107,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-params</artifactId>
     </dependency>
+      <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-hdfs-client</artifactId>
+      </dependency>
   </dependencies>
 
   <build>

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java

@@ -18,216 +18,111 @@
 
 package org.apache.hadoop.ozone.snapshot;
 
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffReportProto;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DiffReportEntryProto;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DiffReportEntryProto.DiffTypeProto;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OFSPath;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.util.PersistentList;
 
-import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+
 /**
- * Snapshot diff report.
+ * Snapshot Diff Report.
  */
-public class SnapshotDiffReport {
-
-  private static final String LINE_SEPARATOR = System.getProperty(
-      "line.separator", "\n");
-
-  /**
-   * Types of the difference, which include CREATE, MODIFY, DELETE, and RENAME.
-   * Each type has a label for representation:
-   * +  CREATE
-   * M  MODIFY
-   * -  DELETE
-   * R  RENAME
-   */
-  public enum DiffType {
-    CREATE("+"),
-    MODIFY("M"),
-    DELETE("-"),
-    RENAME("R");
-
-    private final String label;
-
-    DiffType(String label) {
-      this.label = label;
-    }
-
-    public String getLabel() {
-      return label;
-    }
-
-    public DiffTypeProto toProtobuf() {
-      return DiffTypeProto.valueOf(this.name());
-    }
-
-    public static DiffType fromProtobuf(final DiffTypeProto type) {
-      return DiffType.valueOf(type.name());
-    }
-  }
-
-  /**
-   * Snapshot diff report entry.
-   */
-  public static final class DiffReportEntry {
-
-    /**
-     * The type of diff.
-     */
-    private final DiffType type;
-
-    /**
-     * Source File/Object path.
-     */
-    private final String sourcePath;
-
-    /**
-     * Destination File/Object path, if this is a re-name operation.
-     */
-    private final String targetPath;
-
-    private DiffReportEntry(final DiffType type, final String sourcePath,
-                            final String targetPath) {
-      this.type = type;
-      this.sourcePath = sourcePath;
-      this.targetPath = targetPath;
-    }
-
-    public static DiffReportEntry of(final DiffType type,
-                                     final String sourcePath) {
-      return of(type, sourcePath, null);
-    }
-
-    public static DiffReportEntry of(final DiffType type,
-                                     final String sourcePath,
-                                     final String targetPath) {
-      return new DiffReportEntry(type, sourcePath, targetPath);
-
-    }
-
-    @Override
-    public String toString() {
-      String str = type.getLabel() + "\t" + sourcePath;
-      if (type == DiffType.RENAME) {
-        str += " -> " + targetPath;
-      }
-      return str;
-    }
-
-    public DiffType getType() {
-      return type;
-    }
-
-    @Override
-    public boolean equals(Object other) {
-      if (this == other) {
-        return true;
-      }
-      if (other instanceof DiffReportEntry) {
-        DiffReportEntry entry = (DiffReportEntry) other;
-        return this.type.equals(entry.getType()) && this.sourcePath
-            .equals(entry.sourcePath) && (this.targetPath != null ?
-            this.targetPath.equals(entry.targetPath) : true);
-      }
-      return false;
-    }
-
-    @Override
-    public int hashCode() {
-      return toString().hashCode();
-    }
+public class SnapshotDiffReport
+    extends org.apache.hadoop.hdfs.protocol.SnapshotDiffReport {
 
-    public DiffReportEntryProto toProtobuf() {
-      final DiffReportEntryProto.Builder builder = DiffReportEntryProto
-          .newBuilder();
-      builder.setDiffType(type.toProtobuf()).setSourcePath(sourcePath);
-      if (targetPath != null) {
-        builder.setTargetPath(targetPath);
-      }
-      return builder.build();
-    }
-
-    public static DiffReportEntry fromProtobuf(
-        final DiffReportEntryProto entry) {
-      return of(DiffType.fromProtobuf(entry.getDiffType()),
-          entry.getSourcePath(),
-          entry.hasTargetPath() ? entry.getTargetPath() : null);
-    }
+  private static final String LINE_SEPARATOR =
+      System.getProperty("line.separator", "\n");
 
-  }
-
-
-  /**
-   * Volume name to which the snapshot bucket belongs.
-   */
-  private final String volumeName;
-
-  /**
-   * Bucket name to which the snapshot belongs.
-   */
-  private final String bucketName;
-  /**
-   * start point of the diff.
-   */
-  private final String fromSnapshot;
-
-  /**
-   * end point of the diff.
-   */
-  private final String toSnapshot;
-
-  /**
-   * list of diff.
-   */
-  private final List<DiffReportEntry> diffList;
-
-  public SnapshotDiffReport(final String volumeName, final String bucketName,
-                            final String fromSnapshot, final String toSnapshot,
-                            List<DiffReportEntry> entryList) {
+  public SnapshotDiffReport(String snapshotRoot, String fromSnapshot,
+      String toSnapshot, PersistentList<DiffReportEntry> entryList, String volumeName,
+      String bucketName) {
+    // TODO handle conversion from PersistentList to java.util.List
+    super(snapshotRoot, fromSnapshot, toSnapshot, entryList);
     this.volumeName = volumeName;
     this.bucketName = bucketName;
-    this.fromSnapshot = fromSnapshot;
-    this.toSnapshot = toSnapshot;
-    this.diffList = entryList != null ? entryList : Collections.emptyList();
   }
 
-  public List<DiffReportEntry> getDiffList() {
-    return diffList;
-  }
+  private String volumeName;
+
+  private String bucketName;
 
   @Override
   public String toString() {
     StringBuilder str = new StringBuilder();
-    String from = "snapshot " + fromSnapshot;
-    String to = "snapshot " + toSnapshot;
+    String from = "snapshot " + getFromSnapshot();
+    String to = "snapshot " + getToSnapshot();
     str.append("Difference between ").append(from).append(" and ").append(to)
-        .append(":")
-        .append(LINE_SEPARATOR);
-    for (DiffReportEntry entry : diffList) {
+        .append(":").append(LINE_SEPARATOR);
+    for (DiffReportEntry entry : getDiffList()) {
       str.append(entry.toString()).append(LINE_SEPARATOR);
     }
     return str.toString();
   }
 
-  public SnapshotDiffReportProto toProtobuf() {
-    final SnapshotDiffReportProto.Builder builder = SnapshotDiffReportProto
-        .newBuilder();
-    builder.setVolumeName(volumeName)
-        .setBucketName(bucketName)
-        .setFromSnapshot(fromSnapshot)
-        .setToSnapshot(toSnapshot);
-    builder.addAllDiffList(diffList.stream().map(DiffReportEntry::toProtobuf)
-        .collect(Collectors.toList()));
+  public OzoneManagerProtocolProtos.SnapshotDiffReportProto toProtobuf() {
+    final OzoneManagerProtocolProtos.SnapshotDiffReportProto.Builder builder =
+        OzoneManagerProtocolProtos.SnapshotDiffReportProto.newBuilder();
+    builder.setVolumeName(volumeName).setBucketName(bucketName)
+        .setFromSnapshot(getFromSnapshot()).setToSnapshot(getToSnapshot());
+    builder.addAllDiffList(
+        getDiffList().stream().map(x -> toProtobufDiffReportEntry(x))
+            .collect(Collectors.toList()));
     return builder.build();
   }
 
+  public static OzoneManagerProtocolProtos
+      .DiffReportEntryProto toProtobufDiffReportEntry(DiffReportEntry entry) {
+    final OzoneManagerProtocolProtos.DiffReportEntryProto.Builder builder =
+        OzoneManagerProtocolProtos.DiffReportEntryProto.newBuilder();
+    builder.setDiffType(toProtobufDiffType(entry.getType()))
+        .setSourcePath(new String(entry.getSourcePath()));
+    if (entry.getTargetPath() != null) {
+      String targetPath = new String(entry.getTargetPath());
+      builder.setTargetPath(targetPath);
+    }
+    return builder.build();
+  }
+
+  public static OzoneManagerProtocolProtos.DiffReportEntryProto
+      .DiffTypeProto toProtobufDiffType(DiffType type) {
+    return OzoneManagerProtocolProtos.DiffReportEntryProto
+        .DiffTypeProto.valueOf(type.name());
+  }
+
   public static SnapshotDiffReport fromProtobuf(
-      final SnapshotDiffReportProto report) {
-    return new SnapshotDiffReport(report.getVolumeName(),
-        report.getBucketName(), report.getFromSnapshot(),
+      final OzoneManagerProtocolProtos.SnapshotDiffReportProto report) {
+    Path bucketPath = new Path(
+        OZONE_URI_DELIMITER + report.getVolumeName()
+            + OZONE_URI_DELIMITER + report.getBucketName());
+    OFSPath path = new OFSPath(bucketPath, new OzoneConfiguration());
+
+    // TODO handle conversion from PersistentList to java.util.List
+    return new SnapshotDiffReport(path.toString(), report.getFromSnapshot(),
         report.getToSnapshot(), report.getDiffListList().stream()
-        .map(DiffReportEntry::fromProtobuf).collect(Collectors.toList()));
+        .map(SnapshotDiffReport::fromProtobufDiffReportEntry)
+        .collect(Collectors.toList()), report.getVolumeName(),
+        report.getBucketName());
+  }
+
+  public static DiffType fromProtobufDiffType(
+      final OzoneManagerProtocolProtos.DiffReportEntryProto
+          .DiffTypeProto type) {
+    return DiffType.valueOf(type.name());
   }
 
+  public static DiffReportEntry fromProtobufDiffReportEntry(
+      final OzoneManagerProtocolProtos.DiffReportEntryProto entry) {
+    if (entry == null) {
+      return null;
+    }
+    DiffType type = fromProtobufDiffType(entry.getDiffType());
+    return type == null ? null :
+        new DiffReportEntry(type, entry.getSourcePath().getBytes(),
+            entry.hasTargetPath() ? entry.getTargetPath().getBytes() : null);
+  }
 }