HDDS-8704. [Snapshot] Add CLI to list SnapDiff jobs

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java

@@ -684,4 +684,22 @@ public SnapshotDiffResponse snapshotDiff(String volumeName,
     return proxy.snapshotDiff(volumeName, bucketName, fromSnapshot, toSnapshot,
         token, pageSize, forceFullDiff, cancel);
   }
+
+  /**
+   * Get a list of the SnapshotDiff jobs for a bucket based on the JobStatus.
+   * @param volumeName Name of the volume to which the snapshotted bucket belong
+   * @param bucketName Name of the bucket to which the snapshots belong
+   * @param jobStatus JobStatus to be used to filter the snapshot diff jobs
+   * @param listAll Option to specify whether to list all jobs or not
+   * @return a list of SnapshotDiffJob objects
+   * @throws IOException in case there is a failure while getting a response.
+   */
+  public List<OzoneSnapshotDiff> listSnapshotDiffJobs(String volumeName,
+                                                    String bucketName,
+                                                    String jobStatus,
+                                                    boolean listAll)
+      throws IOException {
+    return proxy.listSnapshotDiffJobs(volumeName,
+        bucketName, jobStatus, listAll);
+  }
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshotDiff.java

@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.client;
+
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus;
+
+/**
+ * A class that encapsulates SnapshotDiffJob.
+ */
+public class OzoneSnapshotDiff {
+
+  private final String volumeName;
+  private final String bucketName;
+  private final String fromSnapshot;
+  private final String toSnapshot;
+  private final JobStatus jobStatus;
+
+  public OzoneSnapshotDiff(String volumeName,
+                           String bucketName,
+                           String fromSnapshot,
+                           String toSnapshot,
+                           JobStatus jobStatus) {
+    this.volumeName = volumeName;
+    this.bucketName = bucketName;
+    this.fromSnapshot = fromSnapshot;
+    this.toSnapshot = toSnapshot;
+    this.jobStatus = jobStatus;
+  }
+
+  public String getVolumeName() {
+    return volumeName;
+  }
+
+  public String getBucketName() {
+    return bucketName;
+  }
+
+  public String getFromSnapshot() {
+    return fromSnapshot;
+  }
+
+  public String getToSnapshot() {
+    return toSnapshot;
+  }
+
+  public JobStatus getJobStatus() {
+    return jobStatus;
+  }
+
+  public static OzoneSnapshotDiff fromSnapshotDiffJob(
+      SnapshotDiffJob snapshotDiffJob) {
+    return new OzoneSnapshotDiff(
+        snapshotDiffJob.getVolume(),
+        snapshotDiffJob.getBucket(),
+        snapshotDiffJob.getFromSnapshot(),
+        snapshotDiffJob.getToSnapshot(),
+        snapshotDiffJob.getStatus());
+  }
+}

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -38,6 +38,7 @@
 import org.apache.hadoop.ozone.client.OzoneMultipartUploadList;
 import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
 import org.apache.hadoop.ozone.client.OzoneSnapshot;
+import org.apache.hadoop.ozone.client.OzoneSnapshotDiff;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.TenantArgs;
 import org.apache.hadoop.ozone.client.VolumeArgs;
@@ -1078,6 +1079,21 @@ SnapshotDiffResponse snapshotDiff(String volumeName, String bucketName,
                                     boolean forceFullDiff, boolean cancel)
       throws IOException;
 
+  /**
+   * Get a list of the SnapshotDiff jobs for a bucket based on the JobStatus.
+   * @param volumeName Name of the volume to which the snapshotted bucket belong
+   * @param bucketName Name of the bucket to which the snapshots belong
+   * @param jobStatus JobStatus to be used to filter the snapshot diff jobs
+   * @param listAll Option to specify whether to list all jobs or not
+   * @return a list of SnapshotDiffJob objects
+   * @throws IOException in case there is a failure while getting a response.
+   */
+  List<OzoneSnapshotDiff> listSnapshotDiffJobs(String volumeName,
+                                               String bucketName,
+                                               String jobStatus,
+                                               boolean listAll)
+      throws IOException;
+
   /**
    * Time to be set for given Ozone object. This operations updates modification
    * time and access time for the given key.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -88,6 +88,7 @@
 import org.apache.hadoop.ozone.client.OzoneMultipartUploadList;
 import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
 import org.apache.hadoop.ozone.client.OzoneSnapshot;
+import org.apache.hadoop.ozone.client.OzoneSnapshotDiff;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.TenantArgs;
 import org.apache.hadoop.ozone.client.VolumeArgs;
@@ -996,6 +997,23 @@ public SnapshotDiffResponse snapshotDiff(String volumeName,
         fromSnapshot, toSnapshot, token, pageSize, forceFullDiff, cancel);
   }
 
+  @Override
+  public List<OzoneSnapshotDiff> listSnapshotDiffJobs(String volumeName,
+                                                    String bucketName,
+                                                    String jobStatus,
+                                                    boolean listAll)
+      throws IOException {
+    Preconditions.checkArgument(StringUtils.isNotBlank(volumeName),
+        "volume can't be null or empty.");
+    Preconditions.checkArgument(StringUtils.isNotBlank(bucketName),
+        "bucket can't be null or empty.");
+
+    return ozoneManagerClient.listSnapshotDiffJobs(
+        volumeName, bucketName, jobStatus, listAll).stream()
+        .map(OzoneSnapshotDiff::fromSnapshotDiffJob)
+        .collect(Collectors.toList());
+  }
+
   /**
    * List snapshots in a volume/bucket.
    * @param volumeName     volume name

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -266,6 +266,7 @@ public static boolean isReadOnly(
       // operation SetRangerServiceVersion.
     case GetKeyInfo:
     case SnapshotDiff:
+    case ListSnapshotDiffJobs:
     case TransferLeadership:
       return true;
     case CreateVolume:

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffJob.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java

@@ -15,14 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.ozone.om.snapshot;
+package org.apache.hadoop.ozone.om.helpers;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.util.Objects;
 import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffJobProto;
 import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus;
 
 /**
@@ -185,6 +186,34 @@ public int hashCode() {
         fromSnapshot, toSnapshot, forceFullDiff, totalDiffEntries);
   }
 
+  public SnapshotDiffJobProto toProtoBuf() {
+    return SnapshotDiffJobProto.newBuilder()
+        .setCreationTime(creationTime)
+        .setJobId(jobId)
+        .setStatus(status.toProtobuf())
+        .setVolume(volume)
+        .setBucket(bucket)
+        .setFromSnapshot(fromSnapshot)
+        .setToSnapshot(toSnapshot)
+        .setForceFullDiff(forceFullDiff)
+        .setTotalDiffEntries(totalDiffEntries)
+        .build();
+  }
+
+  public static SnapshotDiffJob getFromProtoBuf(
+      SnapshotDiffJobProto diffJobProto) {
+    return new SnapshotDiffJob(
+        diffJobProto.getCreationTime(),
+        diffJobProto.getJobId(),
+        JobStatus.fromProtobuf(diffJobProto.getStatus()),
+        diffJobProto.getVolume(),
+        diffJobProto.getBucket(),
+        diffJobProto.getFromSnapshot(),
+        diffJobProto.getToSnapshot(),
+        diffJobProto.getForceFullDiff(),
+        diffJobProto.getTotalDiffEntries());
+  }
+
   /**
    * Codec to encode SnapshotDiffJob as byte array.
    */

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java

@@ -53,6 +53,7 @@
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
 import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
 import org.apache.hadoop.ozone.om.helpers.TenantStateList;
 import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
@@ -732,6 +733,23 @@ default SnapshotDiffResponse snapshotDiff(String volumeName,
         "this to be implemented");
   }
 
+  /**
+   * Get a list of the SnapshotDiff jobs for a bucket based on the JobStatus.
+   * @param volumeName Name of the volume to which the snapshotted bucket belong
+   * @param bucketName Name of the bucket to which the snapshots belong
+   * @param jobStatus JobStatus to be used to filter the snapshot diff jobs
+   * @return a list of SnapshotDiffJob objects
+   * @throws IOException in case there is a failure while getting a response.
+   */
+  default List<SnapshotDiffJob> listSnapshotDiffJobs(String volumeName,
+                                                     String bucketName,
+                                                     String jobStatus,
+                                                     boolean listAll)
+      throws IOException {
+    throw new UnsupportedOperationException("OzoneManager does not require " +
+        "this to be implemented");
+  }
+
   /**
    * Assign admin role to a user identified by an accessId in a tenant.
    * @param accessId access ID.

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -65,6 +65,7 @@
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
 import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
 import org.apache.hadoop.ozone.om.helpers.TenantStateList;
 import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
@@ -1249,6 +1250,35 @@ public SnapshotDiffResponse snapshotDiff(String volumeName,
         JobCancelResult.fromProtobuf(diffResponse.getJobCancelResult()));
   }
 
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public List<SnapshotDiffJob> listSnapshotDiffJobs(String volumeName,
+                                                    String bucketName,
+                                                    String jobStatus,
+                                                    boolean listAll)
+      throws IOException {
+    final OzoneManagerProtocolProtos
+        .ListSnapshotDiffJobRequest.Builder requestBuilder =
+        OzoneManagerProtocolProtos
+            .ListSnapshotDiffJobRequest.newBuilder()
+            .setVolumeName(volumeName)
+            .setBucketName(bucketName)
+            .setJobStatus(jobStatus)
+            .setListAll(listAll);
+
+    final OMRequest omRequest = createOMRequest(Type.ListSnapshotDiffJobs)
+        .setListSnapshotDiffJobRequest(requestBuilder)
+        .build();
+    final OMResponse omResponse = submitRequest(omRequest);
+    handleError(omResponse);
+    return omResponse.getListSnapshotDiffJobResponse()
+        .getSnapshotDiffJobList().stream()
+        .map(SnapshotDiffJob::getFromProtoBuf)
+        .collect(Collectors.toList());
+  }
+
   /**
    * {@inheritDoc}
    */

hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot

@@ -53,6 +53,13 @@ Snapshot Diff
                     Should contain      ${result}       +    ${KEY_TWO}
                     Should contain      ${result}       +    ${KEY_THREE}
 
+List Snapshot Diff Jobs
+    ${result} =     Execute             ozone sh snapshot listDiff /${VOLUME}/${BUCKET} --all
+                    Should contain      ${result}        ${VOLUME}
+                    Should contain      ${result}        ${BUCKET}
+                    Should contain      ${result}        ${SNAPSHOT_ONE}
+                    Should contain      ${result}        ${SNAPSHOT_TWO}
+
 Read Snapshot
     Key Should Match Local File         /${VOLUME}/${BUCKET}/${SNAPSHOT_INDICATOR}/${SNAPSHOT_ONE}/${KEY_ONE}       /etc/hosts
     Key Should Match Local File         /${VOLUME}/${BUCKET}/${SNAPSHOT_INDICATOR}/${SNAPSHOT_TWO}/${KEY_TWO}       /etc/passwd

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java

@@ -900,6 +900,57 @@ public void testSnapDiffMultipleBuckets() throws Exception {
     Assert.assertEquals(1, diff1.getDiffList().size());
   }
 
+  @Test
+  public void testListSnapshotDiffWithInvalidParameters()
+      throws Exception {
+    String volume = "vol-" + RandomStringUtils.randomNumeric(5);
+    String bucket = "buck-" + RandomStringUtils.randomNumeric(5);
+
+    String volBucketErrorMessage = "Provided volume name " + volume +
+        " or bucket name " + bucket + " doesn't exist";
+
+    Exception volBucketEx = Assertions.assertThrows(OMException.class,
+        () -> store.listSnapshotDiffJobs(volume, bucket,
+            "", true));
+    Assertions.assertEquals(volBucketErrorMessage,
+        volBucketEx.getMessage());
+
+    // Create the volume and the bucket.
+    store.createVolume(volume);
+    OzoneVolume ozVolume = store.getVolume(volume);
+    ozVolume.createBucket(bucket);
+
+    Assertions.assertDoesNotThrow(() ->
+        store.listSnapshotDiffJobs(volume, bucket, "", true));
+
+    // There are no snapshots, response should be empty.
+    Assertions.assertTrue(store
+        .listSnapshotDiffJobs(volume, bucket,
+            "", true).isEmpty());
+
+    OzoneBucket ozBucket = ozVolume.getBucket(bucket);
+    // Create keys and take snapshots.
+    String key1 = "key-1-" + RandomStringUtils.randomNumeric(5);
+    createFileKey(ozBucket, key1);
+    String snap1 = "snap-1-" + RandomStringUtils.randomNumeric(5);
+    createSnapshot(volume, bucket, snap1);
+
+    String key2 = "key-2-" + RandomStringUtils.randomNumeric(5);
+    createFileKey(ozBucket, key2);
+    String snap2 = "snap-2-" + RandomStringUtils.randomNumeric(5);
+    createSnapshot(volume, bucket, snap2);
+
+    store.snapshotDiff(volume, bucket, snap1, snap2, null, 0, true, false);
+
+    String invalidStatus = "invalid";
+    String statusErrorMessage = "Invalid job status: " + invalidStatus;
+
+    Exception statusEx = Assertions.assertThrows(OMException.class,
+        () -> store.listSnapshotDiffJobs(volume, bucket,
+            invalidStatus, false));
+    Assertions.assertEquals(statusErrorMessage,
+        statusEx.getMessage());
+  }
 
   /**
    * Tests snapdiff when there are multiple sst files in the from & to