apache · prashantpogde · Jun 21, 2023 · Jun 2, 2023 · Jun 2, 2023 · Jun 8, 2023
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -40,6 +40,7 @@
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
 import org.apache.hadoop.ozone.om.helpers.TenantStateList;
 import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
 import org.apache.hadoop.ozone.om.helpers.TenantUserList;
@@ -611,4 +612,21 @@ public SnapshotDiffResponse snapshotDiff(String volumeName,
     return proxy.snapshotDiff(volumeName, bucketName, fromSnapshot, toSnapshot,
         token, pageSize, forceFullDiff);
   }
+
+  /**
+   * Get a list of the SnapshotDiff jobs for a bucket based on the JobStatus.
+   * @param volumeName Name of the volume to which the snapshotted bucket belong
+   * @param bucketName Name of the bucket to which the snapshots belong
+   * @param jobStatus JobStatus to be used to filter the snapshot diff jobs
+   * @return a list of SnapshotDiffJob objects
+   * @throws IOException in case there is a failure while getting a response.
+   */
+  public List<SnapshotDiffJob> listSnapshotDiffJobs(String volumeName,
+                                                    String bucketName,
+                                                    String jobStatus,
+                                                    boolean listAll)
+      throws IOException {
+    return proxy.listSnapshotDiffJobs(volumeName,
+        bucketName, jobStatus, listAll);
+  }
 }
diff --git a/...op-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/...op-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -55,6 +55,7 @@
 import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
 import org.apache.hadoop.ozone.om.helpers.TenantStateList;
 import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
 import org.apache.hadoop.ozone.om.helpers.TenantUserList;
@@ -1072,6 +1073,20 @@ SnapshotDiffResponse snapshotDiff(String volumeName, String bucketName,
                                     boolean forceFullDiff)
       throws IOException;
 
+  /**
+   * Get a list of the SnapshotDiff jobs for a bucket based on the JobStatus.
+   * @param volumeName Name of the volume to which the snapshotted bucket belong
+   * @param bucketName Name of the bucket to which the snapshots belong
+   * @param jobStatus JobStatus to be used to filter the snapshot diff jobs
+   * @return a list of SnapshotDiffJob objects
+   * @throws IOException in case there is a failure while getting a response.
+   */
+  List<SnapshotDiffJob> listSnapshotDiffJobs(String volumeName,
+                                             String bucketName,
+                                             String jobStatus,
+                                             boolean listAll)
+      throws IOException;
+
   /**
    * Time to be set for given Ozone object. This operations updates modification
    * time and access time for the given key.

diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -134,6 +134,7 @@
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
 import org.apache.hadoop.ozone.om.helpers.TenantStateList;
 import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
 import org.apache.hadoop.ozone.om.helpers.TenantUserList;
@@ -996,6 +997,21 @@ public SnapshotDiffResponse snapshotDiff(String volumeName,
         fromSnapshot, toSnapshot, token, pageSize, forceFullDiff);
   }
 
+  @Override
+  public List<SnapshotDiffJob> listSnapshotDiffJobs(String volumeName,
+                                                    String bucketName,
+                                                    String jobStatus,
+                                                    boolean listAll)
+      throws IOException {
+    Preconditions.checkArgument(Strings.isNotBlank(volumeName),
+        "volume can't be null or empty.");
+    Preconditions.checkArgument(Strings.isNotBlank(bucketName),
+        "bucket can't be null or empty.");
+
+    return ozoneManagerClient.listSnapshotDiffJobs(
+        volumeName, bucketName, jobStatus, listAll);
+  }
+
   /**
    * List snapshots in a volume/bucket.
    * @param volumeName volume name

diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -265,6 +265,7 @@ public static boolean isReadOnly(
       // operation SetRangerServiceVersion.
     case GetKeyInfo:
     case SnapshotDiff:
+    case ListSnapshotDiffJob:
     case TransferLeadership:
       return true;
     case CreateVolume:

diff --git a/...op/ozone/om/snapshot/SnapshotDiffJob.java → ...oop/ozone/om/helpers/SnapshotDiffJob.java b/...op/ozone/om/snapshot/SnapshotDiffJob.java → ...oop/ozone/om/helpers/SnapshotDiffJob.java
@@ -15,14 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.ozone.om.snapshot;
+package org.apache.hadoop.ozone.om.helpers;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.util.Objects;
 import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffJobProto;
 import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus;
 
 /**
@@ -185,6 +186,34 @@ public int hashCode() {
         fromSnapshot, toSnapshot, forceFullDiff, totalDiffEntries);
   }
 
+  public SnapshotDiffJobProto toProtoBuf() {
+    return SnapshotDiffJobProto.newBuilder()
+        .setCreationTime(creationTime)
+        .setJobId(jobId)
+        .setStatus(status.toProtobuf())
+        .setVolume(volume)
+        .setBucket(bucket)
+        .setFromSnapshot(fromSnapshot)
+        .setToSnapshot(toSnapshot)
+        .setForceFullDiff(forceFullDiff)
+        .setTotalDiffEntries(totalDiffEntries)
+        .build();
+  }
+
+  public static SnapshotDiffJob getFromProtoBuf(
+      SnapshotDiffJobProto diffJobProto) {
+    return new SnapshotDiffJob(
+        diffJobProto.getCreationTime(),
+        diffJobProto.getJobId(),
+        JobStatus.fromProtobuf(diffJobProto.getStatus()),
+        diffJobProto.getVolume(),
+        diffJobProto.getBucket(),
+        diffJobProto.getFromSnapshot(),
+        diffJobProto.getToSnapshot(),
+        diffJobProto.getForceFullDiff(),
+        diffJobProto.getTotalDiffEntries());
+  }
+
   /**
    * Codec to encode SnapshotDiffJob as byte array.
    */

diff --git a/...-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/...-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -52,6 +52,7 @@
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
 import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
 import org.apache.hadoop.ozone.om.helpers.TenantStateList;
 import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
@@ -724,6 +725,23 @@ default SnapshotDiffResponse snapshotDiff(String volumeName,
         "this to be implemented");
   }
 
+  /**
+   * Get a list of the SnapshotDiff jobs for a bucket based on the JobStatus.
+   * @param volumeName Name of the volume to which the snapshotted bucket belong
+   * @param bucketName Name of the bucket to which the snapshots belong
+   * @param jobStatus JobStatus to be used to filter the snapshot diff jobs
+   * @return a list of SnapshotDiffJob objects
+   * @throws IOException in case there is a failure while getting a response.
+   */
+  default List<SnapshotDiffJob> listSnapshotDiffJobs(String volumeName,
+                                                     String bucketName,
+                                                     String jobStatus,
+                                                     boolean listAll)
+      throws IOException {
+    throw new UnsupportedOperationException("OzoneManager does not require " +
+        "this to be implemented");
+  }
+
   /**
    * Assign admin role to a user identified by an accessId in a tenant.
    * @param accessId access ID.

diff --git a/...ava/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/...ava/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -64,6 +64,7 @@
 import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
 import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
 import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
 import org.apache.hadoop.ozone.om.helpers.TenantStateList;
 import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
@@ -1231,6 +1232,35 @@ public SnapshotDiffResponse snapshotDiff(String volumeName,
         diffResponse.getWaitTimeInMs());
   }
 
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public List<SnapshotDiffJob> listSnapshotDiffJobs(String volumeName,
+                                                    String bucketName,
+                                                    String jobStatus,
+                                                    boolean listAll)
+      throws IOException {
+    final OzoneManagerProtocolProtos
+        .ListSnapshotDiffJobRequest.Builder requestBuilder =
+        OzoneManagerProtocolProtos
+            .ListSnapshotDiffJobRequest.newBuilder()
+            .setVolumeName(volumeName)
+            .setBucketName(bucketName)
+            .setJobStatus(jobStatus)
+            .setListAll(listAll);
+
+    final OMRequest omRequest = createOMRequest(Type.ListSnapshotDiffJob)
+        .setListSnapshotDiffJobRequest(requestBuilder)
+        .build();
+    final OMResponse omResponse = submitRequest(omRequest);
+    handleError(omResponse);
+    return omResponse.getListSnapshotDiffJobResponse()
+        .getSnapshotDiffJobList().stream()
+        .map(SnapshotDiffJob::getFromProtoBuf)
+        .collect(Collectors.toList());
+  }
+
   /**
    * {@inheritDoc}
    */

diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java
@@ -41,6 +41,15 @@ public JobStatusProto toProtobuf() {
     public static JobStatus fromProtobuf(JobStatusProto jobStatusProto) {
       return JobStatus.valueOf(jobStatusProto.name());
     }
+
+    public static JobStatus getJobStatusFromString(String jobStatus) {
+      for (JobStatus status : JobStatus.values()) {
+        if (status.toString().equalsIgnoreCase(jobStatus)) {
+          return status;
+        }
+      }
+      return null;
+    }
   }
 
   private final SnapshotDiffReportOzone snapshotDiffReport;

diff --git a/hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot b/hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot
@@ -53,6 +53,13 @@ Snapshot Diff
                     Should contain      ${result}       +    ${KEY_TWO}
                     Should contain      ${result}       +    ${KEY_THREE}
 
+List Snapshot Diff Jobs
+    ${result} =     Execute             ozone sh snapshot listSnapshotDiff /${VOLUME}/${BUCKET} --all
+                    Should contain      ${result}        ${VOLUME}
+                    Should contain      ${result}        ${BUCKET}
+                    Should contain      ${result}        ${SNAPSHOT_ONE}
+                    Should contain      ${result}        ${SNAPSHOT_TWO}
+
 Read Snapshot
     Key Should Match Local File         /${VOLUME}/${BUCKET}/${SNAPSHOT_INDICATOR}/${SNAPSHOT_ONE}/${KEY_ONE}       /etc/hosts
     Key Should Match Local File         /${VOLUME}/${BUCKET}/${SNAPSHOT_INDICATOR}/${SNAPSHOT_TWO}/${KEY_TWO}       /etc/passwd

diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -135,6 +135,7 @@ enum Type {
   SnapshotPurge = 118;
   RecoverLease = 119;
   SetTimes = 120;
+  ListSnapshotDiffJob = 121;
 }
 
 message OMRequest {
@@ -254,6 +255,8 @@ message OMRequest {
 
   optional RecoverLeaseRequest              RecoverLeaseRequest            = 119;
   optional SetTimesRequest                   SetTimesRequest               = 120;
+
+  optional ListSnapshotDiffJobRequest       ListSnapshotDiffJobRequest   = 121;
 }
 
 message OMResponse {
@@ -365,6 +368,8 @@ message OMResponse {
   optional SnapshotPurgeResponse              SnapshotPurgeResponse         = 118;
   optional RecoverLeaseResponse              RecoverLeaseResponse          = 119;
   optional SetTimesResponse                   SetTimesResponse             = 120;
+
+  optional ListSnapshotDiffJobResponse       ListSnapshotDiffJobResponse   = 121;
 }
 
 enum Status {
@@ -789,6 +794,18 @@ message SnapshotInfo {
   optional int64 dbTxSequenceNumber = 12;
  }
 
+message SnapshotDiffJobProto {
+  optional uint64 creationTime = 1;
+  optional string jobId = 2;
+  optional SnapshotDiffResponse.JobStatusProto status = 3;
+  optional string volume = 4;
+  optional string bucket = 5;
+  optional string fromSnapshot = 6;
+  optional string toSnapshot = 7;
+  optional bool forceFullDiff = 8;
+  optional uint64 totalDiffEntries = 9;
+}
+
 message OzoneObj {
   enum ObjectType {
     VOLUME = 1;
@@ -1709,6 +1726,13 @@ message SnapshotDiffRequest {
   optional bool forceFullDiff = 7;
 }
 
+message ListSnapshotDiffJobRequest {
+  optional string volumeName = 1;
+  optional string bucketName = 2;
+  optional string jobStatus = 3;
+  optional bool listAll = 4;
+}
+
 message DeleteSnapshotRequest {
   optional string volumeName = 1;
   optional string bucketName = 2;
@@ -1791,6 +1815,10 @@ message SnapshotDiffResponse {
   optional int64 waitTimeInMs = 3;
 }
 
+message ListSnapshotDiffJobResponse {
+  repeated SnapshotDiffJobProto snapshotDiffJob = 1;
+}
+
 message DeleteSnapshotResponse {
 
 }

diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -57,7 +57,7 @@
 import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
 import org.apache.hadoop.ozone.om.service.SnapshotDiffCleanupService;
-import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffJob;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
 import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager;
 import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
 import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
@@ -487,6 +487,11 @@ public static String getOzonePathKeyWithVolumeBucketNames(
     return OM_KEY_PREFIX + volumeId + OM_KEY_PREFIX + bucketId + OM_KEY_PREFIX;
   }
 
+  @VisibleForTesting
+  public SnapshotDiffManager getSnapshotDiffManager() {
+    return snapshotDiffManager;
+  }
+
   /**
    * Helper method to locate the end key with the given prefix and iterator.
    * @param keyIter TableIterator
@@ -681,6 +686,14 @@ public SnapshotDiffResponse getSnapshotDiffReport(final String volume,
     return snapshotDiffReport;
   }
 
+  public List<SnapshotDiffJob> getSnapshotDiffList(final String volumeName,
+                                                   final String bucketName,
+                                                   final String jobStatus,
+                                                   final boolean listAll) {
+    return snapshotDiffManager.getSnapshotDiffJobList(
+        volumeName, bucketName, jobStatus, listAll);
+  }
+
   private void validateSnapshotsExistAndActive(final String volumeName,
                                                final String bucketName,
                                                final String fromSnapshotName,

diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -87,6 +87,7 @@
 import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
 import org.apache.hadoop.hdds.utils.db.TableIterator;
 import org.apache.hadoop.ozone.OzoneManagerVersion;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
 import org.apache.hadoop.ozone.om.ratis_snapshot.OmRatisSnapshotProvider;
 import org.apache.hadoop.ozone.om.ha.OMHAMetrics;
 import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
@@ -4531,6 +4532,14 @@ public SnapshotDiffResponse snapshotDiff(String volume,
         fromSnapshot, toSnapshot, token, pageSize, forceFullDiff);
   }
 
+  public List<SnapshotDiffJob> listSnapshotDiffJobs(String volume,
+                                                    String bucket,
+                                                    String jobStatus,
+                                                    boolean listAll) {
+    return omSnapshotManager.getSnapshotDiffList(volume,
+        bucket, jobStatus, listAll);
+  }
+
   @Override // ReconfigureProtocol
   public String getServerName() {
     return "OM";

diff --git a/...-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDiffCleanupService.java b/...-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDiffCleanupService.java
@@ -29,7 +29,7 @@
 import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions;
 import org.apache.hadoop.ozone.om.OzoneManager;
-import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffJob;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.RocksDBException;