diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java index 9ab17f96766f..d205dfdae2ee 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java @@ -684,4 +684,22 @@ public SnapshotDiffResponse snapshotDiff(String volumeName, return proxy.snapshotDiff(volumeName, bucketName, fromSnapshot, toSnapshot, token, pageSize, forceFullDiff, cancel); } + + /** + * Get a list of the SnapshotDiff jobs for a bucket based on the JobStatus. + * @param volumeName Name of the volume to which the snapshotted bucket belong + * @param bucketName Name of the bucket to which the snapshots belong + * @param jobStatus JobStatus to be used to filter the snapshot diff jobs + * @param listAll Option to specify whether to list all jobs or not + * @return a list of SnapshotDiffJob objects + * @throws IOException in case there is a failure while getting a response. + */ + public List listSnapshotDiffJobs(String volumeName, + String bucketName, + String jobStatus, + boolean listAll) + throws IOException { + return proxy.listSnapshotDiffJobs(volumeName, + bucketName, jobStatus, listAll); + } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshotDiff.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshotDiff.java new file mode 100644 index 000000000000..a6ef2dc4bb27 --- /dev/null +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshotDiff.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.client; + +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; +import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus; + +/** + * A class that encapsulates SnapshotDiffJob. + */ +public class OzoneSnapshotDiff { + + private final String volumeName; + private final String bucketName; + private final String fromSnapshot; + private final String toSnapshot; + private final JobStatus jobStatus; + + public OzoneSnapshotDiff(String volumeName, + String bucketName, + String fromSnapshot, + String toSnapshot, + JobStatus jobStatus) { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.fromSnapshot = fromSnapshot; + this.toSnapshot = toSnapshot; + this.jobStatus = jobStatus; + } + + public String getVolumeName() { + return volumeName; + } + + public String getBucketName() { + return bucketName; + } + + public String getFromSnapshot() { + return fromSnapshot; + } + + public String getToSnapshot() { + return toSnapshot; + } + + public JobStatus getJobStatus() { + return jobStatus; + } + + public static OzoneSnapshotDiff fromSnapshotDiffJob( + SnapshotDiffJob snapshotDiffJob) { + return new OzoneSnapshotDiff( + snapshotDiffJob.getVolume(), + snapshotDiffJob.getBucket(), + snapshotDiffJob.getFromSnapshot(), + snapshotDiffJob.getToSnapshot(), + snapshotDiffJob.getStatus()); + } +} diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index b7bba4c2cdfc..83e29be2b26b 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts; import org.apache.hadoop.ozone.client.OzoneSnapshot; +import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.TenantArgs; import org.apache.hadoop.ozone.client.VolumeArgs; @@ -1078,6 +1079,21 @@ SnapshotDiffResponse snapshotDiff(String volumeName, String bucketName, boolean forceFullDiff, boolean cancel) throws IOException; + /** + * Get a list of the SnapshotDiff jobs for a bucket based on the JobStatus. + * @param volumeName Name of the volume to which the snapshotted bucket belong + * @param bucketName Name of the bucket to which the snapshots belong + * @param jobStatus JobStatus to be used to filter the snapshot diff jobs + * @param listAll Option to specify whether to list all jobs or not + * @return a list of SnapshotDiffJob objects + * @throws IOException in case there is a failure while getting a response. + */ + List listSnapshotDiffJobs(String volumeName, + String bucketName, + String jobStatus, + boolean listAll) + throws IOException; + /** * Time to be set for given Ozone object. This operations updates modification * time and access time for the given key. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index f8fd36d465b6..3f06452eb5ee 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -88,6 +88,7 @@ import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts; import org.apache.hadoop.ozone.client.OzoneSnapshot; +import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.TenantArgs; import org.apache.hadoop.ozone.client.VolumeArgs; @@ -996,6 +997,23 @@ public SnapshotDiffResponse snapshotDiff(String volumeName, fromSnapshot, toSnapshot, token, pageSize, forceFullDiff, cancel); } + @Override + public List listSnapshotDiffJobs(String volumeName, + String bucketName, + String jobStatus, + boolean listAll) + throws IOException { + Preconditions.checkArgument(StringUtils.isNotBlank(volumeName), + "volume can't be null or empty."); + Preconditions.checkArgument(StringUtils.isNotBlank(bucketName), + "bucket can't be null or empty."); + + return ozoneManagerClient.listSnapshotDiffJobs( + volumeName, bucketName, jobStatus, listAll).stream() + .map(OzoneSnapshotDiff::fromSnapshotDiffJob) + .collect(Collectors.toList()); + } + /** * List snapshots in a volume/bucket. * @param volumeName volume name diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 03b23501433c..4eea2aa7a298 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -266,6 +266,7 @@ public static boolean isReadOnly( // operation SetRangerServiceVersion. case GetKeyInfo: case SnapshotDiff: + case ListSnapshotDiffJobs: case TransferLeadership: return true; case CreateVolume: diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffJob.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java similarity index 85% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffJob.java rename to hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java index 50e968a75c8e..380483f15a13 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffJob.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.ozone.om.snapshot; +package org.apache.hadoop.ozone.om.helpers; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.Objects; import org.apache.hadoop.hdds.utils.db.Codec; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffJobProto; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus; /** @@ -185,6 +186,34 @@ public int hashCode() { fromSnapshot, toSnapshot, forceFullDiff, totalDiffEntries); } + public SnapshotDiffJobProto toProtoBuf() { + return SnapshotDiffJobProto.newBuilder() + .setCreationTime(creationTime) + .setJobId(jobId) + .setStatus(status.toProtobuf()) + .setVolume(volume) + .setBucket(bucket) + .setFromSnapshot(fromSnapshot) + .setToSnapshot(toSnapshot) + .setForceFullDiff(forceFullDiff) + .setTotalDiffEntries(totalDiffEntries) + .build(); + } + + public static SnapshotDiffJob getFromProtoBuf( + SnapshotDiffJobProto diffJobProto) { + return new SnapshotDiffJob( + diffJobProto.getCreationTime(), + diffJobProto.getJobId(), + JobStatus.fromProtobuf(diffJobProto.getStatus()), + diffJobProto.getVolume(), + diffJobProto.getBucket(), + diffJobProto.getFromSnapshot(), + diffJobProto.getToSnapshot(), + diffJobProto.getForceFullDiff(), + diffJobProto.getTotalDiffEntries()); + } + /** * Codec to encode SnapshotDiffJob as byte array. */ diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index e4e38b1377f3..a156a84f069a 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.TenantStateList; import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue; @@ -732,6 +733,23 @@ default SnapshotDiffResponse snapshotDiff(String volumeName, "this to be implemented"); } + /** + * Get a list of the SnapshotDiff jobs for a bucket based on the JobStatus. + * @param volumeName Name of the volume to which the snapshotted bucket belong + * @param bucketName Name of the bucket to which the snapshots belong + * @param jobStatus JobStatus to be used to filter the snapshot diff jobs + * @return a list of SnapshotDiffJob objects + * @throws IOException in case there is a failure while getting a response. + */ + default List listSnapshotDiffJobs(String volumeName, + String bucketName, + String jobStatus, + boolean listAll) + throws IOException { + throw new UnsupportedOperationException("OzoneManager does not require " + + "this to be implemented"); + } + /** * Assign admin role to a user identified by an accessId in a tenant. * @param accessId access ID. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index b789988dccea..ed2d3a69e129 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -65,6 +65,7 @@ import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.TenantStateList; import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue; @@ -1249,6 +1250,35 @@ public SnapshotDiffResponse snapshotDiff(String volumeName, JobCancelResult.fromProtobuf(diffResponse.getJobCancelResult())); } + /** + * {@inheritDoc} + */ + @Override + public List listSnapshotDiffJobs(String volumeName, + String bucketName, + String jobStatus, + boolean listAll) + throws IOException { + final OzoneManagerProtocolProtos + .ListSnapshotDiffJobRequest.Builder requestBuilder = + OzoneManagerProtocolProtos + .ListSnapshotDiffJobRequest.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setJobStatus(jobStatus) + .setListAll(listAll); + + final OMRequest omRequest = createOMRequest(Type.ListSnapshotDiffJobs) + .setListSnapshotDiffJobRequest(requestBuilder) + .build(); + final OMResponse omResponse = submitRequest(omRequest); + handleError(omResponse); + return omResponse.getListSnapshotDiffJobResponse() + .getSnapshotDiffJobList().stream() + .map(SnapshotDiffJob::getFromProtoBuf) + .collect(Collectors.toList()); + } + /** * {@inheritDoc} */ diff --git a/hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot b/hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot index 18ef86f552f2..7e7f36cb2f48 100644 --- a/hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot +++ b/hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot @@ -53,6 +53,13 @@ Snapshot Diff Should contain ${result} + ${KEY_TWO} Should contain ${result} + ${KEY_THREE} +List Snapshot Diff Jobs + ${result} = Execute ozone sh snapshot listDiff /${VOLUME}/${BUCKET} --all + Should contain ${result} ${VOLUME} + Should contain ${result} ${BUCKET} + Should contain ${result} ${SNAPSHOT_ONE} + Should contain ${result} ${SNAPSHOT_TWO} + Read Snapshot Key Should Match Local File /${VOLUME}/${BUCKET}/${SNAPSHOT_INDICATOR}/${SNAPSHOT_ONE}/${KEY_ONE} /etc/hosts Key Should Match Local File /${VOLUME}/${BUCKET}/${SNAPSHOT_INDICATOR}/${SNAPSHOT_TWO}/${KEY_TWO} /etc/passwd diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java index 32278636fe8c..c95e768f3d01 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java @@ -900,6 +900,57 @@ public void testSnapDiffMultipleBuckets() throws Exception { Assert.assertEquals(1, diff1.getDiffList().size()); } + @Test + public void testListSnapshotDiffWithInvalidParameters() + throws Exception { + String volume = "vol-" + RandomStringUtils.randomNumeric(5); + String bucket = "buck-" + RandomStringUtils.randomNumeric(5); + + String volBucketErrorMessage = "Provided volume name " + volume + + " or bucket name " + bucket + " doesn't exist"; + + Exception volBucketEx = Assertions.assertThrows(OMException.class, + () -> store.listSnapshotDiffJobs(volume, bucket, + "", true)); + Assertions.assertEquals(volBucketErrorMessage, + volBucketEx.getMessage()); + + // Create the volume and the bucket. + store.createVolume(volume); + OzoneVolume ozVolume = store.getVolume(volume); + ozVolume.createBucket(bucket); + + Assertions.assertDoesNotThrow(() -> + store.listSnapshotDiffJobs(volume, bucket, "", true)); + + // There are no snapshots, response should be empty. + Assertions.assertTrue(store + .listSnapshotDiffJobs(volume, bucket, + "", true).isEmpty()); + + OzoneBucket ozBucket = ozVolume.getBucket(bucket); + // Create keys and take snapshots. + String key1 = "key-1-" + RandomStringUtils.randomNumeric(5); + createFileKey(ozBucket, key1); + String snap1 = "snap-1-" + RandomStringUtils.randomNumeric(5); + createSnapshot(volume, bucket, snap1); + + String key2 = "key-2-" + RandomStringUtils.randomNumeric(5); + createFileKey(ozBucket, key2); + String snap2 = "snap-2-" + RandomStringUtils.randomNumeric(5); + createSnapshot(volume, bucket, snap2); + + store.snapshotDiff(volume, bucket, snap1, snap2, null, 0, true, false); + + String invalidStatus = "invalid"; + String statusErrorMessage = "Invalid job status: " + invalidStatus; + + Exception statusEx = Assertions.assertThrows(OMException.class, + () -> store.listSnapshotDiffJobs(volume, bucket, + invalidStatus, false)); + Assertions.assertEquals(statusErrorMessage, + statusEx.getMessage()); + } /** * Tests snapdiff when there are multiple sst files in the from & to diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index d577c557fb57..cb9549834ff7 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -136,6 +136,7 @@ enum Type { RecoverLease = 119; SetTimes = 120; RefetchSecretKey = 121; + ListSnapshotDiffJobs = 122; } message OMRequest { @@ -256,6 +257,8 @@ message OMRequest { optional RecoverLeaseRequest RecoverLeaseRequest = 119; optional SetTimesRequest SetTimesRequest = 120; optional RefetchSecretKeyRequest RefetchSecretKeyRequest = 121; + + optional ListSnapshotDiffJobRequest ListSnapshotDiffJobRequest = 122; } message OMResponse { @@ -368,6 +371,8 @@ message OMResponse { optional RecoverLeaseResponse RecoverLeaseResponse = 119; optional SetTimesResponse SetTimesResponse = 120; optional RefetchSecretKeyResponse RefetchSecretKeyResponse = 121; + + optional ListSnapshotDiffJobResponse ListSnapshotDiffJobResponse = 122; } enum Status { @@ -800,6 +805,18 @@ message SnapshotInfo { optional int64 dbTxSequenceNumber = 12; } +message SnapshotDiffJobProto { + optional uint64 creationTime = 1; + optional string jobId = 2; + optional SnapshotDiffResponse.JobStatusProto status = 3; + optional string volume = 4; + optional string bucket = 5; + optional string fromSnapshot = 6; + optional string toSnapshot = 7; + optional bool forceFullDiff = 8; + optional uint64 totalDiffEntries = 9; +} + message OzoneObj { enum ObjectType { VOLUME = 1; @@ -1725,6 +1742,13 @@ message SnapshotDiffRequest { optional bool cancel = 8; } +message ListSnapshotDiffJobRequest { + required string volumeName = 1; + required string bucketName = 2; + optional string jobStatus = 3; + optional bool listAll = 4; +} + message DeleteSnapshotRequest { optional string volumeName = 1; optional string bucketName = 2; @@ -1819,6 +1843,10 @@ message SnapshotDiffResponse { optional JobCancelResultProto jobCancelResult = 4; } +message ListSnapshotDiffJobResponse { + repeated SnapshotDiffJobProto snapshotDiffJob = 1; +} + message DeleteSnapshotResponse { } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 09afdcb45eea..3a89e1544fd0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -57,7 +57,7 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.service.SnapshotDiffCleanupService; -import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffJob; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; @@ -702,6 +702,38 @@ public SnapshotDiffResponse getSnapshotDiffReport(final String volume, return snapshotDiffReport; } + public List getSnapshotDiffList(final String volumeName, + final String bucketName, + final String jobStatus, + final boolean listAll) + throws IOException { + String volumeKey = ozoneManager.getMetadataManager() + .getVolumeKey(volumeName); + String bucketKey = ozoneManager.getMetadataManager() + .getBucketKey(volumeName, bucketName); + + if (!ozoneManager.getMetadataManager() + .getVolumeTable().isExist(volumeKey) || + !ozoneManager.getMetadataManager() + .getBucketTable().isExist(bucketKey)) { + throw new IOException("Provided volume name " + volumeName + + " or bucket name " + bucketName + " doesn't exist"); + } + OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl) + ozoneManager.getMetadataManager(); + SnapshotChainManager snapshotChainManager = + omMetadataManager.getSnapshotChainManager(); + String snapshotPath = volumeName + OM_KEY_PREFIX + bucketName; + if (snapshotChainManager.getSnapshotChainPath(snapshotPath) == null) { + // Return an empty ArrayList here to avoid + // unnecessarily iterating the SnapshotDiffJob table. + return new ArrayList<>(); + } + + return snapshotDiffManager.getSnapshotDiffJobList( + volumeName, bucketName, jobStatus, listAll); + } + private void validateSnapshotsExistAndActive(final String volumeName, final String bucketName, final String fromSnapshotName, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index c33158293c25..2843127caf51 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -83,6 +83,7 @@ import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneManagerVersion; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.ratis_snapshot.OmRatisSnapshotProvider; import org.apache.hadoop.ozone.om.ha.OMHAMetrics; import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; @@ -4524,6 +4525,15 @@ public SnapshotDiffResponse snapshotDiff(String volume, } } + public List listSnapshotDiffJobs(String volume, + String bucket, + String jobStatus, + boolean listAll) + throws IOException { + return omSnapshotManager.getSnapshotDiffList(volume, + bucket, jobStatus, listAll); + } + private String reconfOzoneAdmins(String newVal) { getConfiguration().set(OZONE_ADMINISTRATORS, newVal); Collection admins = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java index c147c6fdc363..e4f66b6512f3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java @@ -425,4 +425,9 @@ public UUID previousPathSnapshot(String snapshotPath, public String getTableKey(UUID snapshotId) { return snapshotIdToTableKey.get(snapshotId); } + + public LinkedHashMap getSnapshotChainPath( + String path) { + return snapshotChainByPath.get(path); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDiffCleanupService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDiffCleanupService.java index 6a84cc46237d..c0354530bd97 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDiffCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDiffCleanupService.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch; import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions; import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffJob; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 023e5311d851..745d227de076 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -69,6 +69,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithObjectID; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; @@ -427,6 +428,42 @@ public SnapshotDiffResponse cancelSnapshotDiff( return new SnapshotDiffResponse(report, jobStatus, 0L, jobCancelResult); } + public List getSnapshotDiffJobList( + String volumeName, String bucketName, + String jobStatus, boolean listAll) throws IOException { + List jobList = new ArrayList<>(); + + try (ClosableIterator> iterator = + snapDiffJobTable.iterator()) { + while (iterator.hasNext()) { + SnapshotDiffJob snapshotDiffJob = iterator.next().getValue(); + if (Objects.equals(snapshotDiffJob.getVolume(), volumeName) && + Objects.equals(snapshotDiffJob.getBucket(), bucketName)) { + if (listAll) { + jobList.add(snapshotDiffJob); + continue; + } + + if (Objects.equals(snapshotDiffJob.getStatus(), + getJobStatus(jobStatus))) { + jobList.add(snapshotDiffJob); + } + } + } + } + return jobList; + } + + private JobStatus getJobStatus(String jobStatus) + throws IOException { + try { + return JobStatus.valueOf(jobStatus.toUpperCase()); + } catch (IllegalArgumentException ex) { + LOG.info(ex.toString()); + throw new IOException("Invalid job status: " + jobStatus); + } + } + public SnapshotDiffResponse getSnapshotDiffReport( final String volumeName, final String bucketName, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 4ca0cfa8dfdc..daf348302ce6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -54,6 +54,7 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.TenantStateList; import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue; @@ -69,6 +70,8 @@ import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.upgrade.DisallowedUntilLayoutVersion; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListSnapshotDiffJobRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.EchoRPCRequest; @@ -305,6 +308,11 @@ public OMResponse handleReadRequest(OMRequest request) { request.getSnapshotDiffRequest()); responseBuilder.setSnapshotDiffResponse(snapshotDiffReport); break; + case ListSnapshotDiffJobs: + ListSnapshotDiffJobResponse listSnapDiffResponse = + listSnapshotDiffJobs(request.getListSnapshotDiffJobRequest()); + responseBuilder.setListSnapshotDiffJobResponse(listSnapDiffResponse); + break; case EchoRPC: EchoRPCResponse echoRPCResponse = echoRPC(request.getEchoRPCRequest()); @@ -1261,6 +1269,23 @@ private SnapshotDiffResponse snapshotDiff( return builder.build(); } + private ListSnapshotDiffJobResponse listSnapshotDiffJobs( + ListSnapshotDiffJobRequest listSnapshotDiffJobRequest) + throws IOException { + List snapshotDiffJobs = + impl.listSnapshotDiffJobs( + listSnapshotDiffJobRequest.getVolumeName(), + listSnapshotDiffJobRequest.getBucketName(), + listSnapshotDiffJobRequest.getJobStatus(), + listSnapshotDiffJobRequest.getListAll()); + ListSnapshotDiffJobResponse.Builder builder = + ListSnapshotDiffJobResponse.newBuilder(); + for (SnapshotDiffJob diffJob : snapshotDiffJobs) { + builder.addSnapshotDiffJob(diffJob.toProtoBuf()); + } + return builder.build(); + } + public OzoneManager getOzoneManager() { return impl; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java index 9cbddb4e5bd1..a2d6b6885b46 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDiffCleanupService.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffJob; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus; import org.apache.ozone.test.GenericTestUtils; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 7b1a8ad88b95..9d207f7c5411 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -46,6 +46,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.WithObjectID; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; @@ -91,6 +92,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -824,6 +826,112 @@ public void testCancelNewSnapshotDiff() snapshotDiffResponse.getJobCancelResult()); } + private static Stream listSnapshotDiffJobsScenarios() { + return Stream.of( + Arguments.of("queued", false, false), + Arguments.of("done", false, false), + Arguments.of("in_progress", false, true), + Arguments.of("queued", true, true), + Arguments.of("done", true, true), + Arguments.of("in_progress", true, true), + Arguments.of("invalid", true, true), + Arguments.of("", true, true) + ); + } + + @ParameterizedTest + @MethodSource("listSnapshotDiffJobsScenarios") + public void testListSnapshotDiffJobs(String jobStatus, + boolean listAll, + boolean containsJob) + throws IOException { + SnapshotDiffManager snapshotDiffManager = + getMockedSnapshotDiffManager(10); + + String volumeName = "vol-" + RandomStringUtils.randomNumeric(5); + String bucketName = "bucket-" + RandomStringUtils.randomNumeric(5); + + String fromSnapshotName = "snap-" + RandomStringUtils.randomNumeric(5); + String toSnapshotName = "snap-" + RandomStringUtils.randomNumeric(5); + + UUID fromSnapshotUUID = UUID.randomUUID(); + UUID toSnapshotUUID = UUID.randomUUID(); + + setupMocksForRunningASnapDiff(volumeName, bucketName); + + setUpSnapshots(volumeName, bucketName, fromSnapshotName, + toSnapshotName, fromSnapshotUUID, toSnapshotUUID); + + PersistentMap snapDiffJobTable = + snapshotDiffManager.getSnapDiffJobTable(); + String diffJobKey = fromSnapshotUUID + DELIMITER + toSnapshotUUID; + + SnapshotDiffJob diffJob = snapDiffJobTable.get(diffJobKey); + Assertions.assertNull(diffJob); + + // There are no jobs in the table, therefore + // the response list should be empty. + List jobList = snapshotDiffManager + .getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAll); + Assertions.assertTrue(jobList.isEmpty()); + + // SnapshotDiffReport + SnapshotDiffResponse snapshotDiffResponse = snapshotDiffManager + .getSnapshotDiffReport(volumeName, bucketName, + fromSnapshotName, toSnapshotName, + 0, 0, false); + + Assertions.assertEquals(SnapshotDiffResponse.JobStatus.IN_PROGRESS, + snapshotDiffResponse.getJobStatus()); + + diffJob = snapDiffJobTable.get(diffJobKey); + Assertions.assertNotNull(diffJob); + Assertions.assertEquals(SnapshotDiffResponse.JobStatus.IN_PROGRESS, + diffJob.getStatus()); + + jobList = snapshotDiffManager + .getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAll); + + // When listAll is true, jobStatus is ignored. + // If the job is IN_PROGRESS or listAll is used, + // there should be a response. + // Otherwise, response list should be empty. + if (containsJob) { + Assertions.assertTrue(jobList.contains(diffJob)); + } else { + Assertions.assertTrue(jobList.isEmpty()); + } + } + + @Test + public void testListSnapDiffWithInvalidStatus() throws IOException { + SnapshotDiffManager snapshotDiffManager = + getMockedSnapshotDiffManager(10); + + String volumeName = "vol-" + RandomStringUtils.randomNumeric(5); + String bucketName = "bucket-" + RandomStringUtils.randomNumeric(5); + + String fromSnapshotName = "snap-" + RandomStringUtils.randomNumeric(5); + String toSnapshotName = "snap-" + RandomStringUtils.randomNumeric(5); + + UUID fromSnapshotUUID = UUID.randomUUID(); + UUID toSnapshotUUID = UUID.randomUUID(); + + setupMocksForRunningASnapDiff(volumeName, bucketName); + + setUpSnapshots(volumeName, bucketName, fromSnapshotName, + toSnapshotName, fromSnapshotUUID, toSnapshotUUID); + + // SnapshotDiffReport + snapshotDiffManager.getSnapshotDiffReport(volumeName, bucketName, + fromSnapshotName, toSnapshotName, + 0, 0, false); + + // Invalid status, without listAll true, results in an exception. + Assertions.assertThrows(IOException.class, () -> snapshotDiffManager + .getSnapshotDiffJobList(volumeName, bucketName, "invalid", false)); + } + private void setUpSnapshots(String volumeName, String bucketName, String fromSnapshotName, String toSnapshotName, UUID fromSnapshotUUID, UUID toSnapshotUUID) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 8e30380b6a50..e0bed8e033e6 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -645,6 +645,13 @@ public SnapshotDiffResponse snapshotDiff(String volumeName, return null; } + @Override + public List listSnapshotDiffJobs( + String volumeName, String bucketName, + String jobStatus, boolean listAll) { + return null; + } + @Override public void setTimes(OzoneObj obj, String keyName, long mtime, long atime) throws IOException { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotDiffHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotDiffHandler.java new file mode 100644 index 000000000000..d1e6d5fbd8ce --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotDiffHandler.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.shell.snapshot; + +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; +import org.apache.hadoop.ozone.shell.Handler; +import org.apache.hadoop.ozone.shell.OzoneAddress; +import org.apache.hadoop.ozone.shell.bucket.BucketUri; +import picocli.CommandLine; + +import java.io.IOException; +import java.util.List; + +/** + * ozone sh snapshot listDiff. + */ +@CommandLine.Command(name = "listDiff", + aliases = {"listDiffJob", "lsDiff", "lsDiffJob"}, + description = "List snapshotDiff jobs for a bucket.") +public class ListSnapshotDiffHandler extends Handler { + + @CommandLine.Mixin + private BucketUri snapshotPath; + + @CommandLine.Option(names = {"-s", "--status"}, + description = "List jobs based on status.\n" + + "Accepted values are: queued, in_progress, done, failed, rejected", + defaultValue = "in_progress") + private String jobStatus; + + @CommandLine.Option(names = {"-a", "--all"}, + description = "List all jobs regardless of status.", + defaultValue = "false") + private boolean listAll; + + @Override + protected OzoneAddress getAddress() { + return snapshotPath.getValue(); + } + + @Override + protected void execute(OzoneClient client, OzoneAddress address) + throws IOException { + + String volumeName = snapshotPath.getValue().getVolumeName(); + String bucketName = snapshotPath.getValue().getBucketName(); + + List jobList = + client.getObjectStore().listSnapshotDiffJobs( + volumeName, bucketName, jobStatus, listAll); + + int counter = printAsJsonArray(jobList.iterator(), + jobList.size()); + if (isVerbose()) { + System.out.printf("Found : %d snapshot diff jobs for o3://%s/ %s ", + counter, volumeName, bucketName); + } + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotCommands.java index a2169e04dca7..0b3daab6a3f5 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotCommands.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotCommands.java @@ -41,7 +41,8 @@ CreateSnapshotHandler.class, DeleteSnapshotHandler.class, ListSnapshotHandler.class, - SnapshotDiffHandler.class + SnapshotDiffHandler.class, + ListSnapshotDiffHandler.class }, mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class)