diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java index e96d0f84a43..828d956d6de 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.NoSuchElementException; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -46,6 +47,7 @@ import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.security.UserGroupInformation; @@ -748,19 +750,75 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, /** * Get a list of the SnapshotDiff jobs for a bucket based on the JobStatus. - * @param volumeName Name of the volume to which the snapshotted bucket belong - * @param bucketName Name of the bucket to which the snapshots belong - * @param jobStatus JobStatus to be used to filter the snapshot diff jobs - * @param listAll Option to specify whether to list all jobs or not - * @return a list of SnapshotDiffJob objects + * + * @param volumeName Name of the volume to which the snapshotted bucket belong + * @param bucketName Name of the bucket to which the snapshots belong + * @param jobStatus JobStatus to be used to filter the snapshot diff jobs + * @param listAll Option to specify whether to list all jobs or not + * @param prevSnapshotDiffJob list snapshot diff jobs after this snapshot diff job. + * @return an iterator of SnapshotDiffJob objects * @throws IOException in case there is a failure while getting a response. */ - public List listSnapshotDiffJobs(String volumeName, - String bucketName, - String jobStatus, - boolean listAll) + public Iterator listSnapshotDiffJobs(String volumeName, + String bucketName, + String jobStatus, + boolean listAll, + String prevSnapshotDiffJob) throws IOException { - return proxy.listSnapshotDiffJobs(volumeName, - bucketName, jobStatus, listAll); + return new SnapshotDiffJobIterator(volumeName, bucketName, jobStatus, listAll, prevSnapshotDiffJob); + } + + /** + * An Iterator to iterate over {@link SnapshotDiffJobIterator} list. + */ + private final class SnapshotDiffJobIterator implements Iterator { + private final String volumeName; + private final String bucketName; + private final String jobStatus; + private final boolean listAll; + private String lastSnapshotDiffJob; + private Iterator currentIterator; + + private SnapshotDiffJobIterator(String volumeName, + String bucketName, + String jobStatus, + boolean listAll, + String prevSnapshotDiffJob) throws IOException { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.jobStatus = jobStatus; + this.listAll = listAll; + // Initialized the currentIterator and lastSnapshotDiffJob. + getNextListOfSnapshotDiffJobs(prevSnapshotDiffJob); + } + + @Override + public boolean hasNext() { + if (!currentIterator.hasNext() && StringUtils.isNotEmpty(lastSnapshotDiffJob)) { + try { + // fetch the next page if continuationToken is not null. + getNextListOfSnapshotDiffJobs(lastSnapshotDiffJob); + } catch (IOException e) { + LOG.error("Error retrieving next batch of list for snapshot diff jobs.", e); + } + } + return currentIterator.hasNext(); + } + + @Override + public OzoneSnapshotDiff next() { + if (hasNext()) { + return currentIterator.next(); + } + throw new NoSuchElementException(); + } + + private void getNextListOfSnapshotDiffJobs(String prevSnapshotDiffJob) throws IOException { + ListSnapshotDiffJobResponse response = + proxy.listSnapshotDiffJobs(volumeName, bucketName, jobStatus, listAll, prevSnapshotDiffJob, listCacheSize); + this.currentIterator = + response.getSnapshotDiffJobs().stream().map(OzoneSnapshotDiff::fromSnapshotDiffJob).iterator(); + this.lastSnapshotDiffJob = response.getLastSnapshotDiffJob(); + } } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 492cd31b672..c68639df6b7 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -39,7 +39,6 @@ import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts; import org.apache.hadoop.ozone.client.OzoneSnapshot; -import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.TenantArgs; import org.apache.hadoop.ozone.client.VolumeArgs; @@ -67,6 +66,7 @@ import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.token.Token; @@ -1192,13 +1192,17 @@ CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, * @param bucketName Name of the bucket to which the snapshots belong * @param jobStatus JobStatus to be used to filter the snapshot diff jobs * @param listAll Option to specify whether to list all jobs or not + * @param prevSnapshotDiffJob list snapshot diff jobs after this snapshot diff job. + * @param maxListResult maximum entries to be returned from the startSnapshotDiffJob. * @return a list of SnapshotDiffJob objects * @throws IOException in case there is a failure while getting a response. */ - List listSnapshotDiffJobs(String volumeName, - String bucketName, - String jobStatus, - boolean listAll) + ListSnapshotDiffJobResponse listSnapshotDiffJobs(String volumeName, + String bucketName, + String jobStatus, + boolean listAll, + String prevSnapshotDiffJob, + int maxListResult) throws IOException; /** diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 0806ffb8472..aba140d223e 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -74,7 +74,6 @@ import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts; import org.apache.hadoop.ozone.client.OzoneSnapshot; -import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.TenantArgs; import org.apache.hadoop.ozone.client.VolumeArgs; @@ -141,6 +140,7 @@ import org.apache.hadoop.ozone.security.acl.OzoneAclConfig; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -1088,20 +1088,19 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, } @Override - public List listSnapshotDiffJobs(String volumeName, - String bucketName, - String jobStatus, - boolean listAll) + public ListSnapshotDiffJobResponse listSnapshotDiffJobs(String volumeName, + String bucketName, + String jobStatus, + boolean listAll, + String prevSnapshotDiffJob, + int maxListResult) throws IOException { Preconditions.checkArgument(StringUtils.isNotBlank(volumeName), "volume can't be null or empty."); Preconditions.checkArgument(StringUtils.isNotBlank(bucketName), "bucket can't be null or empty."); - - return ozoneManagerClient.listSnapshotDiffJobs( - volumeName, bucketName, jobStatus, listAll).stream() - .map(OzoneSnapshotDiff::fromSnapshotDiffJob) - .collect(Collectors.toList()); + return ozoneManagerClient.listSnapshotDiffJobs(volumeName, bucketName, jobStatus, listAll, prevSnapshotDiffJob, + maxListResult); } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index ab3f576d449..0c36fedfa3d 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -56,7 +56,6 @@ import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; -import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.TenantStateList; import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue; @@ -70,6 +69,7 @@ import org.apache.hadoop.ozone.security.OzoneDelegationTokenSelector; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages; import org.apache.hadoop.security.KerberosInfo; @@ -803,13 +803,18 @@ default CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, * @param volumeName Name of the volume to which the snapshotted bucket belong * @param bucketName Name of the bucket to which the snapshots belong * @param jobStatus JobStatus to be used to filter the snapshot diff jobs + * @param listAll Option to specify whether to list all jobs or not. + * @param prevSnapshotDiffJob list snapshot diff jobs after this snapshot diff job. + * @param maxListResult maximum entries to be returned from the startSnapshotDiffJob. * @return a list of SnapshotDiffJob objects * @throws IOException in case there is a failure while getting a response. */ - default List listSnapshotDiffJobs(String volumeName, - String bucketName, - String jobStatus, - boolean listAll) + default ListSnapshotDiffJobResponse listSnapshotDiffJobs(String volumeName, + String bucketName, + String jobStatus, + boolean listAll, + String prevSnapshotDiffJob, + int maxListResult) throws IOException { throw new UnsupportedOperationException("OzoneManager does not require " + "this to be implemented"); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 08fa029833e..8d5730d9310 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -222,6 +222,7 @@ import org.apache.hadoop.ozone.security.proto.SecurityProtos.GetDelegationTokenRequestProto; import org.apache.hadoop.ozone.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus; @@ -1430,10 +1431,12 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, * {@inheritDoc} */ @Override - public List listSnapshotDiffJobs(String volumeName, - String bucketName, - String jobStatus, - boolean listAll) + public ListSnapshotDiffJobResponse listSnapshotDiffJobs(String volumeName, + String bucketName, + String jobStatus, + boolean listAll, + String prevSnapshotDiffJob, + int maxListResult) throws IOException { final OzoneManagerProtocolProtos .ListSnapshotDiffJobRequest.Builder requestBuilder = @@ -1442,17 +1445,24 @@ public List listSnapshotDiffJobs(String volumeName, .setVolumeName(volumeName) .setBucketName(bucketName) .setJobStatus(jobStatus) - .setListAll(listAll); + .setListAll(listAll) + .setMaxListResult(maxListResult); + + if (prevSnapshotDiffJob != null) { + requestBuilder.setPrevSnapshotDiffJob(prevSnapshotDiffJob); + } final OMRequest omRequest = createOMRequest(Type.ListSnapshotDiffJobs) .setListSnapshotDiffJobRequest(requestBuilder) .build(); final OMResponse omResponse = submitRequest(omRequest); handleError(omResponse); - return omResponse.getListSnapshotDiffJobResponse() - .getSnapshotDiffJobList().stream() + OzoneManagerProtocolProtos.ListSnapshotDiffJobResponse response = omResponse.getListSnapshotDiffJobResponse(); + List ozoneSnapshotDiffs = response.getSnapshotDiffJobList().stream() .map(SnapshotDiffJob::getFromProtoBuf) .collect(Collectors.toList()); + return new ListSnapshotDiffJobResponse(ozoneSnapshotDiffs, + response.hasLastSnapshotDiffJob() ? response.getLastSnapshotDiffJob() : null); } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/ListSnapshotDiffJobResponse.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/ListSnapshotDiffJobResponse.java new file mode 100644 index 00000000000..82a38c9a5a9 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/ListSnapshotDiffJobResponse.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.snapshot; + +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; + +import java.util.List; + +/** + * POJO for list snapshot diff job API. + */ +public final class ListSnapshotDiffJobResponse { + private final List snapshotDiffJobs; + private final String lastSnapshotDiffJob; + + public ListSnapshotDiffJobResponse(List snapshotDiffJobs, String lastSnapshotDiffJob) { + this.snapshotDiffJobs = snapshotDiffJobs; + this.lastSnapshotDiffJob = lastSnapshotDiffJob; + } + + public List getSnapshotDiffJobs() { + return snapshotDiffJobs; + } + + public String getLastSnapshotDiffJob() { + return lastSnapshotDiffJob; + } + + @Override + public String toString() { + return "ListSnapshotDiffJobResponse{" + + "snapshotDiffJobs: '" + snapshotDiffJobs + '\'' + + ", lastSnapshotDiffJob: '" + lastSnapshotDiffJob + '\'' + + '}'; + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index ed399b37042..7c331d4a087 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -79,6 +79,7 @@ import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer; @@ -140,6 +141,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -1736,8 +1738,7 @@ public void testListSnapshotDiffWithInvalidParameters() " or bucket name " + bucket + " doesn't exist"; Exception volBucketEx = assertThrows(OMException.class, - () -> store.listSnapshotDiffJobs(volume, bucket, - "", true)); + () -> store.listSnapshotDiffJobs(volume, bucket, "", true, null)); assertEquals(volBucketErrorMessage, volBucketEx.getMessage()); @@ -1747,12 +1748,11 @@ public void testListSnapshotDiffWithInvalidParameters() ozVolume.createBucket(bucket); assertDoesNotThrow(() -> - store.listSnapshotDiffJobs(volume, bucket, "", true)); + store.listSnapshotDiffJobs(volume, bucket, "", true, null)); // There are no snapshots, response should be empty. - assertTrue(store - .listSnapshotDiffJobs(volume, bucket, - "", true).isEmpty()); + Iterator iterator = store.listSnapshotDiffJobs(volume, bucket, "", true, null); + assertFalse(iterator.hasNext()); OzoneBucket ozBucket = ozVolume.getBucket(bucket); // Create keys and take snapshots. @@ -1773,8 +1773,7 @@ public void testListSnapshotDiffWithInvalidParameters() String statusErrorMessage = "Invalid job status: " + invalidStatus; OMException statusEx = assertThrows(OMException.class, - () -> store.listSnapshotDiffJobs(volume, bucket, - invalidStatus, false)); + () -> store.listSnapshotDiffJobs(volume, bucket, invalidStatus, false, null)); assertEquals(statusErrorMessage, statusEx.getMessage()); } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index b0d26020c8d..ac80069bae1 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1872,6 +1872,8 @@ message ListSnapshotDiffJobRequest { required string bucketName = 2; optional string jobStatus = 3; optional bool listAll = 4; + optional string prevSnapshotDiffJob = 5; + optional uint32 maxListResult = 6; } message DeleteSnapshotRequest { @@ -1997,6 +1999,7 @@ message CancelSnapshotDiffResponse { message ListSnapshotDiffJobResponse { repeated SnapshotDiffJobProto snapshotDiffJob = 1; + optional string lastSnapshotDiffJob = 2; } message DeleteSnapshotResponse { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index a3799b389c5..eb472b1b57e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -64,6 +64,7 @@ import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.ratis.util.function.CheckedFunction; @@ -767,10 +768,12 @@ public SnapshotDiffResponse getSnapshotDiffReport(final String volume, return snapshotDiffReport; } - public List getSnapshotDiffList(final String volumeName, - final String bucketName, - final String jobStatus, - final boolean listAll) + public ListSnapshotDiffJobResponse getSnapshotDiffList(final String volumeName, + final String bucketName, + final String jobStatus, + final boolean listAll, + final String prevSnapshotDiffJob, + int maxListResult) throws IOException { String volumeKey = ozoneManager.getMetadataManager() .getVolumeKey(volumeName); @@ -778,7 +781,7 @@ public List getSnapshotDiffList(final String volumeName, .getBucketKey(volumeName, bucketName); if (!ozoneManager.getMetadataManager() - .getVolumeTable().isExist(volumeKey) || + .getVolumeTable().isExist(volumeKey) || !ozoneManager.getMetadataManager() .getBucketTable().isExist(bucketKey)) { throw new IOException("Provided volume name " + volumeName + @@ -792,11 +795,15 @@ public List getSnapshotDiffList(final String volumeName, if (snapshotChainManager.getSnapshotChainPath(snapshotPath) == null) { // Return an empty ArrayList here to avoid // unnecessarily iterating the SnapshotDiffJob table. - return new ArrayList<>(); + return new ListSnapshotDiffJobResponse(Collections.emptyList(), null); } - return snapshotDiffManager.getSnapshotDiffJobList( - volumeName, bucketName, jobStatus, listAll); + if (maxListResult < 0 || maxListResult > maxPageSize) { + maxListResult = maxPageSize; + } + + return snapshotDiffManager.getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAll, prevSnapshotDiffJob, + maxListResult); } private void validateSnapshotsExistAndActive(final String volumeName, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 680ebe06315..acbba0d44ea 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -90,7 +90,6 @@ import org.apache.hadoop.hdds.utils.db.Table.KeyValue; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneManagerVersion; -import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.lock.OMLockDetails; import org.apache.hadoop.ozone.om.ratis_snapshot.OmRatisSnapshotProvider; import org.apache.hadoop.ozone.om.ha.OMHAMetrics; @@ -105,6 +104,7 @@ import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.security.acl.OzoneAuthorizerFactory; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.ozone.util.OzoneNetUtils; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -4799,13 +4799,15 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volume, fromSnapshot, toSnapshot); } - public List listSnapshotDiffJobs(String volume, - String bucket, - String jobStatus, - boolean listAll) + public ListSnapshotDiffJobResponse listSnapshotDiffJobs(String volume, + String bucket, + String jobStatus, + boolean listAll, + String prevSnapshotDiffJob, + int maxListResult) throws IOException { - return omSnapshotManager.getSnapshotDiffList(volume, - bucket, jobStatus, listAll); + return omSnapshotManager.getSnapshotDiffList(volume, bucket, jobStatus, listAll, prevSnapshotDiffJob, + maxListResult); } public String printCompactionLogDag(String fileNamePrefix, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index a200a36cb25..578cc8b4548 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -48,6 +48,7 @@ import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus; @@ -423,30 +424,46 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff( return new CancelSnapshotDiffResponse(reason); } - public List getSnapshotDiffJobList( - String volumeName, String bucketName, - String jobStatus, boolean listAll) throws IOException { - List jobList = new ArrayList<>(); + public ListSnapshotDiffJobResponse getSnapshotDiffJobList(String volumeName, + String bucketName, + String jobStatus, + boolean listAll, + String prevDiffJob, + int maxEntries) throws IOException { + List jobs = new ArrayList<>(); + String lastSnapshotDiffJob = null; try (ClosableIterator> iterator = - snapDiffJobTable.iterator()) { + snapDiffJobTable.iterator(Optional.ofNullable(prevDiffJob), Optional.empty())) { + Map.Entry entry = null; while (iterator.hasNext()) { - SnapshotDiffJob snapshotDiffJob = iterator.next().getValue(); + entry = iterator.next(); + SnapshotDiffJob snapshotDiffJob = entry.getValue(); + if (Objects.equals(prevDiffJob, entry.getKey())) { + continue; + } + if (Objects.equals(snapshotDiffJob.getVolume(), volumeName) && Objects.equals(snapshotDiffJob.getBucket(), bucketName)) { if (listAll) { - jobList.add(snapshotDiffJob); - continue; - } - - if (Objects.equals(snapshotDiffJob.getStatus(), - getJobStatus(jobStatus))) { - jobList.add(snapshotDiffJob); + jobs.add(snapshotDiffJob); + } else if (Objects.equals(snapshotDiffJob.getStatus(), getJobStatus(jobStatus))) { + jobs.add(snapshotDiffJob); } } + if (jobs.size() == maxEntries) { + break; + } + } + + // If maxEntries are populated and list still has more entries, + // set the continuation token for the next page request otherwise null. + if (jobs.size() == maxEntries && iterator.hasNext()) { + lastSnapshotDiffJob = entry.getKey(); } } - return jobList; + + return new ListSnapshotDiffJobResponse(jobs, lastSnapshotDiffJob); } private JobStatus getJobStatus(String jobStatus) @@ -1588,9 +1605,9 @@ private String getTablePrefix(Map tablePrefixes, // the key Prefix would be volumeId/bucketId and // in case of non-fso - volumeName/bucketName if (tableName.equals( - OmMetadataManagerImpl.DIRECTORY_TABLE) || tableName.equals( + DIRECTORY_TABLE) || tableName.equals( OmMetadataManagerImpl.FILE_TABLE)) { - return tablePrefixes.get(OmMetadataManagerImpl.DIRECTORY_TABLE); + return tablePrefixes.get(DIRECTORY_TABLE); } return tablePrefixes.get(OmMetadataManagerImpl.KEY_TABLE); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 5acb9f36510..c2d0bad088d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -1397,19 +1397,26 @@ private CancelSnapshotDiffResponse cancelSnapshotDiff( } private ListSnapshotDiffJobResponse listSnapshotDiffJobs( - ListSnapshotDiffJobRequest listSnapshotDiffJobRequest) - throws IOException { - List snapshotDiffJobs = - impl.listSnapshotDiffJobs( - listSnapshotDiffJobRequest.getVolumeName(), - listSnapshotDiffJobRequest.getBucketName(), - listSnapshotDiffJobRequest.getJobStatus(), - listSnapshotDiffJobRequest.getListAll()); - ListSnapshotDiffJobResponse.Builder builder = - ListSnapshotDiffJobResponse.newBuilder(); - for (SnapshotDiffJob diffJob : snapshotDiffJobs) { + ListSnapshotDiffJobRequest listSnapshotDiffJobRequest + ) throws IOException { + org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse response = impl.listSnapshotDiffJobs( + listSnapshotDiffJobRequest.getVolumeName(), + listSnapshotDiffJobRequest.getBucketName(), + listSnapshotDiffJobRequest.getJobStatus(), + listSnapshotDiffJobRequest.getListAll(), + listSnapshotDiffJobRequest.getPrevSnapshotDiffJob(), + listSnapshotDiffJobRequest.getMaxListResult()); + + ListSnapshotDiffJobResponse.Builder builder = ListSnapshotDiffJobResponse.newBuilder(); + + for (SnapshotDiffJob diffJob : response.getSnapshotDiffJobs()) { builder.addSnapshotDiffJob(diffJob.toProtoBuf()); } + + if (StringUtils.isNotEmpty(response.getLastSnapshotDiffJob())) { + builder.setLastSnapshotDiffJob(response.getLastSnapshotDiffJob()); + } + return builder.build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index a9e67b00cc9..693addc446a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.om.snapshot.SnapshotTestUtils.StubbedPersistentMap; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus; @@ -1131,11 +1132,13 @@ public void testListSnapshotDiffJobs(String jobStatus, SnapshotDiffJob diffJob = snapDiffJobMap.get(diffJobKey); assertNull(diffJob); + ListSnapshotDiffJobResponse snapshotDiffJobList = snapshotDiffManager + .getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAll, null, 1000); // There are no jobs in the table, therefore // the response list should be empty. - List jobList = snapshotDiffManager - .getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAll); + List jobList = snapshotDiffJobList.getSnapshotDiffJobs(); assertThat(jobList).isEmpty(); + assertNull(snapshotDiffJobList.getLastSnapshotDiffJob()); SnapshotDiffManager spy = spy(snapshotDiffManager); doNothing().when(spy).generateSnapshotDiffReport(eq(diffJobKey), @@ -1155,8 +1158,9 @@ public void testListSnapshotDiffJobs(String jobStatus, assertEquals(SnapshotDiffResponse.JobStatus.IN_PROGRESS, diffJob.getStatus()); - jobList = snapshotDiffManager - .getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAll); + snapshotDiffJobList = snapshotDiffManager + .getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAll, null, 1000); + jobList = snapshotDiffJobList.getSnapshotDiffJobs(); // When listAll is true, jobStatus is ignored. // If the job is IN_PROGRESS or listAll is used, @@ -1167,6 +1171,7 @@ public void testListSnapshotDiffJobs(String jobStatus, } else { assertThat(jobList).isEmpty(); } + assertNull(snapshotDiffJobList.getLastSnapshotDiffJob()); } @Test @@ -1194,7 +1199,7 @@ public void testListSnapDiffWithInvalidStatus() throws IOException { // Invalid status, without listAll true, results in an exception. assertThrows(IOException.class, () -> snapshotDiffManager - .getSnapshotDiffJobList(volumeName, bucketName, "invalid", false)); + .getSnapshotDiffJobList(volumeName, bucketName, "invalid", false, null, 1000)); } @Test diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 0400bc60500..0e4d7413895 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.security.token.Token; @@ -704,9 +705,12 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, } @Override - public List listSnapshotDiffJobs( - String volumeName, String bucketName, - String jobStatus, boolean listAll) { + public ListSnapshotDiffJobResponse listSnapshotDiffJobs(String volumeName, + String bucketName, + String jobStatus, + boolean listAll, + String prevSnapshotDiffJob, + int maxListResult) { return null; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotDiffHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotDiffHandler.java index d1e6d5fbd8c..03285e4804d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotDiffHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotDiffHandler.java @@ -18,14 +18,15 @@ package org.apache.hadoop.ozone.shell.snapshot; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; import org.apache.hadoop.ozone.shell.Handler; +import org.apache.hadoop.ozone.shell.ListOptions; import org.apache.hadoop.ozone.shell.OzoneAddress; import org.apache.hadoop.ozone.shell.bucket.BucketUri; +import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; import picocli.CommandLine; import java.io.IOException; -import java.util.List; +import java.util.Iterator; /** * ozone sh snapshot listDiff. @@ -38,16 +39,14 @@ public class ListSnapshotDiffHandler extends Handler { @CommandLine.Mixin private BucketUri snapshotPath; - @CommandLine.Option(names = {"-s", "--status"}, + @CommandLine.Option(names = {"-js", "--job-status"}, description = "List jobs based on status.\n" + "Accepted values are: queued, in_progress, done, failed, rejected", defaultValue = "in_progress") private String jobStatus; - @CommandLine.Option(names = {"-a", "--all"}, - description = "List all jobs regardless of status.", - defaultValue = "false") - private boolean listAll; + @CommandLine.Mixin + private ListOptions listOptions; @Override protected OzoneAddress getAddress() { @@ -55,21 +54,17 @@ protected OzoneAddress getAddress() { } @Override - protected void execute(OzoneClient client, OzoneAddress address) - throws IOException { - + protected void execute(OzoneClient client, OzoneAddress address) throws IOException { String volumeName = snapshotPath.getValue().getVolumeName(); String bucketName = snapshotPath.getValue().getBucketName(); - List jobList = - client.getObjectStore().listSnapshotDiffJobs( - volumeName, bucketName, jobStatus, listAll); + Iterator iterator = client.getObjectStore() + .listSnapshotDiffJobs(volumeName, bucketName, jobStatus, listOptions.isAll(), listOptions.getStartItem()); + + int counter = printAsJsonArray(iterator, listOptions.getLimit()); - int counter = printAsJsonArray(jobList.iterator(), - jobList.size()); if (isVerbose()) { - System.out.printf("Found : %d snapshot diff jobs for o3://%s/ %s ", - counter, volumeName, bucketName); + System.out.printf("Found : %d snapshot diff jobs for o3://%s/ %s ", counter, volumeName, bucketName); } } }