diff --git a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotDiffHandler.java b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotDiffHandler.java index 2f66764dbe43..7955db9c711b 100644 --- a/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotDiffHandler.java +++ b/hadoop-ozone/cli-shell/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotDiffHandler.java @@ -18,10 +18,11 @@ package org.apache.hadoop.ozone.shell.snapshot; import java.io.IOException; -import java.util.List; +import java.util.Iterator; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; import org.apache.hadoop.ozone.shell.Handler; +import org.apache.hadoop.ozone.shell.ListOptions; import org.apache.hadoop.ozone.shell.OzoneAddress; import org.apache.hadoop.ozone.shell.bucket.BucketUri; import picocli.CommandLine; @@ -37,16 +38,19 @@ public class ListSnapshotDiffHandler extends Handler { @CommandLine.Mixin private BucketUri snapshotPath; - @CommandLine.Option(names = {"-s", "--status"}, + @CommandLine.Option(names = {"--job-status"}, description = "List jobs based on status.\n" + "Accepted values are: queued, in_progress, done, failed, rejected", defaultValue = "in_progress") private String jobStatus; - @CommandLine.Option(names = {"-a", "--all"}, + @CommandLine.Option(names = {"--all-status"}, description = "List all jobs regardless of status.", defaultValue = "false") - private boolean listAll; + private boolean listAllStatus; + + @CommandLine.Mixin + private ListOptions listOptions; @Override protected OzoneAddress getAddress() { @@ -54,21 +58,17 @@ protected OzoneAddress getAddress() { } @Override - protected void execute(OzoneClient client, OzoneAddress address) - throws IOException { - + protected void execute(OzoneClient client, OzoneAddress address) throws IOException { String volumeName = snapshotPath.getValue().getVolumeName(); String bucketName = snapshotPath.getValue().getBucketName(); - List jobList = - client.getObjectStore().listSnapshotDiffJobs( - volumeName, bucketName, jobStatus, listAll); + Iterator iterator = client.getObjectStore() + .listSnapshotDiffJobs(volumeName, bucketName, jobStatus, listAllStatus, listOptions.getStartItem()); + + int counter = printAsJsonArray(iterator, listOptions.getLimit()); - int counter = printAsJsonArray(jobList.iterator(), - jobList.size()); if (isVerbose()) { - System.out.printf("Found : %d snapshot diff jobs for o3://%s/ %s ", - counter, volumeName, bucketName); + System.out.printf("Found : %d snapshot diff jobs for o3://%s/ %s ", counter, volumeName, bucketName); } } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java index b3804f2589ae..456dc9162145 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java @@ -47,6 +47,7 @@ import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.security.UserGroupInformation; @@ -734,16 +735,73 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, * @param volumeName Name of the volume to which the snapshotted bucket belong * @param bucketName Name of the bucket to which the snapshots belong * @param jobStatus JobStatus to be used to filter the snapshot diff jobs - * @param listAll Option to specify whether to list all jobs or not - * @return a list of SnapshotDiffJob objects + * @param listAllStatus Option to specify whether to list all jobs regardless of status + * @param prevSnapshotDiffJob list snapshot diff jobs after this snapshot diff job. + * @return an iterator of SnapshotDiffJob objects * @throws IOException in case there is a failure while getting a response. */ - public List listSnapshotDiffJobs(String volumeName, - String bucketName, - String jobStatus, - boolean listAll) - throws IOException { - return proxy.listSnapshotDiffJobs(volumeName, - bucketName, jobStatus, listAll); + public Iterator listSnapshotDiffJobs( + String volumeName, + String bucketName, + String jobStatus, + boolean listAllStatus, + String prevSnapshotDiffJob + ) throws IOException { + return new SnapshotDiffJobIterator(volumeName, bucketName, jobStatus, listAllStatus, prevSnapshotDiffJob); + } + + /** + * An Iterator to iterate over {@link SnapshotDiffJobIterator} list. + */ + private final class SnapshotDiffJobIterator implements Iterator { + private final String volumeName; + private final String bucketName; + private final String jobStatus; + private final boolean listAllJobs; + private String lastSnapshotDiffJob; + private Iterator currentIterator; + + private SnapshotDiffJobIterator( + String volumeName, + String bucketName, + String jobStatus, + boolean listAllStatus, + String prevSnapshotDiffJob) throws IOException { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.jobStatus = jobStatus; + this.listAllJobs = listAllStatus; + // Initialized the currentIterator and lastSnapshotDiffJob. + getNextListOfSnapshotDiffJobs(prevSnapshotDiffJob); + } + + @Override + public boolean hasNext() { + if (!currentIterator.hasNext() && StringUtils.isNotEmpty(lastSnapshotDiffJob)) { + try { + // fetch the next page if continuationToken is not null. + getNextListOfSnapshotDiffJobs(lastSnapshotDiffJob); + } catch (IOException e) { + LOG.error("Error retrieving next batch of list for snapshot diff jobs.", e); + } + } + return currentIterator.hasNext(); + } + + @Override + public OzoneSnapshotDiff next() { + if (hasNext()) { + return currentIterator.next(); + } + throw new NoSuchElementException(); + } + + private void getNextListOfSnapshotDiffJobs(String prevSnapshotDiffJob) throws IOException { + ListSnapshotDiffJobResponse response = proxy.listSnapshotDiffJobs(volumeName, bucketName, jobStatus, listAllJobs, + prevSnapshotDiffJob, listCacheSize); + this.currentIterator = + response.getSnapshotDiffJobs().stream().map(OzoneSnapshotDiff::fromSnapshotDiffJob).iterator(); + this.lastSnapshotDiffJob = response.getLastSnapshotDiffJob(); + } } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 7ef2c38eb32b..a1155fd8d393 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -38,7 +38,6 @@ import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts; import org.apache.hadoop.ozone.client.OzoneSnapshot; -import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.TenantArgs; import org.apache.hadoop.ozone.client.VolumeArgs; @@ -68,6 +67,7 @@ import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.security.KerberosInfo; @@ -1265,15 +1265,19 @@ CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, * @param volumeName Name of the volume to which the snapshotted bucket belong * @param bucketName Name of the bucket to which the snapshots belong * @param jobStatus JobStatus to be used to filter the snapshot diff jobs - * @param listAll Option to specify whether to list all jobs or not + * @param listAllStatus Option to specify whether to list all jobs regardless of status + * @param prevSnapshotDiffJob list snapshot diff jobs after this snapshot diff job. + * @param maxListResult maximum entries to be returned from the startSnapshotDiffJob. * @return a list of SnapshotDiffJob objects * @throws IOException in case there is a failure while getting a response. */ - List listSnapshotDiffJobs(String volumeName, - String bucketName, - String jobStatus, - boolean listAll) - throws IOException; + ListSnapshotDiffJobResponse listSnapshotDiffJobs( + String volumeName, + String bucketName, + String jobStatus, + boolean listAllStatus, + String prevSnapshotDiffJob, + int maxListResult) throws IOException; /** * Time to be set for given Ozone object. This operations updates modification diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 4c0e99a44693..e32a40d81e0a 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -105,7 +105,6 @@ import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts; import org.apache.hadoop.ozone.client.OzoneSnapshot; -import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.TenantArgs; import org.apache.hadoop.ozone.client.VolumeArgs; @@ -171,6 +170,7 @@ import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.security.UserGroupInformation; @@ -1071,20 +1071,19 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, } @Override - public List listSnapshotDiffJobs(String volumeName, - String bucketName, - String jobStatus, - boolean listAll) - throws IOException { + public ListSnapshotDiffJobResponse listSnapshotDiffJobs( + String volumeName, + String bucketName, + String jobStatus, + boolean listAllStatus, + String prevSnapshotDiffJob, + int maxListResult) throws IOException { Preconditions.checkArgument(StringUtils.isNotBlank(volumeName), "volume can't be null or empty."); Preconditions.checkArgument(StringUtils.isNotBlank(bucketName), "bucket can't be null or empty."); - - return ozoneManagerClient.listSnapshotDiffJobs( - volumeName, bucketName, jobStatus, listAll).stream() - .map(OzoneSnapshotDiff::fromSnapshotDiffJob) - .collect(Collectors.toList()); + return ozoneManagerClient.listSnapshotDiffJobs(volumeName, bucketName, jobStatus, listAllStatus, + prevSnapshotDiffJob, maxListResult); } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 37cd50d10d2f..840f3fa00e26 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -57,7 +57,6 @@ import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; -import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.TenantStateList; import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue; @@ -71,6 +70,7 @@ import org.apache.hadoop.ozone.security.OzoneDelegationTokenSelector; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.ozone.upgrade.UpgradeFinalization; @@ -849,14 +849,19 @@ default CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, * @param volumeName Name of the volume to which the snapshotted bucket belong * @param bucketName Name of the bucket to which the snapshots belong * @param jobStatus JobStatus to be used to filter the snapshot diff jobs + * @param listAllStatus Option to specify whether to list all jobs regardless of status + * @param prevSnapshotDiffJob list snapshot diff jobs after this snapshot diff job. + * @param maxListResult maximum entries to be returned from the startSnapshotDiffJob. * @return a list of SnapshotDiffJob objects * @throws IOException in case there is a failure while getting a response. */ - default List listSnapshotDiffJobs(String volumeName, - String bucketName, - String jobStatus, - boolean listAll) - throws IOException { + default ListSnapshotDiffJobResponse listSnapshotDiffJobs( + String volumeName, + String bucketName, + String jobStatus, + boolean listAllStatus, + String prevSnapshotDiffJob, + int maxListResult) throws IOException { throw new UnsupportedOperationException("OzoneManager does not require " + "this to be implemented"); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 9e2227525c6d..1c5b11fcb100 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -239,6 +239,7 @@ import org.apache.hadoop.ozone.security.proto.SecurityProtos.GetDelegationTokenRequestProto; import org.apache.hadoop.ozone.security.proto.SecurityProtos.RenewDelegationTokenRequestProto; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; @@ -1473,11 +1474,13 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, * {@inheritDoc} */ @Override - public List listSnapshotDiffJobs(String volumeName, - String bucketName, - String jobStatus, - boolean listAll) - throws IOException { + public ListSnapshotDiffJobResponse listSnapshotDiffJobs( + String volumeName, + String bucketName, + String jobStatus, + boolean listAllStatus, + String prevSnapshotDiffJob, + int maxListResult) throws IOException { final OzoneManagerProtocolProtos .ListSnapshotDiffJobRequest.Builder requestBuilder = OzoneManagerProtocolProtos @@ -1485,17 +1488,24 @@ public List listSnapshotDiffJobs(String volumeName, .setVolumeName(volumeName) .setBucketName(bucketName) .setJobStatus(jobStatus) - .setListAll(listAll); + .setListAll(listAllStatus) + .setMaxListResult(maxListResult); + + if (prevSnapshotDiffJob != null) { + requestBuilder.setPrevSnapshotDiffJob(prevSnapshotDiffJob); + } final OMRequest omRequest = createOMRequest(Type.ListSnapshotDiffJobs) .setListSnapshotDiffJobRequest(requestBuilder) .build(); final OMResponse omResponse = submitRequest(omRequest); handleError(omResponse); - return omResponse.getListSnapshotDiffJobResponse() - .getSnapshotDiffJobList().stream() + OzoneManagerProtocolProtos.ListSnapshotDiffJobResponse response = omResponse.getListSnapshotDiffJobResponse(); + List ozoneSnapshotDiffs = response.getSnapshotDiffJobList().stream() .map(SnapshotDiffJob::getFromProtoBuf) .collect(Collectors.toList()); + return new ListSnapshotDiffJobResponse(ozoneSnapshotDiffs, + response.hasLastSnapshotDiffJob() ? response.getLastSnapshotDiffJob() : null); } /** diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/ListSnapshotDiffJobResponse.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/ListSnapshotDiffJobResponse.java new file mode 100644 index 000000000000..0cc24cc723a6 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/ListSnapshotDiffJobResponse.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.snapshot; + +import java.util.List; +import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; + +/** + * POJO for list snapshot diff job API. + */ +public final class ListSnapshotDiffJobResponse { + private final List snapshotDiffJobs; + private final String lastSnapshotDiffJob; + + public ListSnapshotDiffJobResponse(List snapshotDiffJobs, String lastSnapshotDiffJob) { + this.snapshotDiffJobs = snapshotDiffJobs; + this.lastSnapshotDiffJob = lastSnapshotDiffJob; + } + + public List getSnapshotDiffJobs() { + return snapshotDiffJobs; + } + + public String getLastSnapshotDiffJob() { + return lastSnapshotDiffJob; + } + + @Override + public String toString() { + return "ListSnapshotDiffJobResponse{" + + "snapshotDiffJobs: '" + snapshotDiffJobs + '\'' + + ", lastSnapshotDiffJob: '" + lastSnapshotDiffJob + '\'' + + '}'; + } +} diff --git a/hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot b/hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot index 5a3d36b16fd7..c451932b3026 100644 --- a/hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot +++ b/hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot @@ -23,6 +23,7 @@ Test Timeout 10 minutes *** Variables *** ${SNAPSHOT_ONE} ${SNAPSHOT_TWO} +${SNAPSHOT_THREE} ${KEY_ONE} ${KEY_TWO} ${KEY_THREE} @@ -47,11 +48,12 @@ Snapshot Diff Set Suite Variable ${KEY_THREE} ${key_three} ${snapshot_two} = Create snapshot ${VOLUME} ${BUCKET} Set Suite Variable ${SNAPSHOT_TWO} ${snapshot_two} + ${snapshot_three} = Create snapshot ${VOLUME} ${BUCKET} + Set Suite Variable ${SNAPSHOT_THREE} ${snapshot_three} ${result} = Execute ozone sh snapshot diff /${VOLUME}/${BUCKET} ${SNAPSHOT_ONE} ${SNAPSHOT_TWO} Should contain ${result} Snapshot diff job is IN_PROGRESS - ${result} = Execute ozone sh snapshot diff /${VOLUME}/${BUCKET} ${SNAPSHOT_ONE} ${SNAPSHOT_TWO} - Should contain ${result} + ${KEY_TWO} - Should contain ${result} + ${KEY_THREE} + ${result} = Execute ozone sh snapshot diff /${VOLUME}/${BUCKET} ${SNAPSHOT_ONE} ${SNAPSHOT_THREE} + Should contain ${result} Snapshot diff job is IN_PROGRESS Snapshot Diff as JSON ${result} = Execute ozone sh snapshot diff --json /${VOLUME}/${BUCKET} ${SNAPSHOT_ONE} ${SNAPSHOT_TWO} @@ -62,13 +64,18 @@ Snapshot Diff as JSON Should contain echo '${result}' | jq '.snapshotDiffReport.toSnapshot' ${SNAPSHOT_TWO} Should contain echo '${result}' | jq '.snapshotDiffReport.diffList | .[].sourcePath' ${KEY_TWO} Should contain echo '${result}' | jq '.snapshotDiffReport.diffList | .[].sourcePath' ${KEY_THREE} + ${result} = Execute ozone sh snapshot diff --json /${VOLUME}/${BUCKET} ${SNAPSHOT_ONE} ${SNAPSHOT_TWO} + Should contain echo '${result}' | jq '.jobStatus' DONE List Snapshot Diff Jobs - ${result} = Execute ozone sh snapshot listDiff /${VOLUME}/${BUCKET} --all + ${result} = Execute ozone sh snapshot listDiff /${VOLUME}/${BUCKET} --all-status Should contain ${result} ${VOLUME} Should contain ${result} ${BUCKET} Should contain ${result} ${SNAPSHOT_ONE} Should contain ${result} ${SNAPSHOT_TWO} + Should contain ${result} ${SNAPSHOT_THREE} + ${result} = Execute ozone sh snapshot listDiff /${VOLUME}/${BUCKET} --all-status -l=1 | jq 'length' + Should contain ${result} 1 Read Snapshot Key Should Match Local File /${VOLUME}/${BUCKET}/${SNAPSHOT_INDICATOR}/${SNAPSHOT_ONE}/${KEY_ONE} /etc/hosts diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java index d8b05683ff4d..5ba60252f188 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java @@ -51,6 +51,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -107,6 +108,7 @@ import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneSnapshot; +import org.apache.hadoop.ozone.client.OzoneSnapshotDiff; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; @@ -1718,10 +1720,8 @@ public void testListSnapshotDiffWithInvalidParameters() String volErrorMessage = "Volume not found: " + volume; Exception volBucketEx = assertThrows(OMException.class, - () -> store.listSnapshotDiffJobs(volume, bucket, - "", true)); - assertEquals(volErrorMessage, - volBucketEx.getMessage()); + () -> store.listSnapshotDiffJobs(volume, bucket, "", true, null)); + assertEquals(volErrorMessage, volBucketEx.getMessage()); // Create the volume and the bucket. store.createVolume(volume); @@ -1729,12 +1729,11 @@ public void testListSnapshotDiffWithInvalidParameters() createBucket(ozVolume, bucket); assertDoesNotThrow(() -> - store.listSnapshotDiffJobs(volume, bucket, "", true)); + store.listSnapshotDiffJobs(volume, bucket, "", true, null)); // There are no snapshots, response should be empty. - assertTrue(store - .listSnapshotDiffJobs(volume, bucket, - "", true).isEmpty()); + Iterator iterator = store.listSnapshotDiffJobs(volume, bucket, "", true, null); + assertFalse(iterator.hasNext()); OzoneBucket ozBucket = ozVolume.getBucket(bucket); // Create keys and take snapshots. @@ -1755,8 +1754,7 @@ public void testListSnapshotDiffWithInvalidParameters() String statusErrorMessage = "Invalid job status: " + invalidStatus; OMException statusEx = assertThrows(OMException.class, - () -> store.listSnapshotDiffJobs(volume, bucket, - invalidStatus, false)); + () -> store.listSnapshotDiffJobs(volume, bucket, invalidStatus, false, null)); assertEquals(statusErrorMessage, statusEx.getMessage()); } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index df97028a0f31..f885f5afd7c6 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1979,7 +1979,9 @@ message ListSnapshotDiffJobRequest { required string volumeName = 1; required string bucketName = 2; optional string jobStatus = 3; - optional bool listAll = 4; + optional bool listAll = 4; // Option to specify whether to list all jobs regardless of status + optional string prevSnapshotDiffJob = 5; + optional uint32 maxListResult = 6; } message DeleteSnapshotRequest { @@ -2113,6 +2115,7 @@ message CancelSnapshotDiffResponse { message ListSnapshotDiffJobResponse { repeated SnapshotDiffJobProto snapshotDiffJob = 1; + optional string lastSnapshotDiffJob = 2; } message DeleteSnapshotResponse { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java index 11bcd51c9923..2f17ecdb18dd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java @@ -93,6 +93,7 @@ import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer; @@ -412,6 +413,10 @@ private static CodecRegistry createCodecRegistryForSnapDiff() { return registry.build(); } + public int getMaxPageSize() { + return maxPageSize; + } + /** * Get snapshot instance LRU cache size. * @return cache size. @@ -802,18 +807,20 @@ public SnapshotDiffResponse getSnapshotDiffReport(final String volume, return snapshotDiffReport; } - public List getSnapshotDiffList(final String volumeName, - final String bucketName, - final String jobStatus, - final boolean listAll) - throws IOException { + public ListSnapshotDiffJobResponse getSnapshotDiffList( + final String volumeName, + final String bucketName, + final String jobStatus, + final boolean listAllStatus, + final String prevSnapshotDiffJob, + int maxListResult) throws IOException { String volumeKey = ozoneManager.getMetadataManager() .getVolumeKey(volumeName); String bucketKey = ozoneManager.getMetadataManager() .getBucketKey(volumeName, bucketName); if (!ozoneManager.getMetadataManager() - .getVolumeTable().isExist(volumeKey) || + .getVolumeTable().isExist(volumeKey) || !ozoneManager.getMetadataManager() .getBucketTable().isExist(bucketKey)) { throw new IOException("Provided volume name " + volumeName + @@ -827,11 +834,15 @@ public List getSnapshotDiffList(final String volumeName, if (snapshotChainManager.getSnapshotChainPath(snapshotPath) == null) { // Return an empty ArrayList here to avoid // unnecessarily iterating the SnapshotDiffJob table. - return new ArrayList<>(); + return new ListSnapshotDiffJobResponse(Collections.emptyList(), null); + } + + if (maxListResult <= 0 || maxListResult > maxPageSize) { + maxListResult = maxPageSize; } - return snapshotDiffManager.getSnapshotDiffJobList( - volumeName, bucketName, jobStatus, listAll); + return snapshotDiffManager.getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAllStatus, + prevSnapshotDiffJob, maxListResult); } private void validateSnapshotsExistAndActive(final String volumeName, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index eb5d83b5a8ab..b91ab22c4d08 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -250,7 +250,6 @@ import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; -import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.helpers.TenantStateList; import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue; @@ -305,6 +304,7 @@ import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; import org.apache.hadoop.ozone.security.acl.RequestContext; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.PersistedUserVolumeInfo; @@ -4913,18 +4913,20 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volume, } @Override - public List listSnapshotDiffJobs(String volume, - String bucket, - String jobStatus, - boolean listAll) - throws IOException { + public ListSnapshotDiffJobResponse listSnapshotDiffJobs( + String volume, + String bucket, + String jobStatus, + boolean listAllStatus, + String prevSnapshotDiffJob, + int maxListResult) throws IOException { ResolvedBucket resolvedBucket = this.resolveBucketLink(Pair.of(volume, bucket), false); if (isAclEnabled) { omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.LIST, volume, bucket, null); } return omSnapshotManager.getSnapshotDiffList(resolvedBucket.realVolume(), resolvedBucket.realBucket(), - jobStatus, listAll); + jobStatus, listAllStatus, prevSnapshotDiffJob, maxListResult); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 99f34d863de9..6f4a2f6b53ab 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -112,6 +112,7 @@ import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; import org.apache.hadoop.ozone.om.service.SnapshotDeletingService; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus; @@ -422,30 +423,45 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff( return new CancelSnapshotDiffResponse(reason); } - public List getSnapshotDiffJobList( - String volumeName, String bucketName, - String jobStatus, boolean listAll) throws IOException { - List jobList = new ArrayList<>(); + public ListSnapshotDiffJobResponse getSnapshotDiffJobList( + String volumeName, + String bucketName, + String jobStatus, + boolean listAllStatus, + String prevDiffJob, + int maxEntries) throws IOException { + List jobs = new ArrayList<>(); + String lastSnapshotDiffJob = null; try (ClosableIterator> iterator = - snapDiffJobTable.iterator()) { - while (iterator.hasNext()) { - SnapshotDiffJob snapshotDiffJob = iterator.next().getValue(); + snapDiffJobTable.iterator(Optional.ofNullable(prevDiffJob), Optional.empty())) { + Map.Entry entry = null; + while (iterator.hasNext() && jobs.size() < maxEntries) { + entry = iterator.next(); + SnapshotDiffJob snapshotDiffJob = entry.getValue(); + if (Objects.equals(prevDiffJob, entry.getKey())) { + continue; + } + if (Objects.equals(snapshotDiffJob.getVolume(), volumeName) && Objects.equals(snapshotDiffJob.getBucket(), bucketName)) { - if (listAll) { - jobList.add(snapshotDiffJob); - continue; - } - - if (Objects.equals(snapshotDiffJob.getStatus(), - getJobStatus(jobStatus))) { - jobList.add(snapshotDiffJob); + if (listAllStatus) { + jobs.add(snapshotDiffJob); + } else if (Objects.equals(snapshotDiffJob.getStatus(), getJobStatus(jobStatus))) { + jobs.add(snapshotDiffJob); } } } + + // If maxEntries are populated and list still has more entries, + // set the continuation token for the next page request otherwise null. + if (iterator.hasNext()) { + assert entry != null; + lastSnapshotDiffJob = entry.getKey(); + } } - return jobList; + + return new ListSnapshotDiffJobResponse(jobs, lastSnapshotDiffJob); } private JobStatus getJobStatus(String jobStatus) @@ -1563,9 +1579,9 @@ private String getTablePrefix(Map tablePrefixes, // the key Prefix would be volumeId/bucketId and // in case of non-fso - volumeName/bucketName if (tableName.equals( - OmMetadataManagerImpl.DIRECTORY_TABLE) || tableName.equals( + DIRECTORY_TABLE) || tableName.equals( OmMetadataManagerImpl.FILE_TABLE)) { - return tablePrefixes.get(OmMetadataManagerImpl.DIRECTORY_TABLE); + return tablePrefixes.get(DIRECTORY_TABLE); } return tablePrefixes.get(OmMetadataManagerImpl.KEY_TABLE); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 67190972ce23..2810f737388f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -1436,19 +1436,31 @@ private CancelSnapshotDiffResponse cancelSnapshotDiff( } private ListSnapshotDiffJobResponse listSnapshotDiffJobs( - ListSnapshotDiffJobRequest listSnapshotDiffJobRequest) - throws IOException { - List snapshotDiffJobs = - impl.listSnapshotDiffJobs( - listSnapshotDiffJobRequest.getVolumeName(), - listSnapshotDiffJobRequest.getBucketName(), - listSnapshotDiffJobRequest.getJobStatus(), - listSnapshotDiffJobRequest.getListAll()); - ListSnapshotDiffJobResponse.Builder builder = - ListSnapshotDiffJobResponse.newBuilder(); - for (SnapshotDiffJob diffJob : snapshotDiffJobs) { + ListSnapshotDiffJobRequest listSnapshotDiffJobRequest + ) throws IOException { + String prevSnapshotDiffJob = listSnapshotDiffJobRequest.hasPrevSnapshotDiffJob() ? + listSnapshotDiffJobRequest.getPrevSnapshotDiffJob() : null; + int maxListResult = listSnapshotDiffJobRequest.hasMaxListResult() ? + listSnapshotDiffJobRequest.getMaxListResult() : impl.getOmSnapshotManager().getMaxPageSize(); + + org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse response = impl.listSnapshotDiffJobs( + listSnapshotDiffJobRequest.getVolumeName(), + listSnapshotDiffJobRequest.getBucketName(), + listSnapshotDiffJobRequest.getJobStatus(), + listSnapshotDiffJobRequest.getListAll(), + prevSnapshotDiffJob, + maxListResult); + + ListSnapshotDiffJobResponse.Builder builder = ListSnapshotDiffJobResponse.newBuilder(); + + for (SnapshotDiffJob diffJob : response.getSnapshotDiffJobs()) { builder.addSnapshotDiffJob(diffJob.toProtoBuf()); } + + if (StringUtils.isNotEmpty(response.getLastSnapshotDiffJob())) { + builder.setLastSnapshotDiffJob(response.getLastSnapshotDiffJob()); + } + return builder.build(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 674c00fc4f41..2ca6058d3d2e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -136,6 +136,7 @@ import org.apache.hadoop.ozone.om.snapshot.SnapshotTestUtils.StubbedPersistentMap; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus; @@ -1121,7 +1122,7 @@ private static Stream listSnapshotDiffJobsScenarios() { @ParameterizedTest @MethodSource("listSnapshotDiffJobsScenarios") public void testListSnapshotDiffJobs(String jobStatus, - boolean listAll, + boolean listAllStatus, boolean containsJob) throws IOException { String volumeName = "vol-" + RandomStringUtils.randomNumeric(5); @@ -1142,11 +1143,13 @@ public void testListSnapshotDiffJobs(String jobStatus, SnapshotDiffJob diffJob = snapDiffJobMap.get(diffJobKey); assertNull(diffJob); + ListSnapshotDiffJobResponse snapshotDiffJobList = snapshotDiffManager + .getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAllStatus, null, 1000); // There are no jobs in the table, therefore // the response list should be empty. - List jobList = snapshotDiffManager - .getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAll); + List jobList = snapshotDiffJobList.getSnapshotDiffJobs(); assertThat(jobList).isEmpty(); + assertNull(snapshotDiffJobList.getLastSnapshotDiffJob()); SnapshotDiffManager spy = spy(snapshotDiffManager); doNothing().when(spy).generateSnapshotDiffReport(eq(diffJobKey), @@ -1166,11 +1169,12 @@ public void testListSnapshotDiffJobs(String jobStatus, assertEquals(SnapshotDiffResponse.JobStatus.IN_PROGRESS, diffJob.getStatus()); - jobList = snapshotDiffManager - .getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAll); + snapshotDiffJobList = snapshotDiffManager + .getSnapshotDiffJobList(volumeName, bucketName, jobStatus, listAllStatus, null, 1000); + jobList = snapshotDiffJobList.getSnapshotDiffJobs(); - // When listAll is true, jobStatus is ignored. - // If the job is IN_PROGRESS or listAll is used, + // When listAllStatus is true, jobStatus is ignored. + // If the job is IN_PROGRESS or listAllStatus is used, // there should be a response. // Otherwise, response list should be empty. if (containsJob) { @@ -1178,6 +1182,7 @@ public void testListSnapshotDiffJobs(String jobStatus, } else { assertThat(jobList).isEmpty(); } + assertNull(snapshotDiffJobList.getLastSnapshotDiffJob()); } @Test @@ -1203,9 +1208,9 @@ public void testListSnapDiffWithInvalidStatus() throws IOException { spy.getSnapshotDiffReport(volumeName, bucketName, fromSnapshotName, toSnapshotName, 0, 0, false, false); - // Invalid status, without listAll true, results in an exception. + // Invalid status, without listAllStatus true, results in an exception. assertThrows(IOException.class, () -> snapshotDiffManager - .getSnapshotDiffJobList(volumeName, bucketName, "invalid", false)); + .getSnapshotDiffJobList(volumeName, bucketName, "invalid", false, null, 1000)); } @Test diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index c15a85dc2e36..b11322a351c0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -58,6 +58,7 @@ import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.ozone.security.acl.OzoneObj; import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse; +import org.apache.hadoop.ozone.snapshot.ListSnapshotDiffJobResponse; import org.apache.hadoop.ozone.snapshot.ListSnapshotResponse; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; import org.apache.hadoop.security.token.Token; @@ -751,9 +752,13 @@ public CancelSnapshotDiffResponse cancelSnapshotDiff(String volumeName, } @Override - public List listSnapshotDiffJobs( - String volumeName, String bucketName, - String jobStatus, boolean listAll) { + public ListSnapshotDiffJobResponse listSnapshotDiffJobs( + String volumeName, + String bucketName, + String jobStatus, + boolean listAllStatus, + String prevSnapshotDiffJob, + int maxListResult) { return null; }