diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java index 3a0a5cd8e818..d0eac8e8814e 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java @@ -551,14 +551,81 @@ public void deleteSnapshot(String volumeName, /** * List snapshots in a volume/bucket. - * @param volumeName volume name - * @param bucketName bucket name - * @return list of snapshots for volume/bucket snapshotpath. + * @param volumeName volume name + * @param bucketName bucket name + * @param snapshotPrefix snapshot prefix to match + * @param prevSnapshot snapshots will be listed after this snapshot name * @throws IOException */ - public List listSnapshot(String volumeName, String bucketName) - throws IOException { - return proxy.listSnapshot(volumeName, bucketName); + public Iterator listSnapshot( + String volumeName, String bucketName, String snapshotPrefix, + String prevSnapshot) throws IOException { + return new SnapshotIterator( + volumeName, bucketName, snapshotPrefix, prevSnapshot); + } + + /** + * An Iterator to iterate over {@link OzoneSnapshot} list. + */ + private class SnapshotIterator implements Iterator { + + private String volumeName = null; + private String bucketName = null; + private String snapshotPrefix = null; + + private Iterator currentIterator; + private OzoneSnapshot currentValue; + + /** + * Creates an Iterator to iterate over all snapshots after + * prevSnapshot of specified bucket. If prevSnapshot is null it iterates + * from the first snapshot. The returned snapshots match snapshot prefix. + * @param snapshotPrefix snapshot prefix to match + * @param prevSnapshot snapshots will be listed after this snapshot name + */ + SnapshotIterator(String volumeName, String bucketName, + String snapshotPrefix, String prevSnapshot) { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.snapshotPrefix = snapshotPrefix; + this.currentValue = null; + this.currentIterator = getNextListOfSnapshots(prevSnapshot).iterator(); + } + + @Override + public boolean hasNext() { + // IMPORTANT: Without this logic, remote iteration will not work. + // Removing this will break the listSnapshot call if we try to + // list more than 1000 (ozone.client.list.cache ) snapshots. + if (!currentIterator.hasNext() && currentValue != null) { + currentIterator = getNextListOfSnapshots(currentValue.getName()) + .iterator(); + } + return currentIterator.hasNext(); + } + + @Override + public OzoneSnapshot next() { + if (hasNext()) { + currentValue = currentIterator.next(); + return currentValue; + } + throw new NoSuchElementException(); + } + + /** + * Returns the next set of snapshot list using proxy. + * @param prevSnapshot previous snapshot, this will be excluded from result + * @return {@code List} + */ + private List getNextListOfSnapshots(String prevSnapshot) { + try { + return proxy.listSnapshot(volumeName, bucketName, snapshotPrefix, + prevSnapshot, listCacheSize); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } public SnapshotDiffReport snapshotDiff(String volumeName, String bucketName, diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index b0400ac32890..4ae176f94fe2 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -1042,13 +1042,17 @@ void deleteSnapshot(String volumeName, /** * List snapshots in a volume/bucket. - * @param volumeName volume name - * @param bucketName bucket name + * @param volumeName volume name + * @param bucketName bucket name + * @param snapshotPrefix snapshot prefix to match + * @param prevSnapshot start of the list, this snapshot is excluded + * @param maxListResult max numbet of snapshots to return * @return list of snapshots for volume/bucket snapshotpath. * @throws IOException */ - List listSnapshot(String volumeName, String bucketName) - throws IOException; + List listSnapshot( + String volumeName, String bucketName, String snapshotPrefix, + String prevSnapshot, int maxListResult) throws IOException; /** diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 4d1596c216db..ee4c24331055 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -985,19 +985,24 @@ public SnapshotDiffReport snapshotDiff(String volumeName, String bucketName, /** * List snapshots in a volume/bucket. - * @param volumeName volume name - * @param bucketName bucket name + * @param volumeName volume name + * @param bucketName bucket name + * @param snapshotPrefix snapshot prefix to match + * @param prevSnapshot start of the list, this snapshot is excluded + * @param maxListResult max numbet of snapshots to return * @return list of snapshots for volume/bucket snapshotpath. * @throws IOException */ @Override - public List listSnapshot(String volumeName, String bucketName) - throws IOException { + public List listSnapshot( + String volumeName, String bucketName, String snapshotPrefix, + String prevSnapshot, int maxListResult) throws IOException { Preconditions.checkArgument(Strings.isNotBlank(volumeName), "volume can't be null or empty."); Preconditions.checkArgument(Strings.isNotBlank(bucketName), "bucket can't be null or empty."); - return ozoneManagerClient.listSnapshot(volumeName, bucketName).stream() + return ozoneManagerClient.listSnapshot(volumeName, bucketName, + snapshotPrefix, prevSnapshot, maxListResult).stream() .map(snapshotInfo -> OzoneSnapshot.fromSnapshotInfo(snapshotInfo)) .collect(Collectors.toList()); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index ed08f66013fc..bcab84798644 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -664,13 +664,17 @@ default void deleteSnapshot(String volumeName, /** * List snapshots in a volume/bucket. - * @param volumeName volume name - * @param bucketName bucket name + * @param volumeName volume name + * @param bucketName bucket name + * @param snapshotPrefix snapshot prefix to match + * @param prevSnapshot start of the list, this snapshot is excluded + * @param maxListResult max numbet of snapshots to return * @return list of snapshots for volume/bucket snapshotpath. * @throws IOException */ - default List listSnapshot(String volumeName, String bucketName) - throws IOException { + default List listSnapshot( + String volumeName, String bucketName, String snapshotPrefix, + String prevSnapshot, int maxListResult) throws IOException { throw new UnsupportedOperationException("OzoneManager does not require " + "this to be implemented"); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 998bca29d886..c20311d50006 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1153,13 +1153,23 @@ public void deleteSnapshot(String volumeName, * {@inheritDoc} */ @Override - public List listSnapshot(String volumeName, String bucketName) - throws IOException { + public List listSnapshot( + String volumeName, String bucketName, String snapshotPrefix, + String prevSnapshot, int maxListResult) throws IOException { final OzoneManagerProtocolProtos.ListSnapshotRequest.Builder requestBuilder = OzoneManagerProtocolProtos.ListSnapshotRequest.newBuilder() .setVolumeName(volumeName) - .setBucketName(bucketName); + .setBucketName(bucketName) + .setMaxListResult(maxListResult); + + if (prevSnapshot != null) { + requestBuilder.setPrevSnapshot(prevSnapshot); + } + + if (snapshotPrefix != null) { + requestBuilder.setPrefix(snapshotPrefix); + } final OMRequest omRequest = createOMRequest(Type.ListSnapshot) .setListSnapshotRequest(requestBuilder) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index 029768bfdaa7..92d3b2cee758 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -76,6 +76,7 @@ import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneKeyLocation; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts; +import org.apache.hadoop.ozone.client.OzoneSnapshot; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.VolumeArgs; import org.apache.hadoop.ozone.client.io.OzoneInputStream; @@ -4117,4 +4118,86 @@ private OzoneBucket getBucket(OzoneVolume volume) throws IOException { private static ReplicationConfig anyReplication() { return RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE); } + + @Test + public void testListSnapshot() throws IOException { + String volumeA = "vol-a-" + RandomStringUtils.randomNumeric(5); + String volumeB = "vol-b-" + RandomStringUtils.randomNumeric(5); + String bucketA = "buc-a-" + RandomStringUtils.randomNumeric(5); + String bucketB = "buc-b-" + RandomStringUtils.randomNumeric(5); + store.createVolume(volumeA); + store.createVolume(volumeB); + OzoneVolume volA = store.getVolume(volumeA); + OzoneVolume volB = store.getVolume(volumeB); + volA.createBucket(bucketA); + volA.createBucket(bucketB); + volB.createBucket(bucketA); + volB.createBucket(bucketB); + String snapshotPrefixA = "snapshot-a-"; + String snapshotPrefixB = "snapshot-b-"; + for (int i = 0; i < 10; i++) { + store.createSnapshot(volumeA, bucketA, + snapshotPrefixA + i + "-" + RandomStringUtils.randomNumeric(5)); + store.createSnapshot(volumeA, bucketB, + snapshotPrefixA + i + "-" + RandomStringUtils.randomNumeric(5)); + store.createSnapshot(volumeB, bucketA, + snapshotPrefixA + i + "-" + RandomStringUtils.randomNumeric(5)); + store.createSnapshot(volumeB, bucketB, + snapshotPrefixA + i + "-" + RandomStringUtils.randomNumeric(5)); + } + for (int i = 0; i < 10; i++) { + store.createSnapshot(volumeA, bucketA, + snapshotPrefixB + i + "-" + RandomStringUtils.randomNumeric(5)); + store.createSnapshot(volumeA, bucketB, + snapshotPrefixB + i + "-" + RandomStringUtils.randomNumeric(5)); + store.createSnapshot(volumeB, bucketA, + snapshotPrefixB + i + "-" + RandomStringUtils.randomNumeric(5)); + store.createSnapshot(volumeB, bucketB, + snapshotPrefixB + i + "-" + RandomStringUtils.randomNumeric(5)); + } + + Iterator snapshotIter = + store.listSnapshot(volumeA, bucketA, null, null); + int volABucketASnapshotCount = 0; + while (snapshotIter.hasNext()) { + OzoneSnapshot snapshot = snapshotIter.next(); + volABucketASnapshotCount++; + } + Assert.assertEquals(20, volABucketASnapshotCount); + + snapshotIter = store.listSnapshot(volumeA, bucketB, null, null); + int volABucketBSnapshotCount = 0; + while (snapshotIter.hasNext()) { + OzoneSnapshot snapshot = snapshotIter.next(); + volABucketBSnapshotCount++; + } + Assert.assertEquals(20, volABucketASnapshotCount); + + snapshotIter = store.listSnapshot(volumeB, bucketA, null, null); + int volBBucketASnapshotCount = 0; + while (snapshotIter.hasNext()) { + OzoneSnapshot snapshot = snapshotIter.next(); + volBBucketASnapshotCount++; + } + Assert.assertEquals(20, volABucketASnapshotCount); + + snapshotIter = store.listSnapshot(volumeB, bucketB, null, null); + int volBBucketBSnapshotCount = 0; + while (snapshotIter.hasNext()) { + OzoneSnapshot snapshot = snapshotIter.next(); + volBBucketBSnapshotCount++; + } + Assert.assertEquals(20, volABucketASnapshotCount); + + int volABucketASnapshotACount = 0; + snapshotIter = store.listSnapshot(volumeA, bucketA, snapshotPrefixA, null); + while (snapshotIter.hasNext()) { + OzoneSnapshot snapshot = snapshotIter.next(); + Assert.assertTrue(snapshot.getName().startsWith(snapshotPrefixA)); + volABucketASnapshotACount++; + } + Assert.assertEquals(10, volABucketASnapshotACount); + Assert.assertFalse(snapshotIter.hasNext()); + + } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java index 8bd09f543ad9..bd43b46b3f5a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java @@ -430,7 +430,8 @@ public void testSnapshotOps() throws Exception { writeClient.createSnapshot(volumeName, bucketName, snapshot2); // List snapshots - writeClient.listSnapshot(volumeName, bucketName); + writeClient.listSnapshot( + volumeName, bucketName, null, null, Integer.MAX_VALUE); omMetrics = getMetrics("OMMetrics"); assertCounter("NumSnapshotActive", 2L, omMetrics); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 81e776d6b4fb..b296acf5874e 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1674,6 +1674,9 @@ message CreateSnapshotRequest { message ListSnapshotRequest { optional string volumeName = 1; optional string bucketName = 2; + optional string prefix = 3; + optional string prevSnapshot = 4; + optional uint32 maxListResult = 5; } message SnapshotDiffRequest { diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 539a6daf6f24..48948e3697f5 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -224,12 +224,16 @@ List listTrash(String volumeName, String bucketName, /** * List snapshots in a volume/bucket. - * @param volumeName volume name - * @param bucketName bucket name + * @param volumeName volume name + * @param bucketName bucket name + * @param snapshotPrefix snapshot prefix to match + * @param prevSnapshot start of the list, this snapshot is excluded + * @param maxListResult max numbet of snapshots to return * @return list of snapshot */ - List listSnapshot(String volumeName, String bucketName) - throws IOException; + List listSnapshot( + String volumeName, String bucketName, String snapshotPrefix, + String prevSnapshot, int maxListResult) throws IOException; /** * Recover trash allows the user to recover the keys diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index cc5054c1aa97..f9e5e3518931 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1088,7 +1088,7 @@ public List listKeys(String volumeName, String bucketName, if (StringUtil.isNotBlank(keyPrefix)) { seekPrefix = getOzoneKey(volumeName, bucketName, keyPrefix); } else { - seekPrefix = getBucketKey(volumeName, bucketName + OM_KEY_PREFIX); + seekPrefix = getBucketKey(volumeName, bucketName) + OM_KEY_PREFIX; } int currentCount = 0; @@ -1181,8 +1181,9 @@ public List listTrash(String volumeName, String bucketName, } @Override - public List listSnapshot(String volumeName, String bucketName) - throws IOException { + public List listSnapshot( + String volumeName, String bucketName, String snapshotPrefix, + String prevSnapshot, int maxListResult) throws IOException { if (Strings.isNullOrEmpty(volumeName)) { throw new OMException("Volume name is required.", VOLUME_NOT_FOUND); } @@ -1197,39 +1198,69 @@ public List listSnapshot(String volumeName, String bucketName) BUCKET_NOT_FOUND); } - String prefix = getBucketKey(volumeName, bucketName + OM_KEY_PREFIX); + String prefix; + if (StringUtil.isNotBlank(snapshotPrefix)) { + prefix = getOzoneKey(volumeName, bucketName, snapshotPrefix); + } else { + prefix = getBucketKey(volumeName, bucketName + OM_KEY_PREFIX); + } + + String seek; + if (StringUtil.isNotBlank(prevSnapshot)) { + // Seek to the specified snapshot. + seek = getOzoneKey(volumeName, bucketName, prevSnapshot); + } else { + // This allows us to seek directly to the first key with the right prefix. + seek = getOzoneKey(volumeName, bucketName, + StringUtil.isNotBlank( + snapshotPrefix) ? snapshotPrefix : OM_KEY_PREFIX); + } + TreeMap snapshotInfoMap = new TreeMap<>(); - appendSnapshotFromCacheToMap(snapshotInfoMap, prefix); - appendSnapshotFromDBToMap(snapshotInfoMap, prefix); + int count = appendSnapshotFromCacheToMap( + snapshotInfoMap, prefix, seek, maxListResult); + appendSnapshotFromDBToMap( + snapshotInfoMap, prefix, seek, count, maxListResult); return new ArrayList<>(snapshotInfoMap.values()); } - private void appendSnapshotFromCacheToMap( - TreeMap snapshotInfoMap, String prefix) { + private int appendSnapshotFromCacheToMap( + TreeMap snapshotInfoMap, String prefix, + String previous, int maxListResult) { + int count = 0; Iterator, CacheValue>> iterator = snapshotInfoTable.cacheIterator(); - while (iterator.hasNext()) { + while (iterator.hasNext() && count < maxListResult) { Map.Entry, CacheValue> entry = iterator.next(); String snapshotKey = entry.getKey().getCacheKey(); SnapshotInfo snapshotInfo = entry.getValue().getCacheValue(); - if (snapshotInfo != null && snapshotKey.startsWith(prefix)) { + if (snapshotInfo != null && snapshotKey.startsWith(prefix) && + snapshotKey.compareTo(previous) > 0) { snapshotInfoMap.put(snapshotKey, snapshotInfo); + count++; } } + return count; } - private void appendSnapshotFromDBToMap(TreeMap snapshotInfoMap, String prefix) + private void appendSnapshotFromDBToMap(TreeMap snapshotInfoMap, + String prefix, String previous, + int count, int maxListResult) throws IOException { try (TableIterator> snapshotIter = snapshotInfoTable.iterator()) { KeyValue< String, SnapshotInfo> snapshotinfo; - snapshotIter.seek(prefix); - while (snapshotIter.hasNext()) { + snapshotIter.seek(previous); + while (snapshotIter.hasNext() && count < maxListResult) { snapshotinfo = snapshotIter.next(); - if (snapshotinfo != null && snapshotinfo.getKey().startsWith(prefix)) { + if (snapshotinfo != null && + snapshotinfo.getKey().compareTo(previous) == 0) { + continue; + } + if (snapshotinfo != null && snapshotinfo.getKey().startsWith(prefix)) { CacheValue cacheValue = snapshotInfoTable.getCacheValue( new CacheKey<>(snapshotinfo.getKey())); @@ -1238,6 +1269,7 @@ private void appendSnapshotFromDBToMap(TreeMap snapshotInfoMap, String prefix) // in cache. if (cacheValue == null) { snapshotInfoMap.put(snapshotinfo.getKey(), snapshotinfo.getValue()); + count++; } } else { break; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 169e87fa396d..52fa7ac9b10c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2777,8 +2777,9 @@ public List listTrash(String volumeName, } @Override - public List listSnapshot(String volumeName, String bucketName) - throws IOException { + public List listSnapshot( + String volumeName, String bucketName, String snapshotPrefix, + String prevSnapshot, int maxListResult) throws IOException { if (isAclEnabled) { omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.LIST, volumeName, bucketName, null); @@ -2788,7 +2789,8 @@ public List listSnapshot(String volumeName, String bucketName) auditMap.put(OzoneConsts.BUCKET, bucketName); try { metrics.incNumSnapshotLists(); - return metadataManager.listSnapshot(volumeName, bucketName); + return metadataManager.listSnapshot(volumeName, bucketName, + snapshotPrefix, prevSnapshot, maxListResult); } catch (Exception ex) { metrics.incNumSnapshotListFails(); auditSuccess = false; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 6ab84a5f028d..31af0f16e15e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -1243,7 +1243,8 @@ private OzoneManagerProtocolProtos.ListSnapshotResponse getSnapshots( OzoneManagerProtocolProtos.ListSnapshotRequest request) throws IOException { List snapshotInfos = impl.listSnapshot( - request.getVolumeName(), request.getBucketName()); + request.getVolumeName(), request.getBucketName(), request.getPrefix(), + request.getPrevSnapshot(), request.getMaxListResult()); List snapshotInfoList = snapshotInfos.stream().map(SnapshotInfo::getProtobuf) .collect(Collectors.toList()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 1ff17bce38a0..8d3e5ddc0a46 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -686,26 +686,64 @@ public void testListSnapshot() throws Exception { OMRequestTestUtils.addVolumeToDB(vol1, omMetadataManager); addBucketsToCache(vol1, bucket1); - String snapshotName = "snapshot"; + String prefixA = "snapshotA"; + String prefixB = "snapshotB"; + TreeSet snapshotsASet = new TreeSet<>(); - for (int i = 1; i <= 10; i++) { + for (int i = 1; i <= 100; i++) { if (i % 2 == 0) { + snapshotsASet.add(prefixA + i); OMRequestTestUtils.addSnapshotToTable(vol1, bucket1, - snapshotName + i, omMetadataManager); + prefixA + i, omMetadataManager); } else { OMRequestTestUtils.addSnapshotToTableCache(vol1, bucket1, - snapshotName + i, omMetadataManager); + prefixB + i, omMetadataManager); } } //Test listing all snapshots. List snapshotInfos = omMetadataManager.listSnapshot(vol1, - bucket1); - assertEquals(10, snapshotInfos.size()); + bucket1, null, null, 100); + assertEquals(100, snapshotInfos.size()); + + snapshotInfos = omMetadataManager.listSnapshot(vol1, + bucket1, prefixA, null, 50); + assertEquals(50, snapshotInfos.size()); + for (SnapshotInfo snapshotInfo : snapshotInfos) { + assertTrue(snapshotInfo.getName().startsWith(prefixA)); + } + + String startSnapshot = prefixA + 38; + snapshotInfos = omMetadataManager.listSnapshot(vol1, + bucket1, prefixA, startSnapshot, 50); + assertEquals(snapshotsASet.tailSet(startSnapshot).size() - 1, + snapshotInfos.size()); for (SnapshotInfo snapshotInfo : snapshotInfos) { - assertTrue(snapshotInfo.getName().startsWith(snapshotName)); + assertTrue(snapshotInfo.getName().startsWith(prefixA)); + assertTrue(snapshotInfo.getName().compareTo(startSnapshot) > 0); } + startSnapshot = null; + TreeSet expectedSnapshot = new TreeSet<>(); + for (int i = 1; i <= 5; i++) { + snapshotInfos = omMetadataManager.listSnapshot( + vol1, bucket1, prefixA, startSnapshot, 10); + assertEquals(10, snapshotInfos.size()); + + for (SnapshotInfo snapshotInfo : snapshotInfos) { + expectedSnapshot.add(snapshotInfo.getName()); + assertTrue(snapshotInfo.getName().startsWith(prefixA)); + startSnapshot = snapshotInfo.getName(); + } + } + assertEquals(snapshotsASet, expectedSnapshot); + + // As now we have iterated all 50 snapshots, calling next time should + // return empty list. + snapshotInfos = omMetadataManager.listSnapshot(vol1, bucket1, + startSnapshot, prefixA, 10); + + assertEquals(snapshotInfos.size(), 0); } @ParameterizedTest @@ -721,7 +759,7 @@ public void testListSnapshotWithInvalidPath(String volume, addBucketsToCache(vol1, bucket1); OMException oe = assertThrows(OMException.class, - () -> omMetadataManager.listSnapshot(volume, bucket)); + () -> omMetadataManager.listSnapshot(volume, bucket, null, null, 100)); assertEquals(expectedResultCode, oe.getResult()); } @@ -758,14 +796,14 @@ public void testListSnapshotDoesNotListOtherBucketSnapshots() //Test listing snapshots only lists snapshots of specified bucket List snapshotInfos1 = omMetadataManager.listSnapshot(vol1, - bucket1); + bucket1, null, null, Integer.MAX_VALUE); assertEquals(2, snapshotInfos1.size()); for (SnapshotInfo snapshotInfo : snapshotInfos1) { assertTrue(snapshotInfo.getName().startsWith(snapshotName1)); } List snapshotInfos2 = omMetadataManager.listSnapshot(vol1, - bucket2); + bucket2, null, null, Integer.MAX_VALUE); assertEquals(5, snapshotInfos2.size()); for (SnapshotInfo snapshotInfo : snapshotInfos2) { assertTrue(snapshotInfo.getName().startsWith(snapshotName2)); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index aa1cb345dfa4..3bb232f3384b 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -617,17 +617,19 @@ public String createSnapshot(String volumeName, } @Override + public List listSnapshot( + String volumeName, String bucketName, String snapshotPrefix, + String prevSnapshot, int maxListResult) throws IOException { + return null; + } + public void deleteSnapshot(String volumeName, String bucketName, String snapshotName) throws IOException { } - @Override - public List listSnapshot(String volumeName, String bucketName) - throws IOException { - return null; - } + public SnapshotDiffReport snapshotDiff(String volumeName, String bucketName, String fromSnapshot, String toSnapshot) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java index 7ad4041bef3a..8ac9471f2607 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java @@ -25,7 +25,7 @@ import picocli.CommandLine; import java.io.IOException; -import java.util.List; +import java.util.Iterator; /** * ozone sh snapshot list. @@ -50,10 +50,9 @@ protected void execute(OzoneClient client, OzoneAddress address) String volumeName = snapshotPath.getValue().getVolumeName(); String bucketName = snapshotPath.getValue().getBucketName(); - List snapshotInfos = client.getObjectStore() - .listSnapshot(volumeName, bucketName); - int counter = printAsJsonArray(snapshotInfos.iterator(), - snapshotInfos.size()); + Iterator snapshotInfos = client.getObjectStore() + .listSnapshot(volumeName, bucketName, null, null); + int counter = printAsJsonArray(snapshotInfos, Integer.MAX_VALUE); if (isVerbose()) { out().printf("Found : %d snapshots for o3://%s/ %s ", counter, volumeName, bucketName);