From 2743e82caa540735926cd7e92202efca7d4d316f Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 5 Feb 2025 23:59:47 +0800 Subject: [PATCH 01/29] Add keyMarker, uploadIdMarker, maxUploads into ListMultipartUploads service in omClientPb to support its pagination implementation --- .../interface-client/src/main/proto/OmClientProtocol.proto | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 92c2b6b4cc5a..ec943dd5c30c 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1759,11 +1759,16 @@ message ListMultipartUploadsRequest { required string volume = 1; required string bucket = 2; required string prefix = 3; + optional string keyMarker = 4; + optional string uploadIdMarker = 5; + optional int32 maxUploads = 6; } message ListMultipartUploadsResponse { optional bool isTruncated = 1; repeated MultipartUploadInfo uploadsList = 2; + optional string nextKeyMarker = 3; + optional string nextUploadIdMarker = 4; } message MultipartUploadInfo { From fd44b3be66e2ade8b22d11a51a4b66f16d3695d4 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 6 Feb 2025 00:01:40 +0800 Subject: [PATCH 02/29] Add keyMarker, uploadIdMarker, and maxUploads to related interface --- .../ozone/client/protocol/ClientProtocol.java | 2 +- .../om/protocol/OzoneManagerProtocol.java | 2 +- .../hadoop/ozone/om/OMMetadataManager.java | 71 ++++++++++++++++++- .../apache/hadoop/ozone/om/KeyManager.java | 2 +- .../ozone/client/ClientProtocolStub.java | 5 +- 5 files changed, 76 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index c0bffaf89501..7a7115e37325 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -668,7 +668,7 @@ OzoneMultipartUploadPartListParts listParts(String volumeName, * Return with the inflight multipart uploads. */ OzoneMultipartUploadList listMultipartUploads(String volumename, - String bucketName, String prefix) throws IOException; + String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException; /** * Get a valid Delegation Token. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 7f633d7ea73d..11c3fafb8bfe 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -610,7 +610,7 @@ OmMultipartUploadListParts listParts(String volumeName, String bucketName, * List in-flight uploads. */ OmMultipartUploadList listMultipartUploads(String volumeName, - String bucketName, String prefix) throws IOException; + String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException; /** * Gets s3Secret for given kerberos user. diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index ae57c18354d2..712f574a2eaa 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -490,12 +490,79 @@ long countRowsInTable(Table table) long countEstimatedRowsInTable(Table table) throws IOException; + + class MultipartUploadKeys { + private final Set keys; + private final String nextKeyMarker; + private final String nextUploadIdMarker; + private final boolean isTruncated; + + public MultipartUploadKeys(Set keys, String nextKeyMarker, String nextUploadIdMarker, boolean isTruncated) { + this.keys = keys; + this.nextKeyMarker = nextKeyMarker; + this.nextUploadIdMarker = nextUploadIdMarker; + this.isTruncated = isTruncated; + } + + public Set getKeys() { + return keys; + } + + public String getNextKeyMarker() { + return nextKeyMarker; + } + + public String getNextUploadIdMarker() { + return nextUploadIdMarker; + } + + public boolean isTruncated() { + return isTruncated; + } + + public static Builder newBuilder() { + return new Builder(); + } + + public static class Builder { + private Set keys; + private String nextKeyMarker = ""; + private String nextUploadIdMarker = ""; + private boolean isTruncated = false; + + public Builder setKeys(Set keys) { + this.keys = keys; + return this; + } + + public Builder setNextKeyMarker(String nextKeyMarker) { + this.nextKeyMarker = nextKeyMarker; + return this; + } + + public Builder setNextUploadIdMarker(String nextUploadIdMarker) { + this.nextUploadIdMarker = nextUploadIdMarker; + return this; + } + + public Builder setIsTruncated(boolean isTruncated) { + this.isTruncated = isTruncated; + return this; + } + + public MultipartUploadKeys build() { + return new MultipartUploadKeys(keys, nextKeyMarker, nextUploadIdMarker, isTruncated); + } + + } + } + /** * Return the existing upload keys which includes volumeName, bucketName, * keyName. */ - Set getMultipartUploadKeys(String volumeName, - String bucketName, String prefix) throws IOException; + MultipartUploadKeys getMultipartUploadKeys(String volumeName, + String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException; /** * Gets the DirectoryTable. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index db3d47dfcdb9..2351c3ec644b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -203,7 +203,7 @@ List getExpiredMultipartUploads( OmMultipartUploadList listMultipartUploads(String volumeName, - String bucketName, String prefix) throws OMException; + String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws OMException; /** * Returns list of parts of a multipart upload key. diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 41584c9786d4..83ee6762e12b 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -385,7 +385,10 @@ public OzoneMultipartUploadPartListParts listParts(String volumeName, @Override public OzoneMultipartUploadList listMultipartUploads(String volumename, String bucketName, - String prefix) + String prefix, + String keyMarker, + String uploadIdMarker, + int maxUploads) throws IOException { return null; } From 552590c4b35cbf7ac91c881dd8c965d91171fce4 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 6 Feb 2025 00:04:05 +0800 Subject: [PATCH 03/29] Add new arguments into data transform process --- .../hadoop/ozone/client/OzoneBucket.java | 5 ++-- .../client/OzoneMultipartUploadList.java | 23 ++++++++++++++++++- .../hadoop/ozone/client/rpc/RpcClient.java | 9 +++++--- .../om/helpers/OmMultipartUploadList.java | 23 ++++++++++++++++++- ...ManagerProtocolClientSideTranslatorPB.java | 10 ++++++-- .../hadoop/ozone/om/KeyManagerImpl.java | 13 +++++++---- .../apache/hadoop/ozone/om/OzoneManager.java | 4 ++-- .../OzoneManagerRequestHandler.java | 5 +++- 8 files changed, 75 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 1a40b536909d..3ff118b0842f 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -1031,9 +1031,10 @@ public List listStatus(String keyName, boolean recursive, * * @param prefix Optional string to filter for the selected keys. */ - public OzoneMultipartUploadList listMultipartUploads(String prefix) + public OzoneMultipartUploadList listMultipartUploads(String prefix, + String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { - return proxy.listMultipartUploads(volumeName, getName(), prefix); + return proxy.listMultipartUploads(volumeName, getName(), prefix, keyMarker, uploadIdMarker, maxUploads); } /** diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java index 38377ebc1763..790d3dc240e4 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java @@ -28,11 +28,20 @@ public class OzoneMultipartUploadList { private List uploads; + private String nextKeyMarker; + private String nextUploadIdMarker; + private boolean isTruncated; public OzoneMultipartUploadList( - List uploads) { + List uploads, + String nextKeyMarker, + String nextUploadIdMarker, + boolean isTruncated) { Preconditions.checkNotNull(uploads); this.uploads = uploads; + this.nextKeyMarker = nextKeyMarker; + this.nextUploadIdMarker = nextUploadIdMarker; + this.isTruncated = isTruncated; } public List getUploads() { @@ -43,4 +52,16 @@ public void setUploads( List uploads) { this.uploads = uploads; } + + public String getNextKeyMarker() { + return nextKeyMarker; + } + + public String getNextUploadIdMarker() { + return nextUploadIdMarker; + } + + public boolean isTruncated() { + return isTruncated; + } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 2923ee98cad7..177773b9360f 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2126,10 +2126,10 @@ public OzoneMultipartUploadPartListParts listParts(String volumeName, @Override public OzoneMultipartUploadList listMultipartUploads(String volumeName, - String bucketName, String prefix) throws IOException { + String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { OmMultipartUploadList omMultipartUploadList = - ozoneManagerClient.listMultipartUploads(volumeName, bucketName, prefix); + ozoneManagerClient.listMultipartUploads(volumeName, bucketName, prefix, keyMarker, uploadIdMarker, maxUploads); List uploads = omMultipartUploadList.getUploads() .stream() .map(upload -> new OzoneMultipartUpload(upload.getVolumeName(), @@ -2139,7 +2139,10 @@ public OzoneMultipartUploadList listMultipartUploads(String volumeName, upload.getCreationTime(), upload.getReplicationConfig())) .collect(Collectors.toList()); - OzoneMultipartUploadList result = new OzoneMultipartUploadList(uploads); + OzoneMultipartUploadList result = new OzoneMultipartUploadList(uploads, + omMultipartUploadList.getNextKeyMarker(), + omMultipartUploadList.getNextUploadIdMarker(), + omMultipartUploadList.isTruncated()); return result; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadList.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadList.java index 0c13a0d4a92f..0a7e94f70026 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadList.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadList.java @@ -26,10 +26,19 @@ public class OmMultipartUploadList { private List uploads; + private String nextKeyMarker; + private String nextUploadIdMarker; + private boolean isTruncated; public OmMultipartUploadList( - List uploads) { + List uploads, + String nextKeyMarker, + String nextUploadIdMarker, + boolean isTruncated) { this.uploads = uploads; + this.nextKeyMarker = nextKeyMarker; + this.nextUploadIdMarker = nextUploadIdMarker; + this.isTruncated = isTruncated; } public List getUploads() { @@ -41,4 +50,16 @@ public void setUploads( this.uploads = uploads; } + public String getNextKeyMarker() { + return nextKeyMarker; + } + + public String getNextUploadIdMarker() { + return nextUploadIdMarker; + } + + public boolean isTruncated() { + return isTruncated; + } + } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 6b23b0f2682b..9ad5bf0ebd67 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1810,12 +1810,15 @@ public OmMultipartUploadListParts listParts(String volumeName, @Override public OmMultipartUploadList listMultipartUploads(String volumeName, String bucketName, - String prefix) throws IOException { + String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { ListMultipartUploadsRequest request = ListMultipartUploadsRequest .newBuilder() .setVolume(volumeName) .setBucket(bucketName) .setPrefix(prefix == null ? "" : prefix) + .setKeyMarker(keyMarker == null ? "" : keyMarker) + .setUploadIdMarker(uploadIdMarker == null ? "" : uploadIdMarker) + .setMaxUploads(maxUploads) .build(); OMRequest omRequest = createOMRequest(Type.ListMultipartUploads) @@ -1839,7 +1842,10 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, )) .collect(Collectors.toList()); - OmMultipartUploadList response = new OmMultipartUploadList(uploadList); + OmMultipartUploadList response = new OmMultipartUploadList(uploadList, + listMultipartUploadsResponse.getNextKeyMarker(), + listMultipartUploadsResponse.getNextUploadIdMarker(), + listMultipartUploadsResponse.getIsTruncated()); return response; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 735dfca55900..471cd6d51887 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -820,7 +820,7 @@ public boolean isSstFilteringSvcEnabled() { @Override public OmMultipartUploadList listMultipartUploads(String volumeName, - String bucketName, String prefix) throws OMException { + String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws OMException { Preconditions.checkNotNull(volumeName); Preconditions.checkNotNull(bucketName); @@ -828,11 +828,11 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, bucketName); try { - Set multipartUploadKeys = + OMMetadataManager.MultipartUploadKeys multipartUploadKeys = metadataManager - .getMultipartUploadKeys(volumeName, bucketName, prefix); + .getMultipartUploadKeys(volumeName, bucketName, prefix, keyMarker, uploadIdMarker, maxUploads); - List collect = multipartUploadKeys.stream() + List collect = multipartUploadKeys.getKeys().stream() .map(OmMultipartUpload::from) .peek(upload -> { try { @@ -855,7 +855,10 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, }) .collect(Collectors.toList()); - return new OmMultipartUploadList(collect); + return new OmMultipartUploadList(collect, + multipartUploadKeys.getNextKeyMarker(), + multipartUploadKeys.getNextUploadIdMarker(), + multipartUploadKeys.isTruncated()); } catch (IOException ex) { LOG.error("List Multipart Uploads Failed: volume: " + volumeName + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index f1d31d130cf1..d0c89058c58f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3760,7 +3760,7 @@ public OmMultipartUploadListParts listParts(final String volumeName, @Override public OmMultipartUploadList listMultipartUploads(String volumeName, - String bucketName, String prefix) throws IOException { + String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { ResolvedBucket bucket = resolveBucketLink(Pair.of(volumeName, bucketName)); @@ -3771,7 +3771,7 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, try { OmMultipartUploadList omMultipartUploadList = keyManager.listMultipartUploads(bucket.realVolume(), - bucket.realBucket(), prefix); + bucket.realBucket(), prefix, keyMarker, uploadIdMarker, maxUploads); AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction .LIST_MULTIPART_UPLOADS, auditMap)); return omMultipartUploadList; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index a31783f29c48..0af7e8043388 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -1026,7 +1026,7 @@ private ListMultipartUploadsResponse listMultipartUploads( OmMultipartUploadList omMultipartUploadList = impl.listMultipartUploads(request.getVolume(), request.getBucket(), - request.getPrefix()); + request.getPrefix(), request.getKeyMarker(), request.getUploadIdMarker(), request.getMaxUploads()); List info = omMultipartUploadList .getUploads() @@ -1057,6 +1057,9 @@ private ListMultipartUploadsResponse listMultipartUploads( ListMultipartUploadsResponse response = ListMultipartUploadsResponse.newBuilder() .addAllUploadsList(info) + .setIsTruncated(omMultipartUploadList.isTruncated()) + .setNextKeyMarker(omMultipartUploadList.getNextKeyMarker()) + .setNextUploadIdMarker(omMultipartUploadList.getNextUploadIdMarker()) .build(); return response; From 274dfcb0754c3f68dcf2d0576eadd50b93815730 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 6 Feb 2025 00:05:41 +0800 Subject: [PATCH 04/29] Implement pagination logic for getMultipartUploadKeys in om metadata manager --- .../ozone/om/OmMetadataManagerImpl.java | 57 +++++++++++++++---- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 2c562349ad6d..94bb64f632aa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -38,6 +38,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.validation.constraints.NotNull; + import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; @@ -1929,46 +1931,81 @@ public long countEstimatedRowsInTable(Table table) } @Override - public Set getMultipartUploadKeys( - String volumeName, String bucketName, String prefix) throws IOException { + public MultipartUploadKeys getMultipartUploadKeys( + String volumeName, String bucketName, String prefix, @NotNull String keyMarker, + @NotNull String uploadIdMarker, int maxUploads) throws IOException { - Set response = new TreeSet<>(); + MultipartUploadKeys.Builder resultBuilder = MultipartUploadKeys.newBuilder(); + Set responseKeys = new TreeSet<>(); Set aborted = new TreeSet<>(); + String lastKey = null; Iterator, CacheValue>> cacheIterator = getMultipartInfoTable().cacheIterator(); + + if (keyMarker != "") { + prefix = keyMarker; + } + String prefixKey = OmMultipartUpload.getDbKey(volumeName, bucketName, prefix); + prefixKey = prefixKey + OM_KEY_PREFIX + uploadIdMarker; + // First iterate all the entries in cache. - while (cacheIterator.hasNext()) { + while (cacheIterator.hasNext() && responseKeys.size() < maxUploads) { Map.Entry, CacheValue> cacheEntry = cacheIterator.next(); if (cacheEntry.getKey().getCacheKey().startsWith(prefixKey)) { // Check if it is marked for delete, due to abort mpu if (cacheEntry.getValue().getCacheValue() != null) { - response.add(cacheEntry.getKey().getCacheKey()); + responseKeys.add(cacheEntry.getKey().getCacheKey()); + lastKey = cacheEntry.getKey().getCacheKey(); } else { aborted.add(cacheEntry.getKey().getCacheKey()); } } } + + if (cacheIterator.hasNext() && responseKeys.size() == maxUploads) { + resultBuilder.setIsTruncated(true); + } + // prefixed iterator will only iterate until keys match the prefix try (TableIterator> iterator = getMultipartInfoTable().iterator(prefixKey)) { - while (iterator.hasNext()) { + while (iterator.hasNext() && responseKeys.size() < maxUploads) { KeyValue entry = iterator.next(); - // If it is marked for abort, skip it. - if (!aborted.contains(entry.getKey())) { - response.add(entry.getKey()); + if (entry.getKey().startsWith(prefixKey)) { + // If it is marked for abort, skip it. + if (!aborted.contains(entry.getKey())) { + responseKeys.add(entry.getKey()); + lastKey = entry.getKey(); + } + } else { + break; } } + + if (iterator.hasNext() && responseKeys.size() == maxUploads) { + resultBuilder.setIsTruncated(true); + } + } + + // Should we set nextKeyMarker and nextUploadIdMarker if not truncated? + if (lastKey != null) { + String[] parts = lastKey.split(OM_KEY_PREFIX); + // parts[] would be like: ["", "volumeName", "bucketName", "keyName", "uploadId"] + resultBuilder.setNextKeyMarker(parts[parts.length - 2]); // keyName + resultBuilder.setNextUploadIdMarker(parts[parts.length - 1]); // uploadId } - return response; + return resultBuilder + .setKeys(responseKeys) + .build(); } @Override From 659c363df064451f5f405052644524e121bb3827 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 6 Feb 2025 00:17:02 +0800 Subject: [PATCH 05/29] Include max-uploads, key-marker and upload-id-marker as part of listMultipartUploads s3g request --- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index e68a59e7f76b..3dc92a4fa52e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.audit.S3GAction; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.client.OzoneMultipartUpload; import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -47,6 +48,7 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; +import javax.validation.constraints.Max; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -137,7 +139,7 @@ public Response get( if (uploads != null) { s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; - return listMultipartUploads(bucketName, prefix); + return listMultipartUploads(bucketName, prefix, 100, null, null); } if (prefix == null) { @@ -328,7 +330,10 @@ public Response put(@PathParam("bucket") String bucketName, public Response listMultipartUploads( @PathParam("bucket") String bucketName, - @QueryParam("prefix") String prefix) + @QueryParam("prefix") String prefix, + @QueryParam("max-uploads") @DefaultValue("100") @Max(1000) int maxUploads, + @QueryParam("key-marker") String keyMarker, + @QueryParam("upload-id-marker") String uploadIdMarker) throws OS3Exception, IOException { long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; @@ -336,11 +341,14 @@ public Response listMultipartUploads( OzoneBucket bucket = getBucket(bucketName); try { - OzoneMultipartUploadList ozoneMultipartUploadList = - bucket.listMultipartUploads(prefix); + OzoneMultipartUploadList ozoneMultipartUploadList = + bucket.listMultipartUploads(prefix, keyMarker, uploadIdMarker, maxUploads); ListMultipartUploadsResult result = new ListMultipartUploadsResult(); result.setBucket(bucketName); + result.setKeyMarker(ozoneMultipartUploadList.getNextKeyMarker()); + result.setUploadIdMarker(ozoneMultipartUploadList.getNextUploadIdMarker()); + result.setTruncated(ozoneMultipartUploadList.isTruncated()); ozoneMultipartUploadList.getUploads().forEach(upload -> result.addUpload( new ListMultipartUploadsResult.Upload( From 0c13a3f4688f931c0ad4f675176d6237d9cc1668 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 6 Feb 2025 00:17:43 +0800 Subject: [PATCH 06/29] Updates tests to use new client interface --- .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 10 +++++----- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 12 ++++++------ .../ozone/s3/endpoint/TestPermissionCheck.java | 5 +++-- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java index 58183e87705f..fad3f78f8036 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java @@ -810,7 +810,7 @@ public void testListMultipartUpload() throws Exception { uploadPart(bucket, key2, uploadID2, 1, "data".getBytes(UTF_8)); uploadPart(bucket, key3, uploadID3, 1, "data".getBytes(UTF_8)); - OzoneMultipartUploadList listMPUs = bucket.listMultipartUploads("dir1"); + OzoneMultipartUploadList listMPUs = bucket.listMultipartUploads("dir1", "", "", 1000); assertEquals(3, listMPUs.getUploads().size()); List expectedList = new ArrayList<>(keys); for (OzoneMultipartUpload mpu : listMPUs.getUploads()) { @@ -818,7 +818,7 @@ public void testListMultipartUpload() throws Exception { } assertEquals(0, expectedList.size()); - listMPUs = bucket.listMultipartUploads("dir1/dir2"); + listMPUs = bucket.listMultipartUploads("dir1/dir2", "", "", 1000); assertEquals(2, listMPUs.getUploads().size()); expectedList = new ArrayList<>(); expectedList.add(key2); @@ -828,7 +828,7 @@ public void testListMultipartUpload() throws Exception { } assertEquals(0, expectedList.size()); - listMPUs = bucket.listMultipartUploads("dir1/dir2/dir3"); + listMPUs = bucket.listMultipartUploads("dir1/dir2/dir3", "", "", 1000); assertEquals(1, listMPUs.getUploads().size()); expectedList = new ArrayList<>(); expectedList.add(key3); @@ -838,7 +838,7 @@ public void testListMultipartUpload() throws Exception { assertEquals(0, expectedList.size()); // partial key - listMPUs = bucket.listMultipartUploads("d"); + listMPUs = bucket.listMultipartUploads("d", "", "", 1000); assertEquals(3, listMPUs.getUploads().size()); expectedList = new ArrayList<>(keys); for (OzoneMultipartUpload mpu : listMPUs.getUploads()) { @@ -847,7 +847,7 @@ public void testListMultipartUpload() throws Exception { assertEquals(0, expectedList.size()); // partial key - listMPUs = bucket.listMultipartUploads(""); + listMPUs = bucket.listMultipartUploads("", "", "", 1000); assertEquals(3, listMPUs.getUploads().size()); expectedList = new ArrayList<>(keys); for (OzoneMultipartUpload mpu : listMPUs.getUploads()) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index 3b04e6a7bd5a..01e1e54c8125 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -235,7 +235,7 @@ public void listMultipartUploads() throws IOException { //WHEN OmMultipartUploadList omMultipartUploadList = - keyManager.listMultipartUploads(volume, "bucket1", ""); + keyManager.listMultipartUploads(volume, "bucket1", "", "", "", 10); //THEN List uploads = omMultipartUploadList.getUploads(); @@ -272,7 +272,7 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { //WHEN OmMultipartUploadList omMultipartUploadList = - keyManager.listMultipartUploads(volume, bucket, ""); + keyManager.listMultipartUploads(volume, bucket, "", "", "", 10); //THEN List uploads = omMultipartUploadList.getUploads(); @@ -296,7 +296,7 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { volume, bucket, "dir/ozonekey4"); omMultipartUploadList = - keyManager.listMultipartUploads(volume, bucket, "dir/ozone"); + keyManager.listMultipartUploads(volume, bucket, "dir/ozone", "", "", 10); //THEN uploads = omMultipartUploadList.getUploads(); @@ -312,7 +312,7 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { // Now list. omMultipartUploadList = - keyManager.listMultipartUploads(volume, bucket, "dir/ozone"); + keyManager.listMultipartUploads(volume, bucket, "dir/ozone", "", "", 10); //THEN uploads = omMultipartUploadList.getUploads(); @@ -327,7 +327,7 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { // Now list. omMultipartUploadList = - keyManager.listMultipartUploads(volume, bucket, "dir/ozone"); + keyManager.listMultipartUploads(volume, bucket, "dir/ozone", "", "", 10); //THEN uploads = omMultipartUploadList.getUploads(); @@ -355,7 +355,7 @@ public void listMultipartUploadsWithPrefix() throws IOException { //WHEN OmMultipartUploadList omMultipartUploadList = - keyManager.listMultipartUploads(volumeName, "bucket1", "dir"); + keyManager.listMultipartUploads(volumeName, "bucket1", "dir", "", "", 10); //THEN List uploads = omMultipartUploadList.getUploads(); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index fc828e101bad..ade1f26d649d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -46,6 +46,7 @@ import static java.net.HttpURLConnection.HTTP_FORBIDDEN; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; import static org.mockito.Mockito.anyLong; @@ -146,12 +147,12 @@ public void testDeleteBucket() throws IOException { @Test public void testListMultiUpload() throws IOException { when(objectStore.getS3Bucket(anyString())).thenReturn(bucket); - doThrow(exception).when(bucket).listMultipartUploads(anyString()); + doThrow(exception).when(bucket).listMultipartUploads(anyString(), anyString(), anyString(), anyInt()); BucketEndpoint bucketEndpoint = EndpointBuilder.newBucketEndpointBuilder() .setClient(client) .build(); OS3Exception e = assertThrows(OS3Exception.class, () -> - bucketEndpoint.listMultipartUploads("bucketName", "prefix")); + bucketEndpoint.listMultipartUploads("bucketName", "prefix", 10, "", "")); assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); } From aab4c1c12d567e093dfcd9b183d0979d99705999 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 6 Feb 2025 00:27:01 +0800 Subject: [PATCH 07/29] Fix checkstyle --- .../org/apache/hadoop/ozone/om/OMMetadataManager.java | 6 ++++++ .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 8 ++++---- .../apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java | 5 ++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 712f574a2eaa..ff3dde843e3f 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -491,6 +491,9 @@ long countEstimatedRowsInTable(Table table) throws IOException; + /** + * This class is used to store the result of listMultipartUploads. + */ class MultipartUploadKeys { private final Set keys; private final String nextKeyMarker; @@ -524,6 +527,9 @@ public static Builder newBuilder() { return new Builder(); } + /** + * Builder class for MultipartUploadKeys. + */ public static class Builder { private Set keys; private String nextKeyMarker = ""; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 94bb64f632aa..99c2de7e67f9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1997,10 +1997,10 @@ public MultipartUploadKeys getMultipartUploadKeys( // Should we set nextKeyMarker and nextUploadIdMarker if not truncated? if (lastKey != null) { - String[] parts = lastKey.split(OM_KEY_PREFIX); - // parts[] would be like: ["", "volumeName", "bucketName", "keyName", "uploadId"] - resultBuilder.setNextKeyMarker(parts[parts.length - 2]); // keyName - resultBuilder.setNextUploadIdMarker(parts[parts.length - 1]); // uploadId + String[] parts = lastKey.split(OM_KEY_PREFIX); + // parts[] would be like: ["", "volumeName", "bucketName", "keyName", "uploadId"] + resultBuilder.setNextKeyMarker(parts[parts.length - 2]); // keyName + resultBuilder.setNextUploadIdMarker(parts[parts.length - 1]); // uploadId } return resultBuilder diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 3dc92a4fa52e..953c27b3483c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -24,7 +24,6 @@ import org.apache.hadoop.ozone.audit.S3GAction; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneKey; -import org.apache.hadoop.ozone.client.OzoneMultipartUpload; import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -341,8 +340,8 @@ public Response listMultipartUploads( OzoneBucket bucket = getBucket(bucketName); try { - OzoneMultipartUploadList ozoneMultipartUploadList = - bucket.listMultipartUploads(prefix, keyMarker, uploadIdMarker, maxUploads); + OzoneMultipartUploadList ozoneMultipartUploadList = + bucket.listMultipartUploads(prefix, keyMarker, uploadIdMarker, maxUploads); ListMultipartUploadsResult result = new ListMultipartUploadsResult(); result.setBucket(bucketName); From 0318f1592a3c1aa336ce8f57daacd2e729414f35 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 6 Feb 2025 00:40:54 +0800 Subject: [PATCH 08/29] Fix findbugs --- .../java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 99c2de7e67f9..c659bcf75aae 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1944,7 +1944,7 @@ public MultipartUploadKeys getMultipartUploadKeys( cacheIterator = getMultipartInfoTable().cacheIterator(); - if (keyMarker != "") { + if (keyMarker.equals("")) { prefix = keyMarker; } From 448f066d68aaeca089feab3d729ed64f222bd4ee Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 6 Feb 2025 16:38:03 +0800 Subject: [PATCH 09/29] Fix the logic of prefix key building with keyMarker and uploadIdMarker --- .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index c659bcf75aae..8241d5117d1a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1944,15 +1944,16 @@ public MultipartUploadKeys getMultipartUploadKeys( cacheIterator = getMultipartInfoTable().cacheIterator(); - if (keyMarker.equals("")) { + if (!keyMarker.equals("")) { prefix = keyMarker; + if (!uploadIdMarker.equals("")) { + prefix = prefix + OM_KEY_PREFIX + uploadIdMarker; + } } String prefixKey = OmMultipartUpload.getDbKey(volumeName, bucketName, prefix); - prefixKey = prefixKey + OM_KEY_PREFIX + uploadIdMarker; - // First iterate all the entries in cache. while (cacheIterator.hasNext() && responseKeys.size() < maxUploads) { Map.Entry, CacheValue> cacheEntry = From edae70b3f2c698b7c8720e81c38b82ec1f76c05f Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 8 Feb 2025 17:58:17 +0800 Subject: [PATCH 10/29] Add maxUploads to OzoneMultipartUploadList and related methods --- .../hadoop/ozone/client/OzoneMultipartUploadList.java | 9 ++++++++- .../org/apache/hadoop/ozone/client/rpc/RpcClient.java | 3 ++- .../apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java index 790d3dc240e4..0cf972be0bb1 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java @@ -31,17 +31,20 @@ public class OzoneMultipartUploadList { private String nextKeyMarker; private String nextUploadIdMarker; private boolean isTruncated; + private int maxUploads; public OzoneMultipartUploadList( List uploads, String nextKeyMarker, String nextUploadIdMarker, - boolean isTruncated) { + boolean isTruncated, + int maxUploads) { Preconditions.checkNotNull(uploads); this.uploads = uploads; this.nextKeyMarker = nextKeyMarker; this.nextUploadIdMarker = nextUploadIdMarker; this.isTruncated = isTruncated; + this.maxUploads = maxUploads; } public List getUploads() { @@ -64,4 +67,8 @@ public String getNextUploadIdMarker() { public boolean isTruncated() { return isTruncated; } + + public int getMaxUploads() { + return maxUploads; + } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 177773b9360f..1e2d8dcc3858 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2142,7 +2142,8 @@ public OzoneMultipartUploadList listMultipartUploads(String volumeName, OzoneMultipartUploadList result = new OzoneMultipartUploadList(uploads, omMultipartUploadList.getNextKeyMarker(), omMultipartUploadList.getNextUploadIdMarker(), - omMultipartUploadList.isTruncated()); + omMultipartUploadList.isTruncated(), + omMultipartUploadList.getUploads().size()); return result; } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 953c27b3483c..d0f6be15c354 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -348,6 +348,7 @@ public Response listMultipartUploads( result.setKeyMarker(ozoneMultipartUploadList.getNextKeyMarker()); result.setUploadIdMarker(ozoneMultipartUploadList.getNextUploadIdMarker()); result.setTruncated(ozoneMultipartUploadList.isTruncated()); + result.setMaxUploads(ozoneMultipartUploadList.getMaxUploads()); ozoneMultipartUploadList.getUploads().forEach(upload -> result.addUpload( new ListMultipartUploadsResult.Upload( From faecf3e77c64444b8588f7fe8bf098b2588621be Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 8 Feb 2025 20:09:46 +0800 Subject: [PATCH 11/29] Add @Min validation for max-uploads parameter in listMultipartUploads --- .../org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index d0f6be15c354..1b8d7d3a52f5 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -48,6 +48,7 @@ import javax.inject.Inject; import javax.validation.constraints.Max; +import javax.validation.constraints.Min; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -330,7 +331,7 @@ public Response put(@PathParam("bucket") String bucketName, public Response listMultipartUploads( @PathParam("bucket") String bucketName, @QueryParam("prefix") String prefix, - @QueryParam("max-uploads") @DefaultValue("100") @Max(1000) int maxUploads, + @QueryParam("max-uploads") @DefaultValue("100") @Min(1) @Max(1000) int maxUploads, @QueryParam("key-marker") String keyMarker, @QueryParam("upload-id-marker") String uploadIdMarker) throws OS3Exception, IOException { From 17fbd659ca22cb6229676ac78bf32183a62be841 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 8 Feb 2025 20:10:08 +0800 Subject: [PATCH 12/29] Refactor listMultipartUploads pagination logic in OmMetadataManagerImpl --- .../ozone/om/OmMetadataManagerImpl.java | 35 ++++++++----------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 8241d5117d1a..0a9695435328 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1943,6 +1943,8 @@ public MultipartUploadKeys getMultipartUploadKeys( Iterator, CacheValue>> cacheIterator = getMultipartInfoTable().cacheIterator(); + String prefixKey = + OmMultipartUpload.getDbKey(volumeName, bucketName, prefix); if (!keyMarker.equals("")) { prefix = keyMarker; @@ -1950,12 +1952,10 @@ public MultipartUploadKeys getMultipartUploadKeys( prefix = prefix + OM_KEY_PREFIX + uploadIdMarker; } } - - String prefixKey = - OmMultipartUpload.getDbKey(volumeName, bucketName, prefix); + String startKey = OmMultipartUpload.getDbKey(volumeName, bucketName, prefix); // First iterate all the entries in cache. - while (cacheIterator.hasNext() && responseKeys.size() < maxUploads) { + while (cacheIterator.hasNext() && responseKeys.size() < maxUploads + 1) { Map.Entry, CacheValue> cacheEntry = cacheIterator.next(); if (cacheEntry.getKey().getCacheKey().startsWith(prefixKey)) { @@ -1969,16 +1969,12 @@ public MultipartUploadKeys getMultipartUploadKeys( } } - - if (cacheIterator.hasNext() && responseKeys.size() == maxUploads) { - resultBuilder.setIsTruncated(true); - } - // prefixed iterator will only iterate until keys match the prefix try (TableIterator> - iterator = getMultipartInfoTable().iterator(prefixKey)) { + iterator = getMultipartInfoTable().iterator()) { + iterator.seek(startKey); - while (iterator.hasNext() && responseKeys.size() < maxUploads) { + while (iterator.hasNext() && responseKeys.size() < maxUploads + 1) { KeyValue entry = iterator.next(); if (entry.getKey().startsWith(prefixKey)) { // If it is marked for abort, skip it. @@ -1990,18 +1986,15 @@ public MultipartUploadKeys getMultipartUploadKeys( break; } } - - if (iterator.hasNext() && responseKeys.size() == maxUploads) { - resultBuilder.setIsTruncated(true); - } } - // Should we set nextKeyMarker and nextUploadIdMarker if not truncated? - if (lastKey != null) { - String[] parts = lastKey.split(OM_KEY_PREFIX); - // parts[] would be like: ["", "volumeName", "bucketName", "keyName", "uploadId"] - resultBuilder.setNextKeyMarker(parts[parts.length - 2]); // keyName - resultBuilder.setNextUploadIdMarker(parts[parts.length - 1]); // uploadId + if (lastKey != null && responseKeys.size() == maxUploads + 1) { + responseKeys.remove(lastKey); + + OmMultipartUpload lastKeyMultipartUpload = OmMultipartUpload.from(lastKey); + resultBuilder.setNextKeyMarker(lastKeyMultipartUpload.getKeyName()); + resultBuilder.setNextUploadIdMarker(lastKeyMultipartUpload.getUploadId()); + resultBuilder.setIsTruncated(true); } return resultBuilder From 76e4999d2cab7ac44c2dca701d3740211dacf476 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 8 Feb 2025 20:17:45 +0800 Subject: [PATCH 13/29] Add test for multipart upload list pagination in FSO --- ...TestOzoneClientMultipartUploadWithFSO.java | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java index fad3f78f8036..8476a60297c9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java @@ -20,6 +20,8 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HashMap; +import java.util.HashSet; +import java.util.stream.Collectors; import javax.xml.bind.DatatypeConverter; import org.apache.commons.codec.digest.DigestUtils; @@ -81,6 +83,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.UUID; @@ -97,6 +100,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * This test verifies all the S3 multipart client apis - prefix layout. */ @@ -114,6 +120,8 @@ public class TestOzoneClientMultipartUploadWithFSO { private OzoneVolume volume; private OzoneBucket bucket; + private static final Logger LOG = LoggerFactory.getLogger(TestOzoneClientMultipartUploadWithFSO.class); + /** * Create a MiniOzoneCluster for testing. *

@@ -856,6 +864,70 @@ public void testListMultipartUpload() throws Exception { assertEquals(0, expectedList.size()); } + @Test + public void testListMultipartUploadsPagination() throws Exception { + int numOfKeys = 25; + List keys = new ArrayList<>(); + Map keyToUploadId = new HashMap<>(); + + // Generate keys + for (int i = 0; i < numOfKeys; i++) { + StringBuilder key = new StringBuilder(); + int depth = 1 + i % 3; // Creates varying depth (1-3 levels) + for (int j = 0; j < depth; j++) { + key.append("dir").append(j + 1).append("/"); + } + key.append("file").append(i); + keys.add(key.toString()); + } + + for (String key : keys) { + String uploadId = initiateMultipartUploadWithAsserts(bucket, key, RATIS, ONE); + keyToUploadId.put(key, uploadId); + uploadPart(bucket, key, uploadId, 1, "data".getBytes(UTF_8)); + } + + // Test full pagination process + final int maxUploads = 10; + final int expectedTruncated = 2; + int truncatedCount = 0; + String keyMarker = ""; + String uploadIdMarker = ""; + Set retrievedKeys = new HashSet<>(); + boolean hasMore = true; + + while (hasMore) { + OzoneMultipartUploadList result = bucket.listMultipartUploads( + "dir", keyMarker, uploadIdMarker, maxUploads); + + assertThat(result.getUploads()) + .as("Number of uploads should not exceed maxUploads") + .hasSizeLessThanOrEqualTo(maxUploads); + + assertEquals(result.getUploads().size(), result.getMaxUploads()); + + for (OzoneMultipartUpload upload : result.getUploads()) { + String key = upload.getKeyName(); + retrievedKeys.add(key); + + assertEquals(keyToUploadId.get(key), upload.getUploadId()); + } + + // Update markers for next iteration + keyMarker = result.getNextKeyMarker(); + uploadIdMarker = result.getNextUploadIdMarker(); + hasMore = result.isTruncated(); + + truncatedCount += result.isTruncated() ? 1 : 0; + } + + assertEquals(keys.size(), retrievedKeys.size()); + assertEquals(expectedTruncated, truncatedCount); + assertThat(retrievedKeys.stream().sorted().collect(Collectors.toList())) + .as("Retrieved keys should match expected keys in order") + .isEqualTo(keys.stream().sorted().collect(Collectors.toList())); + } + @Test void testGetAllPartsWhenZeroPartNumber() throws Exception { String parentDir = "a/b/c/d/e/f/"; From 155ccacf0d9e6ab278a93da1844d351d3e30f50b Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 9 Feb 2025 23:32:01 +0800 Subject: [PATCH 14/29] Reorder parameters in BucketEndpoint#listMultipartUploads method signature --- .../org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java | 6 +++--- .../hadoop/ozone/s3/endpoint/TestPermissionCheck.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 1b8d7d3a52f5..4ce5d3744b82 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -139,7 +139,7 @@ public Response get( if (uploads != null) { s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; - return listMultipartUploads(bucketName, prefix, 100, null, null); + return listMultipartUploads(bucketName, prefix, null, null, maxKeys); } if (prefix == null) { @@ -331,9 +331,9 @@ public Response put(@PathParam("bucket") String bucketName, public Response listMultipartUploads( @PathParam("bucket") String bucketName, @QueryParam("prefix") String prefix, - @QueryParam("max-uploads") @DefaultValue("100") @Min(1) @Max(1000) int maxUploads, @QueryParam("key-marker") String keyMarker, - @QueryParam("upload-id-marker") String uploadIdMarker) + @QueryParam("upload-id-marker") String uploadIdMarker, + @QueryParam("max-uploads") @DefaultValue("1000") @Min(1) @Max(1000) int maxUploads) throws OS3Exception, IOException { long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index ade1f26d649d..e27a504b53fe 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -152,7 +152,7 @@ public void testListMultiUpload() throws IOException { .setClient(client) .build(); OS3Exception e = assertThrows(OS3Exception.class, () -> - bucketEndpoint.listMultipartUploads("bucketName", "prefix", 10, "", "")); + bucketEndpoint.listMultipartUploads("bucketName", "prefix", "", "", 10)); assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); } From c7c8b82675de754a28f045a5705961cbf526ad0c Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 9 Feb 2025 23:33:09 +0800 Subject: [PATCH 15/29] fix typo --- .../main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 471cd6d51887..d0cef006e616 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -848,7 +848,7 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, multipartKeyInfo.getReplicationConfig()); } catch (IOException e) { LOG.warn( - "Open key entry for multipart upload record can be read {}", + "Open key entry for multipart upload record can't be read {}", metadataManager.getOzoneKey(upload.getVolumeName(), upload.getBucketName(), upload.getKeyName())); } From abf7e88d3f9837506dcecde9823427e193868a2c Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 9 Feb 2025 23:40:17 +0800 Subject: [PATCH 16/29] List keys only from DB and check tombstone from table partial cache --- .../ozone/om/OmMetadataManagerImpl.java | 41 ++++-------- .../hadoop/ozone/om/OmTestManagers.java | 4 ++ .../hadoop/ozone/om/TestKeyManagerUnit.java | 63 +++++++------------ 3 files changed, 38 insertions(+), 70 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 0a9695435328..116359affe86 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -38,8 +38,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.validation.constraints.NotNull; - import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; @@ -1932,53 +1930,36 @@ public long countEstimatedRowsInTable(Table table) @Override public MultipartUploadKeys getMultipartUploadKeys( - String volumeName, String bucketName, String prefix, @NotNull String keyMarker, - @NotNull String uploadIdMarker, int maxUploads) throws IOException { + String volumeName, String bucketName, String prefix, String keyMarker, + String uploadIdMarker, int maxUploads) throws IOException { MultipartUploadKeys.Builder resultBuilder = MultipartUploadKeys.newBuilder(); Set responseKeys = new TreeSet<>(); - Set aborted = new TreeSet<>(); String lastKey = null; - Iterator, CacheValue>> - cacheIterator = getMultipartInfoTable().cacheIterator(); - String prefixKey = OmMultipartUpload.getDbKey(volumeName, bucketName, prefix); - if (!keyMarker.equals("")) { + if (!keyMarker.isEmpty()) { prefix = keyMarker; - if (!uploadIdMarker.equals("")) { + if (!uploadIdMarker.isEmpty()) { prefix = prefix + OM_KEY_PREFIX + uploadIdMarker; } } String startKey = OmMultipartUpload.getDbKey(volumeName, bucketName, prefix); - // First iterate all the entries in cache. - while (cacheIterator.hasNext() && responseKeys.size() < maxUploads + 1) { - Map.Entry, CacheValue> cacheEntry = - cacheIterator.next(); - if (cacheEntry.getKey().getCacheKey().startsWith(prefixKey)) { - // Check if it is marked for delete, due to abort mpu - if (cacheEntry.getValue().getCacheValue() != null) { - responseKeys.add(cacheEntry.getKey().getCacheKey()); - lastKey = cacheEntry.getKey().getCacheKey(); - } else { - aborted.add(cacheEntry.getKey().getCacheKey()); - } - } - } - // prefixed iterator will only iterate until keys match the prefix try (TableIterator> - iterator = getMultipartInfoTable().iterator()) { + iterator = getMultipartInfoTable().iterator(prefixKey)) { iterator.seek(startKey); while (iterator.hasNext() && responseKeys.size() < maxUploads + 1) { KeyValue entry = iterator.next(); if (entry.getKey().startsWith(prefixKey)) { // If it is marked for abort, skip it. - if (!aborted.contains(entry.getKey())) { + CacheValue cacheValue = getMultipartInfoTable() + .getCacheValue(new CacheKey(entry.getKey())); + if (cacheValue == null || cacheValue.getCacheValue() != null) { responseKeys.add(entry.getKey()); lastKey = entry.getKey(); } @@ -1992,9 +1973,9 @@ public MultipartUploadKeys getMultipartUploadKeys( responseKeys.remove(lastKey); OmMultipartUpload lastKeyMultipartUpload = OmMultipartUpload.from(lastKey); - resultBuilder.setNextKeyMarker(lastKeyMultipartUpload.getKeyName()); - resultBuilder.setNextUploadIdMarker(lastKeyMultipartUpload.getUploadId()); - resultBuilder.setIsTruncated(true); + resultBuilder.setNextKeyMarker(lastKeyMultipartUpload.getKeyName()) + .setNextUploadIdMarker(lastKeyMultipartUpload.getUploadId()) + .setIsTruncated(true); } return resultBuilder diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java index c7a14bb6eedc..8c8117f7b96e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java @@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine; import org.apache.hadoop.security.authentication.client.AuthenticationException; import java.io.IOException; @@ -82,6 +83,9 @@ public ScmBlockLocationProtocol getScmBlockClient() { public OzoneClient getRpcClient() { return rpcClient; } + public OzoneManagerStateMachine getOzoneManagerStateMachine() { + return om.getOmRatisServer().getOmStateMachine(); + } public OmTestManagers(OzoneConfiguration conf) throws AuthenticationException, IOException, InterruptedException, TimeoutException { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index 01e1e54c8125..898b48f723d8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -69,6 +69,7 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -113,7 +114,7 @@ class TestKeyManagerUnit extends OzoneTestBase { private OzoneManagerProtocol writeClient; private OzoneManager om; - + private OzoneManagerStateMachine omStateMachine; @BeforeAll void setup(@TempDir Path testDir) throws Exception { ExitUtils.disableSystemExit(); @@ -131,6 +132,7 @@ void setup(@TempDir Path testDir) throws Exception { metadataManager = omTestManagers.getMetadataManager(); keyManager = (KeyManagerImpl)omTestManagers.getKeyManager(); writeClient = omTestManagers.getWriteClient(); + omStateMachine = omTestManagers.getOzoneManagerStateMachine(); } @BeforeEach @@ -252,23 +254,18 @@ public void listMultipartUploads() throws IOException { } @Test - public void listMultipartUploadsWithFewEntriesInCache() throws IOException { + public void listMultipartUploadsWithAbortedUpload() throws IOException, InterruptedException { String volume = volumeName(); String bucket = "bucket"; //GIVEN createBucket(metadataManager, volume, bucket); - createBucket(metadataManager, volume, bucket); - - - // Add few to cache and few to DB. - addinitMultipartUploadToCache(volume, bucket, "dir/key1"); + // Add few to DB. + initMultipartUpload(writeClient, volume, bucket, "dir/key1"); initMultipartUpload(writeClient, volume, bucket, "dir/key2"); - - addinitMultipartUploadToCache(volume, bucket, "dir/key3"); - - initMultipartUpload(writeClient, volume, bucket, "dir/key4"); + // Wait for double buffer to flush. (cleanup cache and write records to DB) + omStateMachine.awaitDoubleBufferFlush(); //WHEN OmMultipartUploadList omMultipartUploadList = @@ -276,39 +273,30 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { //THEN List uploads = omMultipartUploadList.getUploads(); - assertEquals(4, uploads.size()); + assertEquals(2, uploads.size()); assertEquals("dir/key1", uploads.get(0).getKeyName()); assertEquals("dir/key2", uploads.get(1).getKeyName()); - assertEquals("dir/key3", uploads.get(2).getKeyName()); - assertEquals("dir/key4", uploads.get(3).getKeyName()); // Add few more to test prefix. - - // Same way add few to cache and few to DB. - addinitMultipartUploadToCache(volume, bucket, "dir/ozonekey1"); - - initMultipartUpload(writeClient, volume, bucket, "dir/ozonekey2"); - - OmMultipartInfo omMultipartInfo3 = addinitMultipartUploadToCache(volume, - bucket, "dir/ozonekey3"); - - OmMultipartInfo omMultipartInfo4 = initMultipartUpload(writeClient, - volume, bucket, "dir/ozonekey4"); + OmMultipartInfo omMultipartInfo1 = initMultipartUpload(writeClient, + volume, bucket, "dir/ozonekey1"); + OmMultipartInfo omMultipartInfo2 = initMultipartUpload(writeClient, + volume, bucket, "dir/ozonekey2"); + // Wait for double buffer to flush. (cleanup cache and write records to DB) + omStateMachine.awaitDoubleBufferFlush(); omMultipartUploadList = keyManager.listMultipartUploads(volume, bucket, "dir/ozone", "", "", 10); //THEN uploads = omMultipartUploadList.getUploads(); - assertEquals(4, uploads.size()); + assertEquals(2, uploads.size()); assertEquals("dir/ozonekey1", uploads.get(0).getKeyName()); assertEquals("dir/ozonekey2", uploads.get(1).getKeyName()); - assertEquals("dir/ozonekey3", uploads.get(2).getKeyName()); - assertEquals("dir/ozonekey4", uploads.get(3).getKeyName()); // Abort multipart upload for key in DB. - abortMultipart(volume, bucket, "dir/ozonekey4", - omMultipartInfo4.getUploadID()); + abortMultipart(volume, bucket, "dir/ozonekey1", + omMultipartInfo1.getUploadID()); // Now list. omMultipartUploadList = @@ -316,14 +304,12 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { //THEN uploads = omMultipartUploadList.getUploads(); - assertEquals(3, uploads.size()); - assertEquals("dir/ozonekey1", uploads.get(0).getKeyName()); - assertEquals("dir/ozonekey2", uploads.get(1).getKeyName()); - assertEquals("dir/ozonekey3", uploads.get(2).getKeyName()); + assertEquals(1, uploads.size()); + assertEquals("dir/ozonekey2", uploads.get(0).getKeyName()); // abort multipart upload for key in cache. - abortMultipart(volume, bucket, "dir/ozonekey3", - omMultipartInfo3.getUploadID()); + abortMultipart(volume, bucket, "dir/ozonekey2", + omMultipartInfo2.getUploadID()); // Now list. omMultipartUploadList = @@ -331,10 +317,7 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { //THEN uploads = omMultipartUploadList.getUploads(); - assertEquals(2, uploads.size()); - assertEquals("dir/ozonekey1", uploads.get(0).getKeyName()); - assertEquals("dir/ozonekey2", uploads.get(1).getKeyName()); - + assertEquals(0, uploads.size()); } @Test From 7a9731af94d372775d34a1678c73f8d3ea999638 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 16 Feb 2025 15:17:07 +0800 Subject: [PATCH 17/29] create a new file for MultipartUploadKeys --- .../ozone/om/helpers/MultipartUploadKeys.java | 92 +++++++++++++++++++ .../hadoop/ozone/om/OMMetadataManager.java | 74 +-------------- .../hadoop/ozone/om/KeyManagerImpl.java | 3 +- .../ozone/om/OmMetadataManagerImpl.java | 1 + 4 files changed, 96 insertions(+), 74 deletions(-) create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java new file mode 100644 index 000000000000..80c91b139350 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.helpers; + +import java.util.Set; + +/** + * This class is used to store the result of OmMetadataManager#getMultipartUploadKeys. + */ +public class MultipartUploadKeys { + private final Set keys; + private final String nextKeyMarker; + private final String nextUploadIdMarker; + private final boolean isTruncated; + + public MultipartUploadKeys(Set keys, String nextKeyMarker, String nextUploadIdMarker, boolean isTruncated) { + this.keys = keys; + this.nextKeyMarker = nextKeyMarker; + this.nextUploadIdMarker = nextUploadIdMarker; + this.isTruncated = isTruncated; + } + + public Set getKeys() { + return keys; + } + + public String getNextKeyMarker() { + return nextKeyMarker; + } + + public String getNextUploadIdMarker() { + return nextUploadIdMarker; + } + + public boolean isTruncated() { + return isTruncated; + } + + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Builder class for MultipartUploadKeys. + */ + public static class Builder { + private Set keys; + private String nextKeyMarker = ""; + private String nextUploadIdMarker = ""; + private boolean isTruncated = false; + + public Builder setKeys(Set keys) { + this.keys = keys; + return this; + } + + public Builder setNextKeyMarker(String nextKeyMarker) { + this.nextKeyMarker = nextKeyMarker; + return this; + } + + public Builder setNextUploadIdMarker(String nextUploadIdMarker) { + this.nextUploadIdMarker = nextUploadIdMarker; + return this; + } + + public Builder setIsTruncated(boolean isTruncated) { + this.isTruncated = isTruncated; + return this; + } + + public MultipartUploadKeys build() { + return new MultipartUploadKeys(keys, nextKeyMarker, nextUploadIdMarker, isTruncated); + } + + } +} diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index ff3dde843e3f..d9e1be1072bf 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult; +import org.apache.hadoop.ozone.om.helpers.MultipartUploadKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo; import org.apache.hadoop.ozone.om.helpers.OmDBUserPrincipalInfo; @@ -490,79 +491,6 @@ long countRowsInTable(Table table) long countEstimatedRowsInTable(Table table) throws IOException; - - /** - * This class is used to store the result of listMultipartUploads. - */ - class MultipartUploadKeys { - private final Set keys; - private final String nextKeyMarker; - private final String nextUploadIdMarker; - private final boolean isTruncated; - - public MultipartUploadKeys(Set keys, String nextKeyMarker, String nextUploadIdMarker, boolean isTruncated) { - this.keys = keys; - this.nextKeyMarker = nextKeyMarker; - this.nextUploadIdMarker = nextUploadIdMarker; - this.isTruncated = isTruncated; - } - - public Set getKeys() { - return keys; - } - - public String getNextKeyMarker() { - return nextKeyMarker; - } - - public String getNextUploadIdMarker() { - return nextUploadIdMarker; - } - - public boolean isTruncated() { - return isTruncated; - } - - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Builder class for MultipartUploadKeys. - */ - public static class Builder { - private Set keys; - private String nextKeyMarker = ""; - private String nextUploadIdMarker = ""; - private boolean isTruncated = false; - - public Builder setKeys(Set keys) { - this.keys = keys; - return this; - } - - public Builder setNextKeyMarker(String nextKeyMarker) { - this.nextKeyMarker = nextKeyMarker; - return this; - } - - public Builder setNextUploadIdMarker(String nextUploadIdMarker) { - this.nextUploadIdMarker = nextUploadIdMarker; - return this; - } - - public Builder setIsTruncated(boolean isTruncated) { - this.isTruncated = isTruncated; - return this; - } - - public MultipartUploadKeys build() { - return new MultipartUploadKeys(keys, nextKeyMarker, nextUploadIdMarker, isTruncated); - } - - } - } - /** * Return the existing upload keys which includes volumeName, bucketName, * keyName. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index d0cef006e616..9fd4ca4587e5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -79,6 +79,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.ListKeysResult; +import org.apache.hadoop.ozone.om.helpers.MultipartUploadKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -828,7 +829,7 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, bucketName); try { - OMMetadataManager.MultipartUploadKeys multipartUploadKeys = + MultipartUploadKeys multipartUploadKeys = metadataManager .getMultipartUploadKeys(volumeName, bucketName, prefix, keyMarker, uploadIdMarker, maxUploads); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 2f3e619e980a..389a5643c5ca 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -62,6 +62,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult; +import org.apache.hadoop.ozone.om.helpers.MultipartUploadKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo; import org.apache.hadoop.ozone.om.helpers.OmDBUserPrincipalInfo; From 6fcdf5f88c85569425bb15d7dd872d5b08dd5bb8 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sun, 16 Feb 2025 15:19:35 +0800 Subject: [PATCH 18/29] list multipartinfo should get all entries from cache, too --- ...TestOzoneClientMultipartUploadWithFSO.java | 5 -- .../ozone/om/OmMetadataManagerImpl.java | 64 +++++++++++++------ .../hadoop/ozone/om/OmTestManagers.java | 4 -- .../hadoop/ozone/om/TestKeyManagerUnit.java | 63 +++++++++++------- 4 files changed, 83 insertions(+), 53 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java index af5412fcea47..f78d8cad993d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java @@ -100,9 +100,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * This test verifies all the S3 multipart client apis - prefix layout. */ @@ -120,8 +117,6 @@ public class TestOzoneClientMultipartUploadWithFSO { private OzoneVolume volume; private OzoneBucket bucket; - private static final Logger LOG = LoggerFactory.getLogger(TestOzoneClientMultipartUploadWithFSO.class); - /** * Create a MiniOzoneCluster for testing. *

diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 389a5643c5ca..6ec0e2c63365 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -32,6 +33,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.TimeUnit; @@ -1941,48 +1943,68 @@ public MultipartUploadKeys getMultipartUploadKeys( String uploadIdMarker, int maxUploads) throws IOException { MultipartUploadKeys.Builder resultBuilder = MultipartUploadKeys.newBuilder(); - Set responseKeys = new TreeSet<>(); - String lastKey = null; + SortedSet responseKeys = new TreeSet<>(); + Set aborted = new HashSet<>(); String prefixKey = OmMultipartUpload.getDbKey(volumeName, bucketName, prefix); - if (!keyMarker.isEmpty()) { + if (StringUtil.isNotBlank(keyMarker)) { prefix = keyMarker; - if (!uploadIdMarker.isEmpty()) { + if (StringUtil.isNotBlank(uploadIdMarker)) { prefix = prefix + OM_KEY_PREFIX + uploadIdMarker; } } - String startKey = OmMultipartUpload.getDbKey(volumeName, bucketName, prefix); + String seekKey = OmMultipartUpload.getDbKey(volumeName, bucketName, prefix); - // prefixed iterator will only iterate until keys match the prefix + Iterator, CacheValue>> + cacheIterator = getMultipartInfoTable().cacheIterator(); + // First iterate all the entries in cache. + while (cacheIterator.hasNext()) { + Map.Entry, CacheValue> cacheEntry = + cacheIterator.next(); + String cacheKey = cacheEntry.getKey().getCacheKey(); + if (cacheKey.startsWith(prefixKey)) { + // Check if it is marked for delete, due to abort mpu + if (cacheEntry.getValue().getCacheValue() != null && + cacheKey.compareTo(seekKey) >= 0) { + responseKeys.add(cacheKey); + } else { + aborted.add(cacheKey); + } + } + } + + int dbKeysCount = 0; + // the prefix iterator will only iterate keys that match the given prefix + // so we don't need to check if the key is started with prefixKey again try (TableIterator> iterator = getMultipartInfoTable().iterator(prefixKey)) { - iterator.seek(startKey); + iterator.seek(seekKey); - while (iterator.hasNext() && responseKeys.size() < maxUploads + 1) { + while (iterator.hasNext() && dbKeysCount < maxUploads + 1) { KeyValue entry = iterator.next(); - if (entry.getKey().startsWith(prefixKey)) { - // If it is marked for abort, skip it. - CacheValue cacheValue = getMultipartInfoTable() - .getCacheValue(new CacheKey(entry.getKey())); - if (cacheValue == null || cacheValue.getCacheValue() != null) { - responseKeys.add(entry.getKey()); - lastKey = entry.getKey(); - } - } else { - break; + // If it is marked for abort, skip it. + if (!aborted.contains(entry.getKey())) { + responseKeys.add(entry.getKey()); + dbKeysCount++; } } } - if (lastKey != null && responseKeys.size() == maxUploads + 1) { - responseKeys.remove(lastKey); - + String lastKey = responseKeys.stream() + .skip(maxUploads) + .findFirst() + .orElse(null); + if (lastKey != null) { + // implies the keyset size is greater than maxUploads OmMultipartUpload lastKeyMultipartUpload = OmMultipartUpload.from(lastKey); resultBuilder.setNextKeyMarker(lastKeyMultipartUpload.getKeyName()) .setNextUploadIdMarker(lastKeyMultipartUpload.getUploadId()) .setIsTruncated(true); + + // keep the [0, maxUploads] keys + responseKeys = responseKeys.subSet(responseKeys.first(), lastKey); } return resultBuilder diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java index 8c8117f7b96e..c7a14bb6eedc 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java @@ -33,7 +33,6 @@ import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus; -import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine; import org.apache.hadoop.security.authentication.client.AuthenticationException; import java.io.IOException; @@ -83,9 +82,6 @@ public ScmBlockLocationProtocol getScmBlockClient() { public OzoneClient getRpcClient() { return rpcClient; } - public OzoneManagerStateMachine getOzoneManagerStateMachine() { - return om.getOmRatisServer().getOmStateMachine(); - } public OmTestManagers(OzoneConfiguration conf) throws AuthenticationException, IOException, InterruptedException, TimeoutException { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index 898b48f723d8..01e1e54c8125 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -69,7 +69,6 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; -import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -114,7 +113,7 @@ class TestKeyManagerUnit extends OzoneTestBase { private OzoneManagerProtocol writeClient; private OzoneManager om; - private OzoneManagerStateMachine omStateMachine; + @BeforeAll void setup(@TempDir Path testDir) throws Exception { ExitUtils.disableSystemExit(); @@ -132,7 +131,6 @@ void setup(@TempDir Path testDir) throws Exception { metadataManager = omTestManagers.getMetadataManager(); keyManager = (KeyManagerImpl)omTestManagers.getKeyManager(); writeClient = omTestManagers.getWriteClient(); - omStateMachine = omTestManagers.getOzoneManagerStateMachine(); } @BeforeEach @@ -254,18 +252,23 @@ public void listMultipartUploads() throws IOException { } @Test - public void listMultipartUploadsWithAbortedUpload() throws IOException, InterruptedException { + public void listMultipartUploadsWithFewEntriesInCache() throws IOException { String volume = volumeName(); String bucket = "bucket"; //GIVEN createBucket(metadataManager, volume, bucket); + createBucket(metadataManager, volume, bucket); + + + // Add few to cache and few to DB. + addinitMultipartUploadToCache(volume, bucket, "dir/key1"); - // Add few to DB. - initMultipartUpload(writeClient, volume, bucket, "dir/key1"); initMultipartUpload(writeClient, volume, bucket, "dir/key2"); - // Wait for double buffer to flush. (cleanup cache and write records to DB) - omStateMachine.awaitDoubleBufferFlush(); + + addinitMultipartUploadToCache(volume, bucket, "dir/key3"); + + initMultipartUpload(writeClient, volume, bucket, "dir/key4"); //WHEN OmMultipartUploadList omMultipartUploadList = @@ -273,30 +276,39 @@ public void listMultipartUploadsWithAbortedUpload() throws IOException, Interrup //THEN List uploads = omMultipartUploadList.getUploads(); - assertEquals(2, uploads.size()); + assertEquals(4, uploads.size()); assertEquals("dir/key1", uploads.get(0).getKeyName()); assertEquals("dir/key2", uploads.get(1).getKeyName()); + assertEquals("dir/key3", uploads.get(2).getKeyName()); + assertEquals("dir/key4", uploads.get(3).getKeyName()); // Add few more to test prefix. - OmMultipartInfo omMultipartInfo1 = initMultipartUpload(writeClient, - volume, bucket, "dir/ozonekey1"); - OmMultipartInfo omMultipartInfo2 = initMultipartUpload(writeClient, - volume, bucket, "dir/ozonekey2"); - // Wait for double buffer to flush. (cleanup cache and write records to DB) - omStateMachine.awaitDoubleBufferFlush(); + + // Same way add few to cache and few to DB. + addinitMultipartUploadToCache(volume, bucket, "dir/ozonekey1"); + + initMultipartUpload(writeClient, volume, bucket, "dir/ozonekey2"); + + OmMultipartInfo omMultipartInfo3 = addinitMultipartUploadToCache(volume, + bucket, "dir/ozonekey3"); + + OmMultipartInfo omMultipartInfo4 = initMultipartUpload(writeClient, + volume, bucket, "dir/ozonekey4"); omMultipartUploadList = keyManager.listMultipartUploads(volume, bucket, "dir/ozone", "", "", 10); //THEN uploads = omMultipartUploadList.getUploads(); - assertEquals(2, uploads.size()); + assertEquals(4, uploads.size()); assertEquals("dir/ozonekey1", uploads.get(0).getKeyName()); assertEquals("dir/ozonekey2", uploads.get(1).getKeyName()); + assertEquals("dir/ozonekey3", uploads.get(2).getKeyName()); + assertEquals("dir/ozonekey4", uploads.get(3).getKeyName()); // Abort multipart upload for key in DB. - abortMultipart(volume, bucket, "dir/ozonekey1", - omMultipartInfo1.getUploadID()); + abortMultipart(volume, bucket, "dir/ozonekey4", + omMultipartInfo4.getUploadID()); // Now list. omMultipartUploadList = @@ -304,12 +316,14 @@ public void listMultipartUploadsWithAbortedUpload() throws IOException, Interrup //THEN uploads = omMultipartUploadList.getUploads(); - assertEquals(1, uploads.size()); - assertEquals("dir/ozonekey2", uploads.get(0).getKeyName()); + assertEquals(3, uploads.size()); + assertEquals("dir/ozonekey1", uploads.get(0).getKeyName()); + assertEquals("dir/ozonekey2", uploads.get(1).getKeyName()); + assertEquals("dir/ozonekey3", uploads.get(2).getKeyName()); // abort multipart upload for key in cache. - abortMultipart(volume, bucket, "dir/ozonekey2", - omMultipartInfo2.getUploadID()); + abortMultipart(volume, bucket, "dir/ozonekey3", + omMultipartInfo3.getUploadID()); // Now list. omMultipartUploadList = @@ -317,7 +331,10 @@ public void listMultipartUploadsWithAbortedUpload() throws IOException, Interrup //THEN uploads = omMultipartUploadList.getUploads(); - assertEquals(0, uploads.size()); + assertEquals(2, uploads.size()); + assertEquals("dir/ozonekey1", uploads.get(0).getKeyName()); + assertEquals("dir/ozonekey2", uploads.get(1).getKeyName()); + } @Test From 1b6bdf7e83da8e7d11675bc76a478e39fc6f70c5 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 18 Feb 2025 03:16:29 +0800 Subject: [PATCH 19/29] Move listMultipartUploads params to GET method handler --- .../ozone/s3/endpoint/BucketEndpoint.java | 23 ++++++---- .../ozone/s3/endpoint/TestBucketAcl.java | 2 +- .../ozone/s3/endpoint/TestBucketList.java | 46 +++++++++---------- .../s3/endpoint/TestPermissionCheck.java | 6 +-- .../s3/metrics/TestS3GatewayMetrics.java | 8 ++-- 5 files changed, 46 insertions(+), 39 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 8750de66c52f..93280911292e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -38,8 +38,6 @@ import java.util.Objects; import java.util.Set; import javax.inject.Inject; -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; @@ -116,6 +114,9 @@ public Response get( @QueryParam("start-after") String startAfter, @QueryParam("uploads") String uploads, @QueryParam("acl") String aclMarker, + @QueryParam("key-marker") String keyMarker, + @QueryParam("upload-id-marker") String uploadIdMarker, + @DefaultValue("1000") @QueryParam("max-uploads") int maxUploads, @Context HttpHeaders hh) throws OS3Exception, IOException { long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.GET_BUCKET; @@ -138,7 +139,7 @@ public Response get( if (uploads != null) { s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; - return listMultipartUploads(bucketName, prefix, null, null, maxKeys); + return listMultipartUploads(bucketName, prefix, keyMarker, uploadIdMarker, maxUploads); } if (prefix == null) { @@ -328,12 +329,18 @@ public Response put(@PathParam("bucket") String bucketName, } public Response listMultipartUploads( - @PathParam("bucket") String bucketName, - @QueryParam("prefix") String prefix, - @QueryParam("key-marker") String keyMarker, - @QueryParam("upload-id-marker") String uploadIdMarker, - @QueryParam("max-uploads") @DefaultValue("1000") @Min(1) @Max(1000) int maxUploads) + String bucketName, + String prefix, + String keyMarker, + String uploadIdMarker, + int maxUploads) throws OS3Exception, IOException { + + if (maxUploads < 1 || maxUploads > 1000) { + throw newError(S3ErrorTable.INVALID_ARGUMENT, "max-uploads", + new Exception("max-uploads must be between 1 and 1000")); + } + long startNanos = Time.monotonicNowNanos(); S3GAction s3GAction = S3GAction.LIST_MULTIPART_UPLOAD; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java index bbc380a20de7..1099fbea98cd 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketAcl.java @@ -82,7 +82,7 @@ public void testGetAcl() throws Exception { when(parameterMap.containsKey(ACL_MARKER)).thenReturn(true); Response response = bucketEndpoint.get(BUCKET_NAME, null, null, null, 0, null, - null, null, null, ACL_MARKER, headers); + null, null, null, ACL_MARKER, null, null, 0, headers); assertEquals(HTTP_OK, response.getStatus()); System.out.println(response.getEntity()); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java index 81960b9d5276..b3589afbfd46 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java @@ -54,7 +54,7 @@ public void listRoot() throws OS3Exception, IOException { ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, "", - null, null, null, null, null) + null, null, null, null, null, null, 0, null) .getEntity(); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); @@ -79,7 +79,7 @@ public void listDir() throws OS3Exception, IOException { ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, - "dir1", null, null, null, null, null).getEntity(); + "dir1", null, null, null, null, null, null, 0, null).getEntity(); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); assertEquals("dir1/", @@ -103,7 +103,7 @@ public void listSubDir() throws OS3Exception, IOException { ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket .get("b1", "/", null, null, 100, "dir1/", null, - null, null, null, null) + null, null, null, null, null, 0, null) .getEntity(); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); @@ -139,7 +139,7 @@ public void listObjectOwner() throws OS3Exception, IOException { getBucket.setRequestIdentifier(new RequestIdentifier()); ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, - "key", null, null, null, null, null).getEntity(); + "key", null, null, null, null, null, null, 0, null).getEntity(); assertEquals(2, getBucketResponse.getContents().size()); assertEquals(user1.getShortUserName(), @@ -163,7 +163,7 @@ public void listWithPrefixAndDelimiter() throws OS3Exception, IOException { ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, - "dir1", null, null, null, null, null).getEntity(); + "dir1", null, null, null, null, null, null, 0, null).getEntity(); assertEquals(3, getBucketResponse.getCommonPrefixes().size()); @@ -183,7 +183,7 @@ public void listWithPrefixAndDelimiter1() throws OS3Exception, IOException { ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, - "", null, null, null, null, null).getEntity(); + "", null, null, null, null, null, null, 0, null).getEntity(); assertEquals(3, getBucketResponse.getCommonPrefixes().size()); assertEquals("file2", getBucketResponse.getContents().get(0) @@ -204,7 +204,7 @@ public void listWithPrefixAndDelimiter2() throws OS3Exception, IOException { getBucket.setRequestIdentifier(new RequestIdentifier()); ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket.get("b1", "/", null, null, 100, "dir1bh", - null, "dir1/dir2/file2", null, null, null).getEntity(); + null, "dir1/dir2/file2", null, null, null, null, 0, null).getEntity(); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); @@ -224,7 +224,7 @@ public void listWithPrefixAndEmptyStrDelimiter() // Should behave the same if delimiter is null ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket.get("b1", "", null, null, 100, "dir1/", - null, null, null, null, null).getEntity(); + null, null, null, null, null, null, 0, null).getEntity(); assertEquals(0, getBucketResponse.getCommonPrefixes().size()); assertEquals(4, getBucketResponse.getContents().size()); @@ -256,7 +256,7 @@ public void listWithContinuationToken() throws OS3Exception, IOException { // First time ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket.get("b1", null, null, null, maxKeys, - "", null, null, null, null, null).getEntity(); + "", null, null, null, null, null, null, 0, null).getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getContents().size()); @@ -265,7 +265,7 @@ public void listWithContinuationToken() throws OS3Exception, IOException { String continueToken = getBucketResponse.getNextToken(); getBucketResponse = (ListObjectResponse) getBucket.get("b1", null, null, null, maxKeys, - "", continueToken, null, null, null, null).getEntity(); + "", continueToken, null, null, null, null, null, 0, null).getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getContents().size()); @@ -275,7 +275,7 @@ public void listWithContinuationToken() throws OS3Exception, IOException { //3rd time getBucketResponse = (ListObjectResponse) getBucket.get("b1", null, null, null, maxKeys, - "", continueToken, null, null, null, null).getEntity(); + "", continueToken, null, null, null, null, null, 0, null).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(1, getBucketResponse.getContents().size()); @@ -308,7 +308,7 @@ public void listWithContinuationTokenDirBreak() getBucketResponse = (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, - "test/", null, null, null, null, null).getEntity(); + "test/", null, null, null, null, null, null, 0, null).getEntity(); assertEquals(0, getBucketResponse.getContents().size()); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); @@ -320,7 +320,7 @@ public void listWithContinuationTokenDirBreak() getBucketResponse = (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, "test/", getBucketResponse.getNextToken(), null, null, null, - null).getEntity(); + null, null, 0, null).getEntity(); assertEquals(1, getBucketResponse.getContents().size()); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); assertEquals("test/dir3/", @@ -352,7 +352,7 @@ public void listWithContinuationToken1() throws OS3Exception, IOException { // First time ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, - "dir", null, null, null, null, null).getEntity(); + "dir", null, null, null, null, null, null, 0, null).getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); @@ -361,7 +361,7 @@ public void listWithContinuationToken1() throws OS3Exception, IOException { String continueToken = getBucketResponse.getNextToken(); getBucketResponse = (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, - "dir", continueToken, null, null, null, null).getEntity(); + "dir", continueToken, null, null, null, null, null, 0, null).getEntity(); assertTrue(getBucketResponse.isTruncated()); assertEquals(2, getBucketResponse.getCommonPrefixes().size()); @@ -369,7 +369,7 @@ public void listWithContinuationToken1() throws OS3Exception, IOException { continueToken = getBucketResponse.getNextToken(); getBucketResponse = (ListObjectResponse) getBucket.get("b1", "/", null, null, maxKeys, - "dir", continueToken, null, null, null, null).getEntity(); + "dir", continueToken, null, null, null, null, null, 0, null).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(1, getBucketResponse.getCommonPrefixes().size()); @@ -389,7 +389,7 @@ public void listWithContinuationTokenFail() throws IOException { getBucket.setRequestIdentifier(new RequestIdentifier()); OS3Exception e = assertThrows(OS3Exception.class, () -> getBucket.get("b1", - "/", null, null, 2, "dir", "random", null, null, null, null) + "/", null, null, 2, "dir", "random", null, null, null, null, null, 1000, null) .getEntity(), "listWithContinuationTokenFail"); assertEquals("random", e.getResource()); assertEquals("Invalid Argument", e.getErrorMessage()); @@ -409,7 +409,7 @@ public void testStartAfter() throws IOException, OS3Exception { ListObjectResponse getBucketResponse = (ListObjectResponse) getBucket.get("b1", null, null, null, 1000, - null, null, null, null, null, null).getEntity(); + null, null, null, null, null, null, null, 0, null).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(5, getBucketResponse.getContents().size()); @@ -420,14 +420,14 @@ public void testStartAfter() throws IOException, OS3Exception { getBucketResponse = (ListObjectResponse) getBucket.get("b1", null, null, null, - 1000, null, null, startAfter, null, null, null).getEntity(); + 1000, null, null, startAfter, null, null, null, null, 0, null).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(4, getBucketResponse.getContents().size()); getBucketResponse = (ListObjectResponse) getBucket.get("b1", null, null, null, - 1000, null, null, "random", null, null, null).getEntity(); + 1000, null, null, "random", null, null, null, null, 0, null).getEntity(); assertFalse(getBucketResponse.isTruncated()); assertEquals(0, getBucketResponse.getContents().size()); @@ -474,7 +474,7 @@ public void testEncodingType() throws IOException, OS3Exception { ListObjectResponse response = (ListObjectResponse) getBucket.get( "b1", delimiter, encodingType, null, 1000, prefix, - null, startAfter, null, null, null).getEntity(); + null, startAfter, null, null, null, null, 0, null).getEntity(); // Assert encodingType == url. // The Object name will be encoded by ObjectKeyNameAdapter @@ -492,7 +492,7 @@ public void testEncodingType() throws IOException, OS3Exception { response = (ListObjectResponse) getBucket.get( "b1", delimiter, null, null, 1000, prefix, - null, startAfter, null, null, null).getEntity(); + null, startAfter, null, null, null, null, 0, null).getEntity(); // Assert encodingType == null. // The Object name will not be encoded by ObjectKeyNameAdapter @@ -517,7 +517,7 @@ public void testEncodingTypeException() throws IOException { getBucket.setRequestIdentifier(new RequestIdentifier()); OS3Exception e = assertThrows(OS3Exception.class, () -> getBucket.get( "b1", null, "unSupportType", null, 1000, null, - null, null, null, null, null).getEntity()); + null, null, null, null, null, null, 0, null).getEntity()); assertEquals(S3ErrorTable.INVALID_ARGUMENT.getCode(), e.getCode()); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index 8f9ae9e24c55..3804e412fe59 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -163,7 +163,7 @@ public void testListKey() throws IOException { .build(); OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get( "bucketName", null, null, null, 1000, - null, null, null, null, null, null)); + null, null, null, null, null, null, null, 0, null)); assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); } @@ -207,8 +207,8 @@ public void testGetAcl() throws Exception { .setClient(client) .build(); OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get( - "bucketName", null, null, null, 1000, null, null, null, null, "acl", - null), "Expected OS3Exception with FORBIDDEN http code."); + "bucketName", null, null, null, 1000, null, null, null, null, "acl", + null, null, 0, null), "Expected OS3Exception with FORBIDDEN http code."); assertEquals(HTTP_FORBIDDEN, e.getHttpCode()); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java index 7c6afc509be5..0081d0d5abc0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java @@ -142,7 +142,7 @@ public void testGetBucketSuccess() throws Exception { bucketEndpoint.get(bucketName, null, null, null, 1000, null, null, "random", null, - null, null).getEntity(); + null, null, null, 0, null).getEntity(); long curMetric = metrics.getGetBucketSuccess(); assertEquals(1L, curMetric - oriMetric); @@ -155,7 +155,7 @@ public void testGetBucketFailure() throws Exception { // Searching for a bucket that does not exist OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get( "newBucket", null, null, null, 1000, null, null, "random", null, - null, null)); + null, null, null, 0, null)); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), e.getCode()); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getErrorMessage(), e.getErrorMessage()); @@ -219,7 +219,7 @@ public void testGetAclSuccess() throws Exception { Response response = bucketEndpoint.get(bucketName, null, null, null, 0, null, null, - null, null, "acl", null); + null, null, "acl", null, null, 0, null); long curMetric = metrics.getGetAclSuccess(); assertEquals(HTTP_OK, response.getStatus()); assertEquals(1L, curMetric - oriMetric); @@ -232,7 +232,7 @@ public void testGetAclFailure() throws Exception { // Failing the getACL endpoint by applying ACL on a non-Existent Bucket OS3Exception e = assertThrows(OS3Exception.class, () -> bucketEndpoint.get( "random_bucket", null, null, null, 0, null, - null, null, null, "acl", null)); + null, null, null, "acl", null, null, 0, null)); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getCode(), e.getCode()); assertEquals(S3ErrorTable.NO_SUCH_BUCKET.getErrorMessage(), e.getErrorMessage()); From a7a47e37a0a449520e3d84181e6638bbe34d5aee Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 18 Feb 2025 03:28:41 +0800 Subject: [PATCH 20/29] Add pagination test for keyManagerImpl and metadataManagerImpl --- .../hadoop/ozone/om/TestKeyManagerUnit.java | 69 +++++++++++++++ .../ozone/om/TestOmMetadataManager.java | 87 +++++++++++++++++++ .../ozone/om/request/OMRequestTestUtils.java | 19 ++++ 3 files changed, 175 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index 5287cd84c78c..979fe7e2ea49 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -23,6 +23,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.anySet; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; @@ -360,6 +361,74 @@ public void listMultipartUploadsWithPrefix() throws IOException { assertEquals("dir/key2", uploads.get(1).getKeyName()); } + @Test + public void testListMultipartUploadsWithPagination() throws IOException { + // GIVEN + final String volumeName = volumeName(); + final String bucketName = "bucket1"; + createBucket(metadataManager, volumeName, bucketName); + + // Create 25 multipart uploads to test pagination + List uploadInfos = new ArrayList<>(); + for (int i = 0; i < 25; i++) { + String key = String.format("key-%03d", i); // pad with zeros for proper sorting + OmMultipartInfo info = initMultipartUpload(writeClient, volumeName, bucketName, key); + uploadInfos.add(info); + } + + // WHEN - First page (10 entries) + OmMultipartUploadList firstPage = keyManager.listMultipartUploads( + volumeName, bucketName, "", "", "", 10); + + // THEN + assertEquals(10, firstPage.getUploads().size()); + assertTrue(firstPage.isTruncated()); + assertNotNull(firstPage.getNextKeyMarker()); + assertNotNull(firstPage.getNextUploadIdMarker()); + + // Verify first page content + for (int i = 0; i < 10; i++) { + assertEquals(String.format("key-%03d", i), + firstPage.getUploads().get(i).getKeyName()); + } + + // WHEN - Second page using markers from first page + OmMultipartUploadList secondPage = keyManager.listMultipartUploads( + volumeName, bucketName, "", + firstPage.getNextKeyMarker(), + firstPage.getNextUploadIdMarker(), + 10); + + // THEN + assertEquals(10, secondPage.getUploads().size()); + assertTrue(secondPage.isTruncated()); + + // Verify second page content + for (int i = 0; i < 10; i++) { + assertEquals(String.format("key-%03d", i + 10), + secondPage.getUploads().get(i).getKeyName()); + } + + // WHEN - Last page + OmMultipartUploadList lastPage = keyManager.listMultipartUploads( + volumeName, bucketName, "", + secondPage.getNextKeyMarker(), + secondPage.getNextUploadIdMarker(), + 10); + + // THEN + assertEquals(5, lastPage.getUploads().size()); + assertFalse(lastPage.isTruncated()); + assertEquals("", lastPage.getNextKeyMarker()); + assertEquals("", lastPage.getNextUploadIdMarker()); + + // Verify last page content + for (int i = 0; i < 5; i++) { + assertEquals(String.format("key-%03d", i + 20), + lastPage.getUploads().get(i).getKeyName()); + } + } + private void createBucket(OMMetadataManager omMetadataManager, String volume, String bucket) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index b995ecdc2d26..5fd6aa286b2e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -28,7 +28,9 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -39,6 +41,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -59,10 +62,12 @@ import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult; +import org.apache.hadoop.ozone.om.helpers.MultipartUploadKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; @@ -1106,4 +1111,86 @@ public void testListSnapshotDoesNotListOtherBucketSnapshots() assertTrue(snapshotInfo.getName().startsWith(snapshotName2)); } } + + @Test + public void testGetMultipartUploadKeys() throws Exception { + String volumeName = "vol1"; + String bucketName = "bucket1"; + String prefix = "dir/"; + int maxUploads = 10; + + // Create volume and bucket + OMRequestTestUtils.addVolumeToDB(volumeName, omMetadataManager); + addBucketsToCache(volumeName, bucketName); + + List expectedKeys = new ArrayList<>(); + for (int i = 0; i < 25; i++) { + String key = prefix + "key" + i; + String uploadId = OMMultipartUploadUtils.getMultipartUploadId(); + + // Create multipart key info + OmKeyInfo keyInfo = OMRequestTestUtils.createOmKeyInfo( + volumeName, bucketName, key, + RatisReplicationConfig.getInstance(ONE)) + .build(); + + OmMultipartKeyInfo multipartKeyInfo = OMRequestTestUtils + .createOmMultipartKeyInfo(uploadId, Time.now(), + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, 0L); + + if (i % 2 == 0) { + OMRequestTestUtils.addMultipartInfoToTable(false, keyInfo, + multipartKeyInfo, 0L, omMetadataManager); + } else { + OMRequestTestUtils.addMultipartInfoToTableCache(keyInfo, + multipartKeyInfo, 0L, omMetadataManager); + } + + expectedKeys.add(key); + } + Collections.sort(expectedKeys); + + // List first page without markers + MultipartUploadKeys result = omMetadataManager.getMultipartUploadKeys( + volumeName, bucketName, prefix, null, null, maxUploads); + + assertEquals(maxUploads, result.getKeys().size()); + assertTrue(result.isTruncated()); + assertNotNull(result.getNextKeyMarker()); + assertNotNull(result.getNextUploadIdMarker()); + + // List next page using markers from first page + MultipartUploadKeys nextPage = omMetadataManager.getMultipartUploadKeys( + volumeName, bucketName, prefix, + result.getNextKeyMarker(), + result.getNextUploadIdMarker(), + maxUploads); + + assertEquals(maxUploads, nextPage.getKeys().size()); + assertTrue(nextPage.isTruncated()); + + // List with different prefix + MultipartUploadKeys differentPrefix = omMetadataManager.getMultipartUploadKeys( + volumeName, bucketName, "different/", null, null, maxUploads); + + assertEquals(0, differentPrefix.getKeys().size()); + assertFalse(differentPrefix.isTruncated()); + + // List all entries with large maxUploads + MultipartUploadKeys allEntries = omMetadataManager.getMultipartUploadKeys( + volumeName, bucketName, prefix, null, null, 100); + + assertEquals(25, allEntries.getKeys().size()); + assertFalse(allEntries.isTruncated()); + + // Verify all keys are present + List actualKeys = new ArrayList<>(); + for (String dbKey : allEntries.getKeys()) { + OmMultipartUpload mpu = OmMultipartUpload.from(dbKey); + actualKeys.add(mpu.getKeyName()); + } + Collections.sort(actualKeys); + assertEquals(expectedKeys, actualKeys); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java index 394ddf029c29..1ce42bacd9e6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java @@ -397,6 +397,25 @@ public static String addMultipartInfoToTable(boolean addToCache, return ozoneDBKey; } + /** + * Add multipart info entry to the multipartInfoTable. + * + * @throws Exception + */ + public static String addMultipartInfoToTableCache( + OmKeyInfo omKeyInfo, OmMultipartKeyInfo omMultipartKeyInfo, + long trxnLogIndex, OMMetadataManager omMetadataManager) throws IOException { + String ozoneDBKey = omMetadataManager.getMultipartKey( + omKeyInfo.getVolumeName(), omKeyInfo.getBucketName(), + omKeyInfo.getKeyName(), omMultipartKeyInfo.getUploadID()); + + omMetadataManager.getMultipartInfoTable() + .addCacheEntry(new CacheKey<>(ozoneDBKey), + CacheValue.get(trxnLogIndex, omMultipartKeyInfo)); + + return ozoneDBKey; + } + public static PartKeyInfo createPartKeyInfo(String volumeName, String bucketName, String keyName, String uploadId, int partNumber) { return PartKeyInfo.newBuilder() From 1067b2e5deeacd7cee29d71809cd0946474ee2df Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 18 Feb 2025 03:33:07 +0800 Subject: [PATCH 21/29] Remove maxUploads from OzoneMultipartUploadList --- .../hadoop/ozone/client/OzoneMultipartUploadList.java | 9 +-------- .../org/apache/hadoop/ozone/client/rpc/RpcClient.java | 3 +-- .../apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java | 4 +--- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java index 01a18aab7aec..1a71182fc135 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadList.java @@ -29,20 +29,17 @@ public class OzoneMultipartUploadList { private String nextKeyMarker; private String nextUploadIdMarker; private boolean isTruncated; - private int maxUploads; public OzoneMultipartUploadList( List uploads, String nextKeyMarker, String nextUploadIdMarker, - boolean isTruncated, - int maxUploads) { + boolean isTruncated) { Preconditions.checkNotNull(uploads); this.uploads = uploads; this.nextKeyMarker = nextKeyMarker; this.nextUploadIdMarker = nextUploadIdMarker; this.isTruncated = isTruncated; - this.maxUploads = maxUploads; } public List getUploads() { @@ -65,8 +62,4 @@ public String getNextUploadIdMarker() { public boolean isTruncated() { return isTruncated; } - - public int getMaxUploads() { - return maxUploads; - } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index a7e75f6ac888..f0ae67786227 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2140,8 +2140,7 @@ public OzoneMultipartUploadList listMultipartUploads(String volumeName, OzoneMultipartUploadList result = new OzoneMultipartUploadList(uploads, omMultipartUploadList.getNextKeyMarker(), omMultipartUploadList.getNextUploadIdMarker(), - omMultipartUploadList.isTruncated(), - omMultipartUploadList.getUploads().size()); + omMultipartUploadList.isTruncated()); return result; } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 93280911292e..3ebe698a6053 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -352,10 +352,8 @@ public Response listMultipartUploads( ListMultipartUploadsResult result = new ListMultipartUploadsResult(); result.setBucket(bucketName); - result.setKeyMarker(ozoneMultipartUploadList.getNextKeyMarker()); - result.setUploadIdMarker(ozoneMultipartUploadList.getNextUploadIdMarker()); + result.setMaxUploads(maxUploads); result.setTruncated(ozoneMultipartUploadList.isTruncated()); - result.setMaxUploads(ozoneMultipartUploadList.getMaxUploads()); ozoneMultipartUploadList.getUploads().forEach(upload -> result.addUpload( new ListMultipartUploadsResult.Upload( From 88ba8a0904ac07c863ab201193af92c5c87285f0 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 18 Feb 2025 03:34:04 +0800 Subject: [PATCH 22/29] Return listMultiPartUpload req params as response --- .../hadoop/ozone/om/helpers/MultipartUploadKeys.java | 6 +++--- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 5 +++++ .../ozone/s3/endpoint/ListMultipartUploadsResult.java | 11 +++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java index 80c91b139350..dc619ee415cf 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java @@ -60,9 +60,9 @@ public static Builder newBuilder() { */ public static class Builder { private Set keys; - private String nextKeyMarker = ""; - private String nextUploadIdMarker = ""; - private boolean isTruncated = false; + private String nextKeyMarker; + private String nextUploadIdMarker; + private boolean isTruncated; public Builder setKeys(Set keys) { this.keys = keys; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 3ebe698a6053..466293eeacad 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -352,6 +352,11 @@ public Response listMultipartUploads( ListMultipartUploadsResult result = new ListMultipartUploadsResult(); result.setBucket(bucketName); + result.setKeyMarker(keyMarker); + result.setUploadIdMarker(uploadIdMarker); + result.setNextKeyMarker(ozoneMultipartUploadList.getNextKeyMarker()); + result.setPrefix(prefix); + result.setNextUploadIdMarker(ozoneMultipartUploadList.getNextUploadIdMarker()); result.setMaxUploads(maxUploads); result.setTruncated(ozoneMultipartUploadList.isTruncated()); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java index e7052fbf090b..98e9f98608b4 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ListMultipartUploadsResult.java @@ -48,6 +48,9 @@ public class ListMultipartUploadsResult { @XmlElement(name = "NextKeyMarker") private String nextKeyMarker; + @XmlElement(name = "Prefix") + private String prefix; + @XmlElement(name = "NextUploadIdMarker") private String nextUploadIdMarker; @@ -92,6 +95,14 @@ public void setNextKeyMarker(String nextKeyMarker) { this.nextKeyMarker = nextKeyMarker; } + public String getPrefix() { + return prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; + } + public String getNextUploadIdMarker() { return nextUploadIdMarker; } From 024ac94106cf45a42ac4cbe5ca14ad886e954716 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 18 Feb 2025 03:35:22 +0800 Subject: [PATCH 23/29] AbstractS3SDKV1Tests to test the listMultipartUploads using AWS SDK --- ...TestOzoneClientMultipartUploadWithFSO.java | 28 ++--- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 101 ++++++++++++++---- 2 files changed, 97 insertions(+), 32 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java index f78d8cad993d..6fac3155a53b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java @@ -920,22 +920,24 @@ public void testListMultipartUploadsPagination() throws Exception { // Test full pagination process final int maxUploads = 10; - final int expectedTruncated = 2; - int truncatedCount = 0; + final int expectedPages = 3; + int pageCount = 0; String keyMarker = ""; String uploadIdMarker = ""; Set retrievedKeys = new HashSet<>(); - boolean hasMore = true; + boolean isTruncated = true; - while (hasMore) { + do { OzoneMultipartUploadList result = bucket.listMultipartUploads( "dir", keyMarker, uploadIdMarker, maxUploads); - assertThat(result.getUploads()) - .as("Number of uploads should not exceed maxUploads") - .hasSizeLessThanOrEqualTo(maxUploads); - - assertEquals(result.getUploads().size(), result.getMaxUploads()); + if (pageCount < 2) { + assertEquals(maxUploads, result.getUploads().size()); + assertTrue(result.isTruncated()); + } else { + assertEquals(numOfKeys - pageCount * maxUploads, result.getUploads().size()); + assertFalse(result.isTruncated()); + } for (OzoneMultipartUpload upload : result.getUploads()) { String key = upload.getKeyName(); @@ -947,13 +949,13 @@ public void testListMultipartUploadsPagination() throws Exception { // Update markers for next iteration keyMarker = result.getNextKeyMarker(); uploadIdMarker = result.getNextUploadIdMarker(); - hasMore = result.isTruncated(); + isTruncated = result.isTruncated(); - truncatedCount += result.isTruncated() ? 1 : 0; - } + pageCount++; + } while (isTruncated); assertEquals(keys.size(), retrievedKeys.size()); - assertEquals(expectedTruncated, truncatedCount); + assertEquals(expectedPages, pageCount); assertThat(retrievedKeys.stream().sorted().collect(Collectors.toList())) .as("Retrieved keys should match expected keys in order") .isEqualTo(keys.stream().sorted().collect(Collectors.toList())); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 44785bd97f05..107d149e4c16 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -74,6 +74,7 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.utils.InputSubstream; import org.apache.ozone.test.OzoneTestBase; +import org.eclipse.jetty.util.StringUtil; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; @@ -95,10 +96,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.stream.Collectors; import static org.apache.hadoop.ozone.OzoneConsts.MB; @@ -663,33 +666,93 @@ public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws Exception @Test public void testListMultipartUploads() { final String bucketName = getBucketName(); - final String multipartKey1 = getKeyName("multipart1"); - final String multipartKey2 = getKeyName("multipart2"); + final String multipartKeyPrefix = getKeyName("multipart"); s3Client.createBucket(bucketName); - List uploadIds = new ArrayList<>(); + // Create 25 multipart uploads to test pagination + List allKeys = new ArrayList<>(); + Map keyToUploadId = new HashMap<>(); - String uploadId1 = initiateMultipartUpload(bucketName, multipartKey1, null, null, null); - uploadIds.add(uploadId1); - String uploadId2 = initiateMultipartUpload(bucketName, multipartKey1, null, null, null); - uploadIds.add(uploadId2); - // TODO: Currently, Ozone sorts based on uploadId instead of MPU init time within the same key. - // Remove this sorting step once HDDS-11532 has been implemented - Collections.sort(uploadIds); - String uploadId3 = initiateMultipartUpload(bucketName, multipartKey2, null, null, null); - uploadIds.add(uploadId3); + for (int i = 0; i < 25; i++) { + String key = String.format("%s-%03d", multipartKeyPrefix, i); + allKeys.add(key); + String uploadId = initiateMultipartUpload(bucketName, key, null, null, null); + keyToUploadId.put(key, uploadId); + } + Collections.sort(allKeys); + + // Test pagination with maxUploads=10 + Set retrievedKeys = new HashSet<>(); + String keyMarker = null; + String uploadIdMarker = null; + boolean truncated = true; + int pageCount = 0; - // TODO: Add test for max uploads threshold and marker once HDDS-11530 has been implemented - ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName); + do { + ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(bucketName) + .withMaxUploads(10) + .withKeyMarker(keyMarker) + .withUploadIdMarker(uploadIdMarker); - MultipartUploadListing result = s3Client.listMultipartUploads(listMultipartUploadsRequest); + MultipartUploadListing result = s3Client.listMultipartUploads(request); - List listUploadIds = result.getMultipartUploads().stream() - .map(MultipartUpload::getUploadId) - .collect(Collectors.toList()); + // Verify page size + if (pageCount < 2) { + assertEquals(10, result.getMultipartUploads().size()); + assertTrue(result.isTruncated()); + } else { + assertEquals(5, result.getMultipartUploads().size()); + assertFalse(result.isTruncated()); + } + + // Collect keys and verify uploadIds + for (MultipartUpload upload : result.getMultipartUploads()) { + String key = upload.getKey(); + retrievedKeys.add(key); + assertEquals(keyToUploadId.get(key), upload.getUploadId()); + } - assertEquals(uploadIds, listUploadIds); + // Verify response + assertNull(result.getPrefix()); + assertEquals(result.getUploadIdMarker(), uploadIdMarker); + assertEquals(result.getKeyMarker(), keyMarker); + assertEquals(result.getMaxUploads(), 10); + + // Update markers for next page + keyMarker = result.getNextKeyMarker(); + uploadIdMarker = result.getNextUploadIdMarker(); + + truncated = result.isTruncated(); + pageCount++; + + } while (truncated); + + // Verify pagination results + assertEquals(3, pageCount, "Should have exactly 3 pages"); + assertEquals(25, retrievedKeys.size(), "Should retrieve all uploads"); + assertEquals( + allKeys, + retrievedKeys.stream().sorted().collect(Collectors.toList()), + "Retrieved keys should match expected keys in order"); + + // Test with prefix + String prefix = multipartKeyPrefix + "-01"; + ListMultipartUploadsRequest prefixRequest = new ListMultipartUploadsRequest(bucketName) + .withPrefix(prefix); + + MultipartUploadListing prefixResult = s3Client.listMultipartUploads(prefixRequest); + + assertEquals(prefix, prefixResult.getPrefix()); + assertEquals( + Arrays.asList(multipartKeyPrefix + "-010", multipartKeyPrefix + "-011", + multipartKeyPrefix + "-012", multipartKeyPrefix + "-013", + multipartKeyPrefix + "-014", multipartKeyPrefix + "-015", + multipartKeyPrefix + "-016", multipartKeyPrefix + "-017", + multipartKeyPrefix + "-018", multipartKeyPrefix + "-019"), + prefixResult.getMultipartUploads().stream() + .map(MultipartUpload::getKey) + .collect(Collectors.toList())); } @Test From bb7fd8d946e7ad3f3738c6b2872d5e80b12c15a8 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 18 Feb 2025 15:21:27 +0800 Subject: [PATCH 24/29] Add listmultipartUpload robot test and some minor fix --- .../ozone/om/helpers/MultipartUploadKeys.java | 4 +- .../main/smoketest/s3/MultipartUpload.robot | 64 ++++++++++++++++--- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 1 - 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java index dc619ee415cf..d74d9cb0c4c6 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java @@ -60,8 +60,8 @@ public static Builder newBuilder() { */ public static class Builder { private Set keys; - private String nextKeyMarker; - private String nextUploadIdMarker; + private String nextKeyMarker = ""; + private String nextUploadIdMarker = ""; private boolean isTruncated; public Builder setKeys(Set keys) { diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot index c12f8e335817..77d01889ed44 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot @@ -318,12 +318,58 @@ Test Multipart Upload Put With Copy and range with IfModifiedSince Compare files /tmp/10mb /tmp/part-result Test Multipart Upload list - ${uploadID1} = Initiate MPU ${BUCKET} ${PREFIX}/listtest/key1 - ${uploadID2} = Initiate MPU ${BUCKET} ${PREFIX}/listtest/key2 - - ${result} = Execute AWSS3APICli list-multipart-uploads --bucket ${BUCKET} --prefix ${PREFIX}/listtest - Should contain ${result} ${uploadID1} - Should contain ${result} ${uploadID2} - - ${count} = Execute and checkrc echo '${result}' | jq -r '.Uploads | length' 0 - Should Be Equal ${count} 2 + # Create 25 multipart uploads to test pagination + ${uploadIDs}= Create List + FOR ${index} IN RANGE 25 + ${key}= Set Variable ${PREFIX}/listtest/key-${index} + ${uploadID}= Initiate MPU ${BUCKET} ${key} + Append To List ${uploadIDs} ${uploadID} + END + + # Test listing with max-items=10 (should get 3 pages: 10, 10, 5) + ${result}= Execute AWSS3APICli list-multipart-uploads --bucket ${BUCKET} --prefix ${PREFIX}/listtest --max-items 10 + + # Verify first page + ${count}= Execute and checkrc echo '${result}' | jq -r '.Uploads | length' 0 + Should Be Equal ${count} 10 + + ${hasNext}= Execute and checkrc echo '${result}' | jq -r 'has("NextToken")' 0 + Should Be Equal ${hasNext} true + + ${nextToken}= Execute and checkrc echo '${result}' | jq -r '.NextToken' 0 + Should Not Be Empty ${nextToken} + + # Get second page + ${result}= Execute AWSS3APICli list-multipart-uploads --bucket ${BUCKET} --prefix ${PREFIX}/listtest --max-items 10 --starting-token ${nextToken} + + # Verify second page + ${count}= Execute and checkrc echo '${result}' | jq -r '.Uploads | length' 0 + Should Be Equal ${count} 10 + + ${hasNext}= Execute and checkrc echo '${result}' | jq -r 'has("NextToken")' 0 + Should Be Equal ${hasNext} true + + ${nextToken}= Execute and checkrc echo '${result}' | jq -r '.NextToken' 0 + Should Not Be Empty ${nextToken} + + # Get last page + ${result}= Execute AWSS3APICli list-multipart-uploads --bucket ${BUCKET} --prefix ${PREFIX}/listtest --max-items 10 --starting-token ${nextToken} + + # Verify last page + ${count}= Execute and checkrc echo '${result}' | jq -r '.Uploads | length' 0 + Should Be Equal ${count} 5 + + ${hasNext}= Execute and checkrc echo '${result}' | jq -r 'has("NextToken")' 0 + Should Be Equal ${hasNext} false + + # Test prefix filtering + ${result}= Execute AWSS3APICli list-multipart-uploads --bucket ${BUCKET} --prefix ${PREFIX}/listtest/key-1 + ${count}= Execute and checkrc echo '${result}' | jq -r '.Uploads | length' 0 + Should Be Equal ${count} 11 # Should match key-1, key-10 through key-19 + + # Cleanup + FOR ${index} IN RANGE 25 + ${key}= Set Variable ${PREFIX}/listtest/key-${index} + ${uploadID}= Get From List ${uploadIDs} ${index} + Abort MPU ${BUCKET} ${key} ${uploadID} 0 + END diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 107d149e4c16..151d022392a9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -74,7 +74,6 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.utils.InputSubstream; import org.apache.ozone.test.OzoneTestBase; -import org.eclipse.jetty.util.StringUtil; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; From 33b2ee69db235745c44aaaed944767c13f3b2972 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 18 Feb 2025 16:24:06 +0800 Subject: [PATCH 25/29] fix findbug --- .../java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index 979fe7e2ea49..ad62cf8b651c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -369,11 +369,9 @@ public void testListMultipartUploadsWithPagination() throws IOException { createBucket(metadataManager, volumeName, bucketName); // Create 25 multipart uploads to test pagination - List uploadInfos = new ArrayList<>(); for (int i = 0; i < 25; i++) { String key = String.format("key-%03d", i); // pad with zeros for proper sorting OmMultipartInfo info = initMultipartUpload(writeClient, volumeName, bucketName, key); - uploadInfos.add(info); } // WHEN - First page (10 entries) From 8f4a1139987dc2f4a3b96487ccb2b85465ea1a05 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 18 Feb 2025 20:01:55 +0800 Subject: [PATCH 26/29] Add backward compatibility --- .../hadoop/ozone/OzoneManagerVersion.java | 3 ++ .../hadoop/ozone/client/rpc/RpcClient.java | 10 +++- .../om/protocol/OzoneManagerProtocol.java | 5 +- ...ManagerProtocolClientSideTranslatorPB.java | 3 +- .../src/main/proto/OmClientProtocol.proto | 1 + .../hadoop/ozone/om/OMMetadataManager.java | 3 +- .../apache/hadoop/ozone/om/KeyManager.java | 3 +- .../hadoop/ozone/om/KeyManagerImpl.java | 8 +-- .../ozone/om/OmMetadataManagerImpl.java | 9 +++- .../apache/hadoop/ozone/om/OzoneManager.java | 5 +- .../OzoneManagerRequestHandler.java | 6 +-- .../hadoop/ozone/om/TestKeyManagerUnit.java | 53 ++++++++++--------- .../ozone/om/TestOmMetadataManager.java | 16 ++++-- 13 files changed, 79 insertions(+), 46 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index a29f6d36043b..41cf8ab28560 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@ -52,6 +52,9 @@ public enum OzoneManagerVersion implements ComponentVersion { S3_PART_AWARE_GET(10, "OzoneManager version that supports S3 get for a specific multipart " + "upload part number"), + S3_LIST_MULTIPART_UPLOADS_PAGINATION(11, + "OzoneManager version that supports S3 list multipart uploads API with pagination"), + FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index f0ae67786227..e0db30ace1ac 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2126,8 +2126,14 @@ public OzoneMultipartUploadPartListParts listParts(String volumeName, public OzoneMultipartUploadList listMultipartUploads(String volumeName, String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { - OmMultipartUploadList omMultipartUploadList = - ozoneManagerClient.listMultipartUploads(volumeName, bucketName, prefix, keyMarker, uploadIdMarker, maxUploads); + OmMultipartUploadList omMultipartUploadList; + if (omVersion.compareTo(OzoneManagerVersion.S3_LIST_MULTIPART_UPLOADS_PAGINATION) >= 0) { + omMultipartUploadList = ozoneManagerClient.listMultipartUploads(volumeName, bucketName, prefix, keyMarker, + uploadIdMarker, maxUploads, false); + } else { + omMultipartUploadList = ozoneManagerClient.listMultipartUploads(volumeName, bucketName, prefix, keyMarker, + uploadIdMarker, maxUploads, true); + } List uploads = omMultipartUploadList.getUploads() .stream() .map(upload -> new OzoneMultipartUpload(upload.getVolumeName(), diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 4bb53a393ac8..fdc9859b8b1d 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -606,9 +606,12 @@ OmMultipartUploadListParts listParts(String volumeName, String bucketName, /** * List in-flight uploads. + * noPagination is for backward compatible as older listMultipartUploads does + * not support pagination. */ OmMultipartUploadList listMultipartUploads(String volumeName, - String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException; + String bucketName, String prefix, + String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws IOException; /** * Gets s3Secret for given kerberos user. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 3b20f4d9a310..b88516980ae1 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1808,7 +1808,7 @@ public OmMultipartUploadListParts listParts(String volumeName, @Override public OmMultipartUploadList listMultipartUploads(String volumeName, String bucketName, - String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { + String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws IOException { ListMultipartUploadsRequest request = ListMultipartUploadsRequest .newBuilder() .setVolume(volumeName) @@ -1817,6 +1817,7 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, .setKeyMarker(keyMarker == null ? "" : keyMarker) .setUploadIdMarker(uploadIdMarker == null ? "" : uploadIdMarker) .setMaxUploads(maxUploads) + .setNoPagination(noPagination) .build(); OMRequest omRequest = createOMRequest(Type.ListMultipartUploads) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index ec943dd5c30c..6a212512a892 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1762,6 +1762,7 @@ message ListMultipartUploadsRequest { optional string keyMarker = 4; optional string uploadIdMarker = 5; optional int32 maxUploads = 6; + optional bool noPagination = 7; // for backward compatibility } message ListMultipartUploadsResponse { diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index b0164b715c60..3818b4ede56f 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -495,7 +495,8 @@ long countEstimatedRowsInTable(Table table) * keyName. */ MultipartUploadKeys getMultipartUploadKeys(String volumeName, - String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException; + String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads, + boolean noPagination) throws IOException; /** * Gets the DirectoryTable. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 6ed211dd618c..04ca29363fc0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -203,7 +203,8 @@ List getExpiredMultipartUploads( OmMultipartUploadList listMultipartUploads(String volumeName, - String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws OMException; + String bucketName, String prefix, + String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws OMException; /** * Returns list of parts of a multipart upload key. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 4272d9ae0e3f..d5d7f8635ddd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -816,7 +816,8 @@ public boolean isSstFilteringSvcEnabled() { @Override public OmMultipartUploadList listMultipartUploads(String volumeName, - String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws OMException { + String bucketName, + String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws OMException { Preconditions.checkNotNull(volumeName); Preconditions.checkNotNull(bucketName); @@ -824,9 +825,8 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, bucketName); try { - MultipartUploadKeys multipartUploadKeys = - metadataManager - .getMultipartUploadKeys(volumeName, bucketName, prefix, keyMarker, uploadIdMarker, maxUploads); + MultipartUploadKeys multipartUploadKeys = metadataManager + .getMultipartUploadKeys(volumeName, bucketName, prefix, keyMarker, uploadIdMarker, maxUploads, noPagination); List collect = multipartUploadKeys.getKeys().stream() .map(OmMultipartUpload::from) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index cbaae356c935..77a0fa24e9ec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1938,7 +1938,7 @@ public long countEstimatedRowsInTable(Table table) @Override public MultipartUploadKeys getMultipartUploadKeys( String volumeName, String bucketName, String prefix, String keyMarker, - String uploadIdMarker, int maxUploads) throws IOException { + String uploadIdMarker, int maxUploads, boolean noPagination) throws IOException { MultipartUploadKeys.Builder resultBuilder = MultipartUploadKeys.newBuilder(); SortedSet responseKeys = new TreeSet<>(); @@ -1980,7 +1980,7 @@ public MultipartUploadKeys getMultipartUploadKeys( iterator = getMultipartInfoTable().iterator(prefixKey)) { iterator.seek(seekKey); - while (iterator.hasNext() && dbKeysCount < maxUploads + 1) { + while (iterator.hasNext() && (noPagination || dbKeysCount < maxUploads + 1)) { KeyValue entry = iterator.next(); // If it is marked for abort, skip it. if (!aborted.contains(entry.getKey())) { @@ -1990,6 +1990,11 @@ public MultipartUploadKeys getMultipartUploadKeys( } } + if (noPagination) { + resultBuilder.setKeys(responseKeys); + return resultBuilder.build(); + } + String lastKey = responseKeys.stream() .skip(maxUploads) .findFirst() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index df3122d6f3d6..9cd5f7fcd27e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3710,7 +3710,8 @@ public OmMultipartUploadListParts listParts(final String volumeName, @Override public OmMultipartUploadList listMultipartUploads(String volumeName, - String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { + String bucketName, + String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws IOException { ResolvedBucket bucket = resolveBucketLink(Pair.of(volumeName, bucketName)); @@ -3721,7 +3722,7 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, try { OmMultipartUploadList omMultipartUploadList = keyManager.listMultipartUploads(bucket.realVolume(), - bucket.realBucket(), prefix, keyMarker, uploadIdMarker, maxUploads); + bucket.realBucket(), prefix, keyMarker, uploadIdMarker, maxUploads, noPagination); AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction .LIST_MULTIPART_UPLOADS, auditMap)); return omMultipartUploadList; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 4f636af845b7..20726d2c0a8d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -1015,9 +1015,9 @@ private ListMultipartUploadsResponse listMultipartUploads( ListMultipartUploadsRequest request) throws IOException { - OmMultipartUploadList omMultipartUploadList = - impl.listMultipartUploads(request.getVolume(), request.getBucket(), - request.getPrefix(), request.getKeyMarker(), request.getUploadIdMarker(), request.getMaxUploads()); + OmMultipartUploadList omMultipartUploadList = impl.listMultipartUploads(request.getVolume(), request.getBucket(), + request.getPrefix(), + request.getKeyMarker(), request.getUploadIdMarker(), request.getMaxUploads(), request.getNoPagination()); List info = omMultipartUploadList .getUploads() diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index ad62cf8b651c..6756baee62e9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -232,7 +232,7 @@ public void listMultipartUploads() throws IOException { //WHEN OmMultipartUploadList omMultipartUploadList = - keyManager.listMultipartUploads(volume, "bucket1", "", "", "", 10); + keyManager.listMultipartUploads(volume, "bucket1", "", "", "", 10, false); //THEN List uploads = omMultipartUploadList.getUploads(); @@ -253,11 +253,10 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { String volume = volumeName(); String bucket = "bucket"; - //GIVEN + // GIVEN createBucket(metadataManager, volume, bucket); createBucket(metadataManager, volume, bucket); - // Add few to cache and few to DB. addinitMultipartUploadToCache(volume, bucket, "dir/key1"); @@ -267,11 +266,11 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { initMultipartUpload(writeClient, volume, bucket, "dir/key4"); - //WHEN - OmMultipartUploadList omMultipartUploadList = - keyManager.listMultipartUploads(volume, bucket, "", "", "", 10); + // WHEN + OmMultipartUploadList omMultipartUploadList = keyManager.listMultipartUploads(volume, bucket, "", "", "", 10, + false); - //THEN + // THEN List uploads = omMultipartUploadList.getUploads(); assertEquals(4, uploads.size()); assertEquals("dir/key1", uploads.get(0).getKeyName()); @@ -292,10 +291,9 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { OmMultipartInfo omMultipartInfo4 = initMultipartUpload(writeClient, volume, bucket, "dir/ozonekey4"); - omMultipartUploadList = - keyManager.listMultipartUploads(volume, bucket, "dir/ozone", "", "", 10); + omMultipartUploadList = keyManager.listMultipartUploads(volume, bucket, "dir/ozone", "", "", 10, false); - //THEN + // THEN uploads = omMultipartUploadList.getUploads(); assertEquals(4, uploads.size()); assertEquals("dir/ozonekey1", uploads.get(0).getKeyName()); @@ -308,10 +306,9 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { omMultipartInfo4.getUploadID()); // Now list. - omMultipartUploadList = - keyManager.listMultipartUploads(volume, bucket, "dir/ozone", "", "", 10); + omMultipartUploadList = keyManager.listMultipartUploads(volume, bucket, "dir/ozone", "", "", 10, false); - //THEN + // THEN uploads = omMultipartUploadList.getUploads(); assertEquals(3, uploads.size()); assertEquals("dir/ozonekey1", uploads.get(0).getKeyName()); @@ -323,10 +320,9 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { omMultipartInfo3.getUploadID()); // Now list. - omMultipartUploadList = - keyManager.listMultipartUploads(volume, bucket, "dir/ozone", "", "", 10); + omMultipartUploadList = keyManager.listMultipartUploads(volume, bucket, "dir/ozone", "", "", 10, false); - //THEN + // THEN uploads = omMultipartUploadList.getUploads(); assertEquals(2, uploads.size()); assertEquals("dir/ozonekey1", uploads.get(0).getKeyName()); @@ -337,7 +333,7 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { @Test public void listMultipartUploadsWithPrefix() throws IOException { - //GIVEN + // GIVEN final String volumeName = volumeName(); createBucket(metadataManager, volumeName, "bucket1"); createBucket(metadataManager, volumeName, "bucket2"); @@ -350,11 +346,11 @@ public void listMultipartUploadsWithPrefix() throws IOException { initMultipartUpload(writeClient, volumeName, "bucket2", "dir/key1"); - //WHEN - OmMultipartUploadList omMultipartUploadList = - keyManager.listMultipartUploads(volumeName, "bucket1", "dir", "", "", 10); + // WHEN + OmMultipartUploadList omMultipartUploadList = keyManager.listMultipartUploads(volumeName, "bucket1", "dir", "", + "", 10, false); - //THEN + // THEN List uploads = omMultipartUploadList.getUploads(); assertEquals(2, uploads.size()); assertEquals("dir/key1", uploads.get(0).getKeyName()); @@ -371,12 +367,12 @@ public void testListMultipartUploadsWithPagination() throws IOException { // Create 25 multipart uploads to test pagination for (int i = 0; i < 25; i++) { String key = String.format("key-%03d", i); // pad with zeros for proper sorting - OmMultipartInfo info = initMultipartUpload(writeClient, volumeName, bucketName, key); + initMultipartUpload(writeClient, volumeName, bucketName, key); } // WHEN - First page (10 entries) OmMultipartUploadList firstPage = keyManager.listMultipartUploads( - volumeName, bucketName, "", "", "", 10); + volumeName, bucketName, "", "", "", 10, false); // THEN assertEquals(10, firstPage.getUploads().size()); @@ -395,7 +391,7 @@ public void testListMultipartUploadsWithPagination() throws IOException { volumeName, bucketName, "", firstPage.getNextKeyMarker(), firstPage.getNextUploadIdMarker(), - 10); + 10, false); // THEN assertEquals(10, secondPage.getUploads().size()); @@ -412,7 +408,7 @@ public void testListMultipartUploadsWithPagination() throws IOException { volumeName, bucketName, "", secondPage.getNextKeyMarker(), secondPage.getNextUploadIdMarker(), - 10); + 10, false); // THEN assertEquals(5, lastPage.getUploads().size()); @@ -425,6 +421,13 @@ public void testListMultipartUploadsWithPagination() throws IOException { assertEquals(String.format("key-%03d", i + 20), lastPage.getUploads().get(i).getKeyName()); } + + // Test with no pagination + OmMultipartUploadList noPagination = keyManager.listMultipartUploads( + volumeName, bucketName, "", "", "", 10, true); + + assertEquals(25, noPagination.getUploads().size()); + assertFalse(noPagination.isTruncated()); } private void createBucket(OMMetadataManager omMetadataManager, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 5fd6aa286b2e..47305fddfd9a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -1153,7 +1153,7 @@ public void testGetMultipartUploadKeys() throws Exception { // List first page without markers MultipartUploadKeys result = omMetadataManager.getMultipartUploadKeys( - volumeName, bucketName, prefix, null, null, maxUploads); + volumeName, bucketName, prefix, null, null, maxUploads, false); assertEquals(maxUploads, result.getKeys().size()); assertTrue(result.isTruncated()); @@ -1165,21 +1165,21 @@ public void testGetMultipartUploadKeys() throws Exception { volumeName, bucketName, prefix, result.getNextKeyMarker(), result.getNextUploadIdMarker(), - maxUploads); + maxUploads, false); assertEquals(maxUploads, nextPage.getKeys().size()); assertTrue(nextPage.isTruncated()); // List with different prefix MultipartUploadKeys differentPrefix = omMetadataManager.getMultipartUploadKeys( - volumeName, bucketName, "different/", null, null, maxUploads); + volumeName, bucketName, "different/", null, null, maxUploads, false); assertEquals(0, differentPrefix.getKeys().size()); assertFalse(differentPrefix.isTruncated()); // List all entries with large maxUploads MultipartUploadKeys allEntries = omMetadataManager.getMultipartUploadKeys( - volumeName, bucketName, prefix, null, null, 100); + volumeName, bucketName, prefix, null, null, 100, false); assertEquals(25, allEntries.getKeys().size()); assertFalse(allEntries.isTruncated()); @@ -1192,5 +1192,13 @@ public void testGetMultipartUploadKeys() throws Exception { } Collections.sort(actualKeys); assertEquals(expectedKeys, actualKeys); + + // Test with no pagination + MultipartUploadKeys noPagination = omMetadataManager.getMultipartUploadKeys( + volumeName, bucketName, prefix, null, null, 10, true); + + assertEquals(25, noPagination.getKeys().size()); + assertFalse(noPagination.isTruncated()); + } } From 1420d92a00f8377b94f13b870b1a7834d11eb9a6 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 19 Feb 2025 14:22:25 +0800 Subject: [PATCH 27/29] Rename `noPagination` to `withPagination` since the old S3G will not set this that field and the default protobuf value will be false. --- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 4 ++-- .../ozone/om/protocol/OzoneManagerProtocol.java | 4 ++-- ...OzoneManagerProtocolClientSideTranslatorPB.java | 5 +++-- .../src/main/proto/OmClientProtocol.proto | 2 +- .../org/apache/hadoop/ozone/om/KeyManager.java | 2 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 6 ++++-- .../org/apache/hadoop/ozone/om/OzoneManager.java | 14 ++++++-------- .../protocolPB/OzoneManagerRequestHandler.java | 2 +- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 2 +- 9 files changed, 21 insertions(+), 20 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index e0db30ace1ac..90afe10fe6bc 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2129,10 +2129,10 @@ public OzoneMultipartUploadList listMultipartUploads(String volumeName, OmMultipartUploadList omMultipartUploadList; if (omVersion.compareTo(OzoneManagerVersion.S3_LIST_MULTIPART_UPLOADS_PAGINATION) >= 0) { omMultipartUploadList = ozoneManagerClient.listMultipartUploads(volumeName, bucketName, prefix, keyMarker, - uploadIdMarker, maxUploads, false); + uploadIdMarker, maxUploads, true); } else { omMultipartUploadList = ozoneManagerClient.listMultipartUploads(volumeName, bucketName, prefix, keyMarker, - uploadIdMarker, maxUploads, true); + uploadIdMarker, maxUploads, false); } List uploads = omMultipartUploadList.getUploads() .stream() diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index fdc9859b8b1d..2bbff617ca7f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -606,12 +606,12 @@ OmMultipartUploadListParts listParts(String volumeName, String bucketName, /** * List in-flight uploads. - * noPagination is for backward compatible as older listMultipartUploads does + * withPagination is for backward compatible as older listMultipartUploads does * not support pagination. */ OmMultipartUploadList listMultipartUploads(String volumeName, String bucketName, String prefix, - String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws IOException; + String keyMarker, String uploadIdMarker, int maxUploads, boolean withPagination) throws IOException; /** * Gets s3Secret for given kerberos user. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index b88516980ae1..41bfdcdea434 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1808,7 +1808,8 @@ public OmMultipartUploadListParts listParts(String volumeName, @Override public OmMultipartUploadList listMultipartUploads(String volumeName, String bucketName, - String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws IOException { + String prefix, + String keyMarker, String uploadIdMarker, int maxUploads, boolean withPagination) throws IOException { ListMultipartUploadsRequest request = ListMultipartUploadsRequest .newBuilder() .setVolume(volumeName) @@ -1817,7 +1818,7 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, .setKeyMarker(keyMarker == null ? "" : keyMarker) .setUploadIdMarker(uploadIdMarker == null ? "" : uploadIdMarker) .setMaxUploads(maxUploads) - .setNoPagination(noPagination) + .setWithPagination(withPagination) .build(); OMRequest omRequest = createOMRequest(Type.ListMultipartUploads) diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 6a212512a892..df97028a0f31 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1762,7 +1762,7 @@ message ListMultipartUploadsRequest { optional string keyMarker = 4; optional string uploadIdMarker = 5; optional int32 maxUploads = 6; - optional bool noPagination = 7; // for backward compatibility + optional bool withPagination = 7; // for backward compatibility } message ListMultipartUploadsResponse { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java index 04ca29363fc0..d25535b151d4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java @@ -204,7 +204,7 @@ List getExpiredMultipartUploads( OmMultipartUploadList listMultipartUploads(String volumeName, String bucketName, String prefix, - String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws OMException; + String keyMarker, String uploadIdMarker, int maxUploads, boolean withPagination) throws OMException; /** * Returns list of parts of a multipart upload key. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index d5d7f8635ddd..8aa9b579faa2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -817,7 +817,8 @@ public boolean isSstFilteringSvcEnabled() { @Override public OmMultipartUploadList listMultipartUploads(String volumeName, String bucketName, - String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws OMException { + String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean withPagination) + throws OMException { Preconditions.checkNotNull(volumeName); Preconditions.checkNotNull(bucketName); @@ -826,7 +827,8 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, try { MultipartUploadKeys multipartUploadKeys = metadataManager - .getMultipartUploadKeys(volumeName, bucketName, prefix, keyMarker, uploadIdMarker, maxUploads, noPagination); + .getMultipartUploadKeys(volumeName, bucketName, prefix, keyMarker, uploadIdMarker, maxUploads, + !withPagination); List collect = multipartUploadKeys.getKeys().stream() .map(OmMultipartUpload::from) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 9cd5f7fcd27e..a72ce02d1ddb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3711,7 +3711,8 @@ public OmMultipartUploadListParts listParts(final String volumeName, @Override public OmMultipartUploadList listMultipartUploads(String volumeName, String bucketName, - String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws IOException { + String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean withPagination) + throws IOException { ResolvedBucket bucket = resolveBucketLink(Pair.of(volumeName, bucketName)); @@ -3720,17 +3721,14 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, metrics.incNumListMultipartUploads(); try { - OmMultipartUploadList omMultipartUploadList = - keyManager.listMultipartUploads(bucket.realVolume(), - bucket.realBucket(), prefix, keyMarker, uploadIdMarker, maxUploads, noPagination); - AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction - .LIST_MULTIPART_UPLOADS, auditMap)); + OmMultipartUploadList omMultipartUploadList = keyManager.listMultipartUploads(bucket.realVolume(), + bucket.realBucket(), prefix, keyMarker, uploadIdMarker, maxUploads, withPagination); + AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_MULTIPART_UPLOADS, auditMap)); return omMultipartUploadList; } catch (IOException ex) { metrics.incNumListMultipartUploadFails(); - AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction - .LIST_MULTIPART_UPLOADS, auditMap, ex)); + AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_MULTIPART_UPLOADS, auditMap, ex)); throw ex; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 20726d2c0a8d..422da953a4cf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -1017,7 +1017,7 @@ private ListMultipartUploadsResponse listMultipartUploads( OmMultipartUploadList omMultipartUploadList = impl.listMultipartUploads(request.getVolume(), request.getBucket(), request.getPrefix(), - request.getKeyMarker(), request.getUploadIdMarker(), request.getMaxUploads(), request.getNoPagination()); + request.getKeyMarker(), request.getUploadIdMarker(), request.getMaxUploads(), request.getWithPagination()); List info = omMultipartUploadList .getUploads() diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index 6756baee62e9..dabd3dd4165c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -424,7 +424,7 @@ public void testListMultipartUploadsWithPagination() throws IOException { // Test with no pagination OmMultipartUploadList noPagination = keyManager.listMultipartUploads( - volumeName, bucketName, "", "", "", 10, true); + volumeName, bucketName, "", "", "", 10, false); assertEquals(25, noPagination.getUploads().size()); assertFalse(noPagination.isTruncated()); From d72d0f8a076d282fed4942d6971e93e6266bc2e4 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 19 Feb 2025 14:22:50 +0800 Subject: [PATCH 28/29] Seperate the base case and pagination for ListMultipartUpload s3 sdk test --- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 81ea03d536a6..f088e2063812 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -663,6 +663,38 @@ public void testLowLevelMultipartUpload(@TempDir Path tempDir) throws Exception @Test public void testListMultipartUploads() { + final String bucketName = getBucketName(); + final String multipartKey1 = getKeyName("multipart1"); + final String multipartKey2 = getKeyName("multipart2"); + + s3Client.createBucket(bucketName); + + List uploadIds = new ArrayList<>(); + + String uploadId1 = initiateMultipartUpload(bucketName, multipartKey1, null, null, null); + uploadIds.add(uploadId1); + String uploadId2 = initiateMultipartUpload(bucketName, multipartKey1, null, null, null); + uploadIds.add(uploadId2); + // TODO: Currently, Ozone sorts based on uploadId instead of MPU init time within the same key. + // Remove this sorting step once HDDS-11532 has been implemented + Collections.sort(uploadIds); + String uploadId3 = initiateMultipartUpload(bucketName, multipartKey2, null, null, null); + uploadIds.add(uploadId3); + + // TODO: Add test for max uploads threshold and marker once HDDS-11530 has been implemented + ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName); + + MultipartUploadListing result = s3Client.listMultipartUploads(listMultipartUploadsRequest); + + List listUploadIds = result.getMultipartUploads().stream() + .map(MultipartUpload::getUploadId) + .collect(Collectors.toList()); + + assertEquals(uploadIds, listUploadIds); + } + + @Test + public void testListMultipartUploadsPagination() { final String bucketName = getBucketName(); final String multipartKeyPrefix = getKeyName("multipart"); From 62f06b31f34906903d6020081ec83d2730c186ee Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 19 Feb 2025 15:03:24 +0800 Subject: [PATCH 29/29] Fix TestKeyManagerUnit#testListMultipartUploadsWithPagination --- .../java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index dabd3dd4165c..b4c886147690 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -372,7 +372,7 @@ public void testListMultipartUploadsWithPagination() throws IOException { // WHEN - First page (10 entries) OmMultipartUploadList firstPage = keyManager.listMultipartUploads( - volumeName, bucketName, "", "", "", 10, false); + volumeName, bucketName, "", "", "", 10, true); // THEN assertEquals(10, firstPage.getUploads().size()); @@ -391,7 +391,7 @@ public void testListMultipartUploadsWithPagination() throws IOException { volumeName, bucketName, "", firstPage.getNextKeyMarker(), firstPage.getNextUploadIdMarker(), - 10, false); + 10, true); // THEN assertEquals(10, secondPage.getUploads().size()); @@ -408,7 +408,7 @@ public void testListMultipartUploadsWithPagination() throws IOException { volumeName, bucketName, "", secondPage.getNextKeyMarker(), secondPage.getNextUploadIdMarker(), - 10, false); + 10, true); // THEN assertEquals(5, lastPage.getUploads().size());