diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java deleted file mode 100644 index d74d9cb0c4c6..000000000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MultipartUploadKeys.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om.helpers; - -import java.util.Set; - -/** - * This class is used to store the result of OmMetadataManager#getMultipartUploadKeys. - */ -public class MultipartUploadKeys { - private final Set keys; - private final String nextKeyMarker; - private final String nextUploadIdMarker; - private final boolean isTruncated; - - public MultipartUploadKeys(Set keys, String nextKeyMarker, String nextUploadIdMarker, boolean isTruncated) { - this.keys = keys; - this.nextKeyMarker = nextKeyMarker; - this.nextUploadIdMarker = nextUploadIdMarker; - this.isTruncated = isTruncated; - } - - public Set getKeys() { - return keys; - } - - public String getNextKeyMarker() { - return nextKeyMarker; - } - - public String getNextUploadIdMarker() { - return nextUploadIdMarker; - } - - public boolean isTruncated() { - return isTruncated; - } - - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Builder class for MultipartUploadKeys. - */ - public static class Builder { - private Set keys; - private String nextKeyMarker = ""; - private String nextUploadIdMarker = ""; - private boolean isTruncated; - - public Builder setKeys(Set keys) { - this.keys = keys; - return this; - } - - public Builder setNextKeyMarker(String nextKeyMarker) { - this.nextKeyMarker = nextKeyMarker; - return this; - } - - public Builder setNextUploadIdMarker(String nextUploadIdMarker) { - this.nextUploadIdMarker = nextUploadIdMarker; - return this; - } - - public Builder setIsTruncated(boolean isTruncated) { - this.isTruncated = isTruncated; - return this; - } - - public MultipartUploadKeys build() { - return new MultipartUploadKeys(keys, nextKeyMarker, nextUploadIdMarker, isTruncated); - } - - } -} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadList.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadList.java index 022196758be0..54d4baeaef65 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadList.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadList.java @@ -22,22 +22,22 @@ /** * List of in-flight MPU uploads. */ -public class OmMultipartUploadList { +public final class OmMultipartUploadList { private List uploads; private String nextKeyMarker; private String nextUploadIdMarker; private boolean isTruncated; - public OmMultipartUploadList( - List uploads, - String nextKeyMarker, - String nextUploadIdMarker, - boolean isTruncated) { - this.uploads = uploads; - this.nextKeyMarker = nextKeyMarker; - this.nextUploadIdMarker = nextUploadIdMarker; - this.isTruncated = isTruncated; + private OmMultipartUploadList(Builder builder) { + this.uploads = builder.uploads; + this.nextKeyMarker = builder.nextKeyMarker; + this.nextUploadIdMarker = builder.nextUploadIdMarker; + this.isTruncated = builder.isTruncated; + } + + public static Builder newBuilder() { + return new Builder(); } public List getUploads() { @@ -61,4 +61,40 @@ public boolean isTruncated() { return isTruncated; } + /** + * Builder class for OmMultipartUploadList. + */ + public static class Builder { + private List uploads; + private String nextKeyMarker = ""; + private String nextUploadIdMarker = ""; + private boolean isTruncated; + + public Builder() { + } + + public Builder setUploads(List uploads) { + this.uploads = uploads; + return this; + } + + public Builder setNextKeyMarker(String nextKeyMarker) { + this.nextKeyMarker = nextKeyMarker; + return this; + } + + public Builder setNextUploadIdMarker(String nextUploadIdMarker) { + this.nextUploadIdMarker = nextUploadIdMarker; + return this; + } + + public Builder setIsTruncated(boolean isTruncated) { + this.isTruncated = isTruncated; + return this; + } + + public OmMultipartUploadList build() { + return new OmMultipartUploadList(this); + } + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 41bfdcdea434..b10781b52df3 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1842,10 +1842,12 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, )) .collect(Collectors.toList()); - OmMultipartUploadList response = new OmMultipartUploadList(uploadList, - listMultipartUploadsResponse.getNextKeyMarker(), - listMultipartUploadsResponse.getNextUploadIdMarker(), - listMultipartUploadsResponse.getIsTruncated()); + OmMultipartUploadList response = OmMultipartUploadList.newBuilder() + .setUploads(uploadList) + .setNextKeyMarker(listMultipartUploadsResponse.getNextKeyMarker()) + .setNextUploadIdMarker(listMultipartUploadsResponse.getNextUploadIdMarker()) + .setIsTruncated(listMultipartUploadsResponse.getIsTruncated()) + .build(); return response; } diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index fa7ad5a19d02..da757e7fd5ba 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -38,7 +38,6 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult; -import org.apache.hadoop.ozone.om.helpers.MultipartUploadKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo; import org.apache.hadoop.ozone.om.helpers.OmDBTenantState; @@ -46,6 +45,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload; import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; @@ -494,8 +494,11 @@ long countEstimatedRowsInTable(Table table) /** * Return the existing upload keys which includes volumeName, bucketName, * keyName. + * @param noPagination if true, returns all keys; if false, applies pagination + * @return When paginated, returns up to maxUploads + 1 entries, where the + * extra entry is used to determine the next page markers */ - MultipartUploadKeys getMultipartUploadKeys(String volumeName, + List getMultipartUploadKeys(String volumeName, String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws IOException; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index ed4adaa586a4..8a735ec20e31 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -79,7 +79,6 @@ import java.security.GeneralSecurityException; import java.security.PrivilegedExceptionAction; import java.time.Duration; -import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -132,7 +131,6 @@ import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.ListKeysResult; -import org.apache.hadoop.ozone.om.helpers.MultipartUploadKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -826,38 +824,25 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, bucketName); try { - - MultipartUploadKeys multipartUploadKeys = metadataManager + List multipartUploadKeys = metadataManager .getMultipartUploadKeys(volumeName, bucketName, prefix, keyMarker, uploadIdMarker, maxUploads, !withPagination); + OmMultipartUploadList.Builder resultBuilder = OmMultipartUploadList.newBuilder(); - List collect = multipartUploadKeys.getKeys().stream() - .map(OmMultipartUpload::from) - .peek(upload -> { - try { - Table keyInfoTable = - metadataManager.getMultipartInfoTable(); - - OmMultipartKeyInfo multipartKeyInfo = - keyInfoTable.get(upload.getDbKey()); - - upload.setCreationTime( - Instant.ofEpochMilli(multipartKeyInfo.getCreationTime())); - upload.setReplicationConfig( - multipartKeyInfo.getReplicationConfig()); - } catch (IOException e) { - LOG.warn( - "Open key entry for multipart upload record can't be read {}", - metadataManager.getOzoneKey(upload.getVolumeName(), - upload.getBucketName(), upload.getKeyName())); - } - }) - .collect(Collectors.toList()); + if (withPagination && multipartUploadKeys.size() == maxUploads + 1) { + int lastIndex = multipartUploadKeys.size() - 1; + OmMultipartUpload lastUpload = multipartUploadKeys.get(lastIndex); + resultBuilder.setNextKeyMarker(lastUpload.getKeyName()) + .setNextUploadIdMarker(lastUpload.getUploadId()) + .setIsTruncated(true); + + // remove next upload from the list + multipartUploadKeys.remove(lastIndex); + } - return new OmMultipartUploadList(collect, - multipartUploadKeys.getNextKeyMarker(), - multipartUploadKeys.getNextUploadIdMarker(), - multipartUploadKeys.isTruncated()); + return resultBuilder + .setUploads(multipartUploadKeys) + .build(); } catch (IOException ex) { LOG.error("List Multipart Uploads Failed: volume: " + volumeName + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 77a0fa24e9ec..79e688f03363 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -45,6 +45,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -56,9 +57,8 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.SortedSet; +import java.util.SortedMap; import java.util.TreeMap; -import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -89,7 +89,6 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult; -import org.apache.hadoop.ozone.om.helpers.MultipartUploadKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo; import org.apache.hadoop.ozone.om.helpers.OmDBTenantState; @@ -1936,12 +1935,11 @@ public long countEstimatedRowsInTable(Table table) } @Override - public MultipartUploadKeys getMultipartUploadKeys( + public List getMultipartUploadKeys( String volumeName, String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads, boolean noPagination) throws IOException { - MultipartUploadKeys.Builder resultBuilder = MultipartUploadKeys.newBuilder(); - SortedSet responseKeys = new TreeSet<>(); + SortedMap responseKeys = new TreeMap<>(); Set aborted = new HashSet<>(); String prefixKey = @@ -1964,9 +1962,10 @@ public MultipartUploadKeys getMultipartUploadKeys( String cacheKey = cacheEntry.getKey().getCacheKey(); if (cacheKey.startsWith(prefixKey)) { // Check if it is marked for delete, due to abort mpu - if (cacheEntry.getValue().getCacheValue() != null && + OmMultipartKeyInfo multipartKeyInfo = cacheEntry.getValue().getCacheValue(); + if (multipartKeyInfo != null && cacheKey.compareTo(seekKey) >= 0) { - responseKeys.add(cacheKey); + responseKeys.put(cacheKey, multipartKeyInfo); } else { aborted.add(cacheKey); } @@ -1984,35 +1983,26 @@ public MultipartUploadKeys getMultipartUploadKeys( KeyValue entry = iterator.next(); // If it is marked for abort, skip it. if (!aborted.contains(entry.getKey())) { - responseKeys.add(entry.getKey()); + responseKeys.put(entry.getKey(), entry.getValue()); dbKeysCount++; } } } - if (noPagination) { - resultBuilder.setKeys(responseKeys); - return resultBuilder.build(); - } + List result = new ArrayList<>(responseKeys.size()); + + for (Map.Entry entry : responseKeys.entrySet()) { + OmMultipartUpload multipartUpload = OmMultipartUpload.from(entry.getKey()); + + multipartUpload.setCreationTime( + Instant.ofEpochMilli(entry.getValue().getCreationTime())); + multipartUpload.setReplicationConfig( + entry.getValue().getReplicationConfig()); - String lastKey = responseKeys.stream() - .skip(maxUploads) - .findFirst() - .orElse(null); - if (lastKey != null) { - // implies the keyset size is greater than maxUploads - OmMultipartUpload lastKeyMultipartUpload = OmMultipartUpload.from(lastKey); - resultBuilder.setNextKeyMarker(lastKeyMultipartUpload.getKeyName()) - .setNextUploadIdMarker(lastKeyMultipartUpload.getUploadId()) - .setIsTruncated(true); - - // keep the [0, maxUploads] keys - responseKeys = responseKeys.subSet(responseKeys.first(), lastKey); + result.add(multipartUpload); } - return resultBuilder - .setKeys(responseKeys) - .build(); + return noPagination || result.size() <= maxUploads ? result : result.subList(0, maxUploads + 1); } @Override diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 47305fddfd9a..e40acfe905e8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -28,9 +28,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -62,7 +60,6 @@ import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult; -import org.apache.hadoop.ozone.om.helpers.MultipartUploadKeys; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; @@ -1152,53 +1149,44 @@ public void testGetMultipartUploadKeys() throws Exception { Collections.sort(expectedKeys); // List first page without markers - MultipartUploadKeys result = omMetadataManager.getMultipartUploadKeys( + List result = omMetadataManager.getMultipartUploadKeys( volumeName, bucketName, prefix, null, null, maxUploads, false); - assertEquals(maxUploads, result.getKeys().size()); - assertTrue(result.isTruncated()); - assertNotNull(result.getNextKeyMarker()); - assertNotNull(result.getNextUploadIdMarker()); + assertEquals(maxUploads + 1, result.size()); // List next page using markers from first page - MultipartUploadKeys nextPage = omMetadataManager.getMultipartUploadKeys( + List nextPage = omMetadataManager.getMultipartUploadKeys( volumeName, bucketName, prefix, - result.getNextKeyMarker(), - result.getNextUploadIdMarker(), + result.get(result.size() - 1).getKeyName(), + result.get(result.size() - 1).getUploadId(), maxUploads, false); - assertEquals(maxUploads, nextPage.getKeys().size()); - assertTrue(nextPage.isTruncated()); + assertEquals(maxUploads + 1, nextPage.size()); // List with different prefix - MultipartUploadKeys differentPrefix = omMetadataManager.getMultipartUploadKeys( + List differentPrefix = omMetadataManager.getMultipartUploadKeys( volumeName, bucketName, "different/", null, null, maxUploads, false); - assertEquals(0, differentPrefix.getKeys().size()); - assertFalse(differentPrefix.isTruncated()); + assertEquals(0, differentPrefix.size()); // List all entries with large maxUploads - MultipartUploadKeys allEntries = omMetadataManager.getMultipartUploadKeys( + List allEntries = omMetadataManager.getMultipartUploadKeys( volumeName, bucketName, prefix, null, null, 100, false); - assertEquals(25, allEntries.getKeys().size()); - assertFalse(allEntries.isTruncated()); + assertEquals(25, allEntries.size()); // Verify all keys are present List actualKeys = new ArrayList<>(); - for (String dbKey : allEntries.getKeys()) { - OmMultipartUpload mpu = OmMultipartUpload.from(dbKey); + for (OmMultipartUpload mpu : allEntries) { actualKeys.add(mpu.getKeyName()); } Collections.sort(actualKeys); assertEquals(expectedKeys, actualKeys); // Test with no pagination - MultipartUploadKeys noPagination = omMetadataManager.getMultipartUploadKeys( + List noPagination = omMetadataManager.getMultipartUploadKeys( volumeName, bucketName, prefix, null, null, 10, true); - assertEquals(25, noPagination.getKeys().size()); - assertFalse(noPagination.isTruncated()); - + assertEquals(25, noPagination.size()); } }