From 43d3fdeaf31235f6197e44933e437452bd023d26 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 10:56:38 +0800 Subject: [PATCH 01/18] Initial implementation --- .../hadoop/ozone/client/OzoneBucket.java | 11 +++- .../ozone/client/protocol/ClientProtocol.java | 25 ++++++-- .../hadoop/ozone/client/rpc/RpcClient.java | 12 ++++ .../S3InitiateMultipartUploadRequest.java | 2 + ...InitiateMultipartUploadRequestWithFSO.java | 2 + .../S3MultipartUploadCompleteRequest.java | 1 + .../ozone/s3/endpoint/ObjectEndpoint.java | 26 +++++--- .../hadoop/ozone/client/OzoneBucketStub.java | 63 +++++++++++++++---- 8 files changed, 116 insertions(+), 26 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 112c76f8c0a8..021a36e94a66 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -679,7 +679,16 @@ public OmMultipartInfo initiateMultipartUpload(String keyName, public OmMultipartInfo initiateMultipartUpload(String keyName, ReplicationConfig config) throws IOException { - return proxy.initiateMultipartUpload(volumeName, name, keyName, config); + return initiateMultipartUpload(keyName, config, Collections.emptyMap()); + } + + /** + * Initiate multipart upload for a specified key. + */ + public OmMultipartInfo initiateMultipartUpload(String keyName, + ReplicationConfig config, Map metadata) + throws IOException { + return proxy.initiateMultipartUpload(volumeName, name, keyName, config, metadata); } /** diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 492cd31b6722..912a3138c478 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -327,7 +327,7 @@ List listBuckets(String volumeName, String bucketPrefix, * @param bucketName Name of the Bucket * @param keyName Name of the Key * @param size Size of the data - * @param metadata custom key value metadata + * @param metadata Custom key value metadata * @return {@link OzoneOutputStream} * */ @@ -509,10 +509,10 @@ OmMultipartInfo initiateMultipartUpload(String volumeName, String /** * Initiate Multipart upload. - * @param volumeName - * @param bucketName - * @param keyName - * @param replicationConfig + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param replicationConfig Replication Config * @return {@link OmMultipartInfo} * @throws IOException */ @@ -520,6 +520,21 @@ OmMultipartInfo initiateMultipartUpload(String volumeName, String bucketName, String keyName, ReplicationConfig replicationConfig) throws IOException; + /** + * Initiate Multipart upload. + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param replicationConfig Replication config + * @param metadata Custom key value metadata + * @return {@link OmMultipartInfo} + * @throws IOException + */ + OmMultipartInfo initiateMultipartUpload(String volumeName, String + bucketName, String keyName, ReplicationConfig replicationConfig, + Map metadata) + throws IOException; + /** * Create a part key for a multipart upload key. * @param volumeName diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index c990d257352b..ae5c67eced6f 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1809,6 +1809,17 @@ public OmMultipartInfo initiateMultipartUpload(String volumeName, String keyName, ReplicationConfig replicationConfig) throws IOException { + return initiateMultipartUpload(volumeName, bucketName, keyName, replicationConfig, + Collections.emptyMap()); + } + + @Override + public OmMultipartInfo initiateMultipartUpload(String volumeName, + String bucketName, + String keyName, + ReplicationConfig replicationConfig, + Map metadata) + throws IOException { verifyVolumeName(volumeName); verifyBucketName(bucketName); HddsClientUtils.checkNotNull(keyName); @@ -1827,6 +1838,7 @@ public OmMultipartInfo initiateMultipartUpload(String volumeName, .setKeyName(keyName) .setReplicationConfig(replicationConfig) .setAcls(getAclList()) + .addAllMetadataGdpr(metadata) .build(); OmMultipartInfo multipartInfo = ozoneManagerClient .initiateMultipartUpload(keyArgs); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index e1772d4009c5..914a707deb0b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -20,6 +20,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.om.helpers.KeyValueUtil; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -211,6 +212,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn .setUpdateID(transactionLogIndex) .setFileEncryptionInfo(keyArgs.hasFileEncryptionInfo() ? OMPBHelper.convert(keyArgs.getFileEncryptionInfo()) : null) + .addAllMetadata(KeyValueUtil.getFromProtobuf(keyArgs.getMetadataList())) .build(); // Add to cache diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java index d1c865fbc7fe..f2423736a396 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequestWithFSO.java @@ -20,6 +20,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.om.helpers.KeyValueUtil; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneConfigUtil; @@ -187,6 +188,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn .setFileEncryptionInfo(keyArgs.hasFileEncryptionInfo() ? OMPBHelper.convert(keyArgs.getFileEncryptionInfo()) : null) .setParentObjectID(pathInfoFSO.getLastKnownParentId()) + .addAllMetadata(KeyValueUtil.getFromProtobuf(keyArgs.getMetadataList())) .build(); // validate and update namespace for missing parent directory diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 83b46de7fd1b..0394a24e9adc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -417,6 +417,7 @@ protected OmKeyInfo getOmKeyInfo(OzoneManager ozoneManager, long trxnLogIndex, .setOmKeyLocationInfos( Collections.singletonList(keyLocationInfoGroup)) .setAcls(dbOpenKeyInfo.getAcls()) + .addAllMetadata(dbOpenKeyInfo.getMetadata()) .addMetadata(OzoneConsts.ETAG, multipartUploadedKeyHash(partKeyInfoMap)); // Check if db entry has ObjectID. This check is required because diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 8b7db9f061d5..34538d86c958 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -289,14 +289,7 @@ public Response put( // Normal put object Map customMetadata = getCustomMetadataFromHeaders(headers.getRequestHeaders()); - if (customMetadata.containsKey(ETAG) - || customMetadata.containsKey(ETAG.toLowerCase())) { - String customETag = customMetadata.get(ETAG) != null ? - customMetadata.get(ETAG) : customMetadata.get(ETAG.toLowerCase()); - customMetadata.remove(ETAG); - customMetadata.remove(ETAG.toLowerCase()); - customMetadata.put(ETAG_CUSTOM, customETag); - } + replaceCustomETagMetadata(customMetadata); if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD" .equals(headers.getHeaderString("x-amz-content-sha256"))) { @@ -754,11 +747,15 @@ public Response initializeMultipartUpload( OzoneBucket ozoneBucket = getBucket(bucket); String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER); + Map customMetadata = + getCustomMetadataFromHeaders(headers.getRequestHeaders()); + replaceCustomETagMetadata(customMetadata); + ReplicationConfig replicationConfig = getReplicationConfig(ozoneBucket, storageType); OmMultipartInfo multipartInfo = - ozoneBucket.initiateMultipartUpload(key, replicationConfig); + ozoneBucket.initiateMultipartUpload(key, replicationConfig, customMetadata); MultipartUploadInitiateResponse multipartUploadInitiateResponse = new MultipartUploadInitiateResponse(); @@ -1326,6 +1323,17 @@ public static boolean checkCopySourceModificationTime( (lastModificationTime <= copySourceIfUnmodifiedSince); } + private void replaceCustomETagMetadata(Map customMetadata) { + if (customMetadata.containsKey(ETAG) + || customMetadata.containsKey(ETAG.toLowerCase())) { + String customETag = customMetadata.get(ETAG) != null ? + customMetadata.get(ETAG) : customMetadata.get(ETAG.toLowerCase()); + customMetadata.remove(ETAG); + customMetadata.remove(ETAG.toLowerCase()); + customMetadata.put(ETAG_CUSTOM, customETag); + } + } + @VisibleForTesting public void setOzoneConfiguration(OzoneConfiguration config) { this.ozoneConfiguration = config; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index d272360fc3cd..41afd529369c 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -26,6 +26,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -72,7 +73,7 @@ public final class OzoneBucketStub extends OzoneBucket { private Map keyContents = new HashMap<>(); - private Map multipartUploadIdMap = new HashMap<>(); + private Map keyToMultipartUpload = new HashMap<>(); private Map> partList = new HashMap<>(); @@ -210,7 +211,7 @@ public OzoneDataStreamOutput createMultipartStreamKey(String key, int partNumber, String uploadID) throws IOException { - String multipartUploadID = multipartUploadIdMap.get(key); + String multipartUploadID = keyToMultipartUpload.get(key).getUploadId(); if (multipartUploadID == null || !multipartUploadID.equals(uploadID)) { throw new OMException(ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR); } else { @@ -358,16 +359,22 @@ public OmMultipartInfo initiateMultipartUpload(String keyName, ReplicationType type, ReplicationFactor factor) throws IOException { - String uploadID = UUID.randomUUID().toString(); - multipartUploadIdMap.put(keyName, uploadID); - return new OmMultipartInfo(getVolumeName(), getName(), keyName, uploadID); + return initiateMultipartUpload(keyName, ReplicationConfig.fromTypeAndFactor(type, factor), + Collections.emptyMap()); } @Override public OmMultipartInfo initiateMultipartUpload(String keyName, ReplicationConfig repConfig) throws IOException { + return initiateMultipartUpload(keyName, repConfig, Collections.emptyMap()); + } + + @Override + public OmMultipartInfo initiateMultipartUpload(String keyName, + ReplicationConfig config, Map metadata) + throws IOException { String uploadID = UUID.randomUUID().toString(); - multipartUploadIdMap.put(keyName, uploadID); + keyToMultipartUpload.put(keyName, new MultipartInfoStub(uploadID, metadata)); return new OmMultipartInfo(getVolumeName(), getName(), keyName, uploadID); } @@ -375,7 +382,7 @@ public OmMultipartInfo initiateMultipartUpload(String keyName, public OzoneOutputStream createMultipartKey(String key, long size, int partNumber, String uploadID) throws IOException { - String multipartUploadID = multipartUploadIdMap.get(key); + String multipartUploadID = keyToMultipartUpload.get(key).getUploadId(); if (multipartUploadID == null || !multipartUploadID.equals(uploadID)) { throw new OMException(ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR); } else { @@ -402,7 +409,7 @@ public void close() throws IOException { public OmMultipartUploadCompleteInfo completeMultipartUpload(String key, String uploadID, Map partsMap) throws IOException { - if (multipartUploadIdMap.get(key) == null) { + if (keyToMultipartUpload.get(key) == null) { throw new OMException(ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR); } else { final Map partsList = partList.get(key); @@ -429,6 +436,18 @@ public OmMultipartUploadCompleteInfo completeMultipartUpload(String key, } keyContents.put(key, output.toByteArray()); } + + keyDetails.put(key, new OzoneKeyDetails( + getVolumeName(), + getName(), + key, + keyContents.get(key).length, + System.currentTimeMillis(), + System.currentTimeMillis(), + new ArrayList<>(), getReplicationConfig(), + keyToMultipartUpload.get(key).getMetadata(), null, + () -> readKey(key), true + )); } return new OmMultipartUploadCompleteInfo(getVolumeName(), getName(), key, @@ -438,17 +457,17 @@ public OmMultipartUploadCompleteInfo completeMultipartUpload(String key, @Override public void abortMultipartUpload(String keyName, String uploadID) throws IOException { - if (multipartUploadIdMap.get(keyName) == null) { + if (keyToMultipartUpload.get(keyName) == null) { throw new OMException(ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR); } else { - multipartUploadIdMap.remove(keyName); + keyToMultipartUpload.remove(keyName); } } @Override public OzoneMultipartUploadPartListParts listParts(String key, String uploadID, int partNumberMarker, int maxParts) throws IOException { - if (multipartUploadIdMap.get(key) == null) { + if (keyToMultipartUpload.get(key) == null) { throw new OMException(ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR); } List partInfoList = new ArrayList<>(); @@ -642,4 +661,26 @@ public Map getMetadata() { } } + /** + * Multipart upload stub to store MPU related information. + */ + private static class MultipartInfoStub { + + private final String uploadId; + private final Map metadata; + + MultipartInfoStub(String uploadId, Map metadata) { + this.uploadId = uploadId; + this.metadata = metadata; + } + + public String getUploadId() { + return uploadId; + } + + public Map getMetadata() { + return metadata; + } + } + } From ad24812d2bb744b828ba048417f77ae1e7c5186c Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 11:08:55 +0800 Subject: [PATCH 02/18] Add acceptance test --- .../dist/src/main/smoketest/s3/MultipartUpload.robot | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot index 3a6ae0e45d45..e2cb94ef9564 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot @@ -87,7 +87,7 @@ Test Multipart Upload Test Multipart Upload Complete - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --metadata="custom-key1=custom-value1,custom-key2=custom-value2" ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} Should contain ${result} ${PREFIX}/multipartKey @@ -117,6 +117,15 @@ Test Multipart Upload Complete Should contain ${result} ETag Should Be Equal As Strings ${resultETag} "${expectedResultETag}-2" +#check whether the user defined metadata can be retrieved + ${result} = Execute AWSS3ApiCli head-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 + Should contain ${result} \"custom-key1\": \"custom-value1\" + Should contain ${result} \"custom-key2\": \"custom-value2\" + + ${result} = Execute ozone sh key info /s3v/${BUCKET}/${PREFIX}/multipartKey1 + Should contain ${result} \"custom-key1\" : \"custom-value1\" + Should contain ${result} \"custom-key2\" : \"custom-value2\" + #read file and check the key ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 /tmp/${PREFIX}-multipartKey1.result Execute cat /tmp/part1 /tmp/part2 > /tmp/${PREFIX}-multipartKey1 From c07fc7393f2584ee6cc5a9edb89263db3d66750e Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 11:20:14 +0800 Subject: [PATCH 03/18] Acceptance test for large metadata --- .../dist/src/main/smoketest/s3/MultipartUpload.robot | 6 ++++++ hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot index e2cb94ef9564..7e268daef1d8 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot @@ -137,6 +137,12 @@ Test Multipart Upload Complete ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 2 /tmp/${PREFIX}-multipartKey1-part2.result Compare files /tmp/part2 /tmp/${PREFIX}-multipartKey1-part2.result +Test Multipart Upload with user defined metadata size larger than 2 KB + ${custom_metadata_value} = Execute printf 'v%.0s' {1..3000} + ${result} = Execute AWSS3APICli and checkrc create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/mpuWithLargeMetadata --metadata="custom-key1=${custom_metadata_value}" 255 + Should contain ${result} MetadataTooLarge + Should not contain ${result} custom-key1: ${custom_metadata_value} + Test Multipart Upload Complete Entity too small ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 05348fbcba4b..583e5a0ab408 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -185,8 +185,9 @@ Create file with user defined metadata with gdpr enabled value in request Create file with user defined metadata size larger than 2 KB Execute echo "Randomtext" > /tmp/testfile2 ${custom_metadata_value} = Execute printf 'v%.0s' {1..3000} - ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="custom-key1=${custom_metadata_value}" - Should not contain ${result} custom-key1: ${custom_metadata_value} + ${result} = Execute AWSS3APICli and checkrc put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="custom-key1=${custom_metadata_value}" 255 + Should contain ${result} MetadataTooLarge + Should not contain ${result} custom-key1: ${custom_metadata_value} Create small file and expect ETag (MD5) in a reponse header Execute head -c 1MB /tmp/small_file From 9648687740e105eba34f7fe0f7baf5a6cf3a95b6 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 11:21:38 +0800 Subject: [PATCH 04/18] Acceptance test for removing gdpr enabled metadata --- hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot index 7e268daef1d8..9ed629d27cd3 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot @@ -87,7 +87,7 @@ Test Multipart Upload Test Multipart Upload Complete - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --metadata="custom-key1=custom-value1,custom-key2=custom-value2" + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --metadata="custom-key1=custom-value1,custom-key2=custom-value2,gdprEnabled=true" ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} Should contain ${result} ${PREFIX}/multipartKey @@ -125,6 +125,7 @@ Test Multipart Upload Complete ${result} = Execute ozone sh key info /s3v/${BUCKET}/${PREFIX}/multipartKey1 Should contain ${result} \"custom-key1\" : \"custom-value1\" Should contain ${result} \"custom-key2\" : \"custom-value2\" + Should not contain ${result} \"gdprEnabled\": \"true\" #read file and check the key ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 /tmp/${PREFIX}-multipartKey1.result From c8aa1b741b377ed1127ea665a73a376067d86703 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 11:26:25 +0800 Subject: [PATCH 05/18] Add method to ClientProtocolStub --- .../hadoop/ozone/client/ClientProtocolStub.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 0400bc60500c..bc562d5d9363 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -56,6 +56,7 @@ import java.io.IOException; import java.net.URI; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -309,8 +310,16 @@ public OmMultipartInfo initiateMultipartUpload(String volumeName, public OmMultipartInfo initiateMultipartUpload(String volumeName, String bucketName, String keyName, ReplicationConfig replicationConfig) throws IOException { + return initiateMultipartUpload(volumeName, bucketName, keyName, replicationConfig, Collections.emptyMap()); + } + + @Override + public OmMultipartInfo initiateMultipartUpload(String volumeName, + String bucketName, String keyName, ReplicationConfig replicationConfig, + Map metadata) + throws IOException { return getBucket(volumeName, bucketName) - .initiateMultipartUpload(keyName, replicationConfig); + .initiateMultipartUpload(keyName, replicationConfig, metadata); } @Override From 1ec06145e2c7549c3baa2ce3abaca5c69d09495d Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 12:17:58 +0800 Subject: [PATCH 06/18] Add more space for the checkrc test --- hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot | 2 +- hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot index 9ed629d27cd3..96feec2f8133 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot @@ -140,7 +140,7 @@ Test Multipart Upload Complete Test Multipart Upload with user defined metadata size larger than 2 KB ${custom_metadata_value} = Execute printf 'v%.0s' {1..3000} - ${result} = Execute AWSS3APICli and checkrc create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/mpuWithLargeMetadata --metadata="custom-key1=${custom_metadata_value}" 255 + ${result} = Execute AWSS3APICli and checkrc create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/mpuWithLargeMetadata --metadata="custom-key1=${custom_metadata_value}" 255 Should contain ${result} MetadataTooLarge Should not contain ${result} custom-key1: ${custom_metadata_value} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 583e5a0ab408..bbff89e71f83 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -185,7 +185,7 @@ Create file with user defined metadata with gdpr enabled value in request Create file with user defined metadata size larger than 2 KB Execute echo "Randomtext" > /tmp/testfile2 ${custom_metadata_value} = Execute printf 'v%.0s' {1..3000} - ${result} = Execute AWSS3APICli and checkrc put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="custom-key1=${custom_metadata_value}" 255 + ${result} = Execute AWSS3APICli and checkrc put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="custom-key1=${custom_metadata_value}" 255 Should contain ${result} MetadataTooLarge Should not contain ${result} custom-key1: ${custom_metadata_value} From 1cc2b1d9633027843c77b2471e474432ba093c43 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 13:13:11 +0800 Subject: [PATCH 07/18] Add unit test for mpu with metadata --- .../hadoop/ozone/client/OzoneBucketStub.java | 3 +- .../endpoint/TestMultipartUploadComplete.java | 55 ++++++++++++++++++- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 41afd529369c..3f72b37a914f 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -276,6 +276,7 @@ public OzoneKey headObject(String key) throws IOException { ozoneKeyDetails.getCreationTime().toEpochMilli(), ozoneKeyDetails.getModificationTime().toEpochMilli(), ozoneKeyDetails.getReplicationConfig(), + ozoneKeyDetails.getMetadata(), ozoneKeyDetails.isFile()); } else { throw new OMException(ResultCodes.KEY_NOT_FOUND); @@ -414,8 +415,6 @@ public OmMultipartUploadCompleteInfo completeMultipartUpload(String key, } else { final Map partsList = partList.get(key); - int count = 1; - ByteArrayOutputStream output = new ByteArrayOutputStream(); int prevPartNumber = 0; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java index 3c0c87a177f6..e3ffe416cf70 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java @@ -30,16 +30,22 @@ import org.junit.jupiter.api.Test; import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MultivaluedHashMap; +import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -54,6 +60,7 @@ public class TestMultipartUploadComplete { private static final ObjectEndpoint REST = new ObjectEndpoint(); + private static final HttpHeaders headers = mock(HttpHeaders.class); private static final OzoneClient CLIENT = new OzoneClientStub(); @BeforeAll @@ -61,8 +68,6 @@ public static void setUp() throws Exception { CLIENT.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET); - - HttpHeaders headers = mock(HttpHeaders.class); when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( "STANDARD"); @@ -73,6 +78,20 @@ public static void setUp() throws Exception { private String initiateMultipartUpload(String key) throws IOException, OS3Exception { + return initiateMultipartUpload(key, Collections.emptyMap()); + } + + private String initiateMultipartUpload(String key, Map metadata) throws IOException, + OS3Exception { + MultivaluedMap metadataHeaders = new MultivaluedHashMap<>(); + + for (Map.Entry entry : metadata.entrySet()) { + metadataHeaders.computeIfAbsent(CUSTOM_METADATA_HEADER_PREFIX + entry.getKey(), k -> new ArrayList<>()) + .add(entry.getValue()); + } + + when(headers.getRequestHeaders()).thenReturn(metadataHeaders); + Response response = REST.initializeMultipartUpload(OzoneConsts.S3_BUCKET, key); MultipartUploadInitiateResponse multipartUploadInitiateResponse = @@ -83,7 +102,6 @@ private String initiateMultipartUpload(String key) throws IOException, assertEquals(200, response.getStatus()); return uploadID; - } private Part uploadPart(String key, String uploadID, int partNumber, String @@ -152,6 +170,37 @@ public void testMultipart() throws Exception { } + @Test + public void testMultipartWithMetadata() throws Exception { + String key = UUID.randomUUID().toString(); + + Map metadata = new HashMap<>(); + metadata.put("custom-key1", "custom-value1"); + metadata.put("custom-key2", "custom-value2"); + + String uploadID = initiateMultipartUpload(key, metadata); + + List partsList = new ArrayList<>(); + + // Upload parts + String content = "Multipart Upload 1"; + int partNumber = 1; + + Part part1 = uploadPart(key, uploadID, partNumber, content); + partsList.add(part1); + + CompleteMultipartUploadRequest completeMultipartUploadRequest = new + CompleteMultipartUploadRequest(); + completeMultipartUploadRequest.setPartList(partsList); + + completeMultipartUpload(key, completeMultipartUploadRequest, uploadID); + + Response headResponse = REST.head(OzoneConsts.S3_BUCKET, key); + + assertEquals("custom-value1", headResponse.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + "custom-key1")); + assertEquals("custom-value2", headResponse.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + "custom-key2")); + } + @Test public void testMultipartInvalidPartOrderError() throws Exception { From 2668798f1d30e75fba07dd07bc9b21c2ce442506 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 13:23:07 +0800 Subject: [PATCH 08/18] Fix test failure due to OzoneBucketStub --- .../org/apache/hadoop/ozone/client/OzoneBucketStub.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 3f72b37a914f..15f75e83da54 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -383,8 +383,8 @@ public OmMultipartInfo initiateMultipartUpload(String keyName, public OzoneOutputStream createMultipartKey(String key, long size, int partNumber, String uploadID) throws IOException { - String multipartUploadID = keyToMultipartUpload.get(key).getUploadId(); - if (multipartUploadID == null || !multipartUploadID.equals(uploadID)) { + MultipartInfoStub multipartInfo = keyToMultipartUpload.get(key); + if (multipartInfo == null || !multipartInfo.getUploadId().equals(uploadID)) { throw new OMException(ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR); } else { ByteArrayOutputStream byteArrayOutputStream = @@ -440,7 +440,7 @@ public OmMultipartUploadCompleteInfo completeMultipartUpload(String key, getVolumeName(), getName(), key, - keyContents.get(key).length, + keyContents.get(key) != null ? keyContents.get(key).length : 0, System.currentTimeMillis(), System.currentTimeMillis(), new ArrayList<>(), getReplicationConfig(), From 5c68af68d7b253a4f1e10433b329ce5e53d37b66 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 13:28:44 +0800 Subject: [PATCH 09/18] Fix TestPermissionCheck --- .../apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index d891573d5f13..cf4dc6ac84cf 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -42,6 +42,7 @@ import java.util.Map; import static java.net.HttpURLConnection.HTTP_FORBIDDEN; +import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; import static org.mockito.Mockito.anyLong; @@ -278,7 +279,7 @@ public void testDeleteKey() throws IOException { @Test public void testMultiUploadKey() throws IOException { when(objectStore.getS3Bucket(anyString())).thenReturn(bucket); - doThrow(exception).when(bucket).initiateMultipartUpload(anyString(), any()); + doThrow(exception).when(bucket).initiateMultipartUpload(anyString(), any(), anyMap()); ObjectEndpoint objectEndpoint = new ObjectEndpoint(); objectEndpoint.setClient(client); objectEndpoint.setHeaders(headers); From 0a101276a119e0efb53d73f0d94ebc9cd1987c64 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 15:10:20 +0800 Subject: [PATCH 10/18] Add the custom metadata in the client side translator --- .../protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 08fa029833e7..24e2fc9d86ca 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1605,6 +1605,7 @@ public OmMultipartInfo initiateMultipartUpload(OmKeyArgs omKeyArgs) throws .setVolumeName(omKeyArgs.getVolumeName()) .setBucketName(omKeyArgs.getBucketName()) .setKeyName(omKeyArgs.getKeyName()) + .addAllMetadata(KeyValueUtil.toProtobuf(omKeyArgs.getMetadata())) .addAllAcls(omKeyArgs.getAcls().stream().map(a -> OzoneAcl.toProtobuf(a)).collect(Collectors.toList())); From 9d2745988ccd4cd44da7efea675f2d02c3e39577 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 15:11:07 +0800 Subject: [PATCH 11/18] Add metadata when there was no existing key --- .../request/s3/multipart/S3MultipartUploadCompleteRequest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 0394a24e9adc..b161bd7be678 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -448,6 +448,9 @@ protected OmKeyInfo getOmKeyInfo(OzoneManager ozoneManager, long trxnLogIndex, omKeyInfo.setModificationTime(keyArgs.getModificationTime()); omKeyInfo.setDataSize(dataSize); omKeyInfo.setReplicationConfig(dbOpenKeyInfo.getReplicationConfig()); + if (dbOpenKeyInfo.getMetadata() != null) { + omKeyInfo.setMetadata(dbOpenKeyInfo.getMetadata()); + } omKeyInfo.getMetadata().put(OzoneConsts.ETAG, multipartUploadedKeyHash(partKeyInfoMap)); } From 524c255a6d75d1c0ebdfbf726a262ce521dfb204 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 15:16:37 +0800 Subject: [PATCH 12/18] Add some integration tests --- .../rpc/TestOzoneRpcClientAbstract.java | 45 ++++++++++++++++++- .../endpoint/TestMultipartUploadComplete.java | 18 ++++---- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index a77edd3abc59..6c967f027907 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -25,6 +25,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -2960,6 +2961,26 @@ public void testMultipartPartNumberExceedingAllowedRange() throws Exception { keyName, sampleData.length(), 10001, uploadID)); } + @ParameterizedTest + @MethodSource("replicationConfigs") + public void testMultipartUploadWithCustomMetadata(ReplicationConfig replication) throws Exception { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + String keyName = UUID.randomUUID().toString(); + + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + // Create custom metadata + Map customMetadata = new HashMap<>(); + customMetadata.put("custom-key1", "custom-value1"); + customMetadata.put("custom-key2", "custom-value2"); + + doMultipartUpload(bucket, keyName, (byte) 98, replication, customMetadata); + } + @Test public void testAbortUploadFail() throws Exception { String volumeName = UUID.randomUUID().toString(); @@ -3593,8 +3614,14 @@ private byte[] generateData(int size, byte val) { private void doMultipartUpload(OzoneBucket bucket, String keyName, byte val, ReplicationConfig replication) throws Exception { + doMultipartUpload(bucket, keyName, val, replication, Collections.emptyMap()); + } + + private void doMultipartUpload(OzoneBucket bucket, String keyName, byte val, + ReplicationConfig replication, Map customMetadata) + throws Exception { // Initiate Multipart upload request - String uploadID = initiateMultipartUpload(bucket, keyName, replication); + String uploadID = initiateMultipartUpload(bucket, keyName, replication, customMetadata); // Upload parts Map partsMap = new TreeMap<>(); @@ -3661,12 +3688,26 @@ private void doMultipartUpload(OzoneBucket bucket, String keyName, byte val, latestVersionLocations.getBlocksLatestVersionOnly() .forEach(omKeyLocationInfo -> assertNotEquals(-1, omKeyLocationInfo.getPartNumber())); + + Map keyMetadata = omKeyInfo.getMetadata(); + assertNotNull(keyMetadata.get(ETAG)); + if (customMetadata != null) { + for (Map.Entry customEntry : customMetadata.entrySet()) { + assertTrue(keyMetadata.containsKey(customEntry.getKey())); + assertEquals(customEntry.getValue(), keyMetadata.get(customEntry.getKey())); + } + } } private String initiateMultipartUpload(OzoneBucket bucket, String keyName, ReplicationConfig replicationConfig) throws Exception { + return initiateMultipartUpload(bucket, keyName, replicationConfig, Collections.emptyMap()); + } + + private String initiateMultipartUpload(OzoneBucket bucket, String keyName, + ReplicationConfig replicationConfig, Map customMetadata) throws Exception { OmMultipartInfo multipartInfo = bucket.initiateMultipartUpload(keyName, - replicationConfig); + replicationConfig, customMetadata); String uploadID = multipartInfo.getUploadID(); assertNotNull(uploadID); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java index e3ffe416cf70..b23dbfb9c05c 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java @@ -60,7 +60,7 @@ public class TestMultipartUploadComplete { private static final ObjectEndpoint REST = new ObjectEndpoint(); - private static final HttpHeaders headers = mock(HttpHeaders.class); + private static final HttpHeaders HEADERS = mock(HttpHeaders.class); private static final OzoneClient CLIENT = new OzoneClientStub(); @BeforeAll @@ -68,10 +68,10 @@ public static void setUp() throws Exception { CLIENT.getObjectStore().createS3Bucket(OzoneConsts.S3_BUCKET); - when(headers.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( + when(HEADERS.getHeaderString(STORAGE_CLASS_HEADER)).thenReturn( "STANDARD"); - REST.setHeaders(headers); + REST.setHeaders(HEADERS); REST.setClient(CLIENT); REST.setOzoneConfiguration(new OzoneConfiguration()); } @@ -90,7 +90,7 @@ private String initiateMultipartUpload(String key, Map metadata) .add(entry.getValue()); } - when(headers.getRequestHeaders()).thenReturn(metadataHeaders); + when(HEADERS.getRequestHeaders()).thenReturn(metadataHeaders); Response response = REST.initializeMultipartUpload(OzoneConsts.S3_BUCKET, key); @@ -171,14 +171,14 @@ public void testMultipart() throws Exception { } @Test - public void testMultipartWithMetadata() throws Exception { + public void testMultipartWithCustomMetadata() throws Exception { String key = UUID.randomUUID().toString(); - Map metadata = new HashMap<>(); - metadata.put("custom-key1", "custom-value1"); - metadata.put("custom-key2", "custom-value2"); + Map customMetadata = new HashMap<>(); + customMetadata.put("custom-key1", "custom-value1"); + customMetadata.put("custom-key2", "custom-value2"); - String uploadID = initiateMultipartUpload(key, metadata); + String uploadID = initiateMultipartUpload(key, customMetadata); List partsList = new ArrayList<>(); From c8889a8f587c9cc9407441348b0c5eb6013383e2 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 6 Apr 2024 15:59:15 +0800 Subject: [PATCH 13/18] Add om request tests --- .../ozone/om/request/OMRequestTestUtils.java | 23 +++++++- .../TestS3InitiateMultipartUploadRequest.java | 12 +++- ...InitiateMultipartUploadRequestWithFSO.java | 11 +++- .../s3/multipart/TestS3MultipartRequest.java | 59 ++++++++++++++++++- .../TestS3MultipartUploadCompleteRequest.java | 23 ++++++-- 5 files changed, 118 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java index 0ebd6946bd29..8103f6616c57 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.UUID; import javax.xml.bind.DatatypeConverter; @@ -49,6 +50,7 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.KeyValueUtil; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -991,11 +993,28 @@ public static String deleteKey(String ozoneKey, */ public static OMRequest createInitiateMPURequest(String volumeName, String bucketName, String keyName) { + return createInitiateMPURequest(volumeName, bucketName, keyName, Collections.emptyMap()); + } + + /** + * Create OMRequest which encapsulates InitiateMultipartUpload request. + * @param volumeName + * @param bucketName + * @param keyName + * @param metadata + */ + public static OMRequest createInitiateMPURequest(String volumeName, + String bucketName, String keyName, Map metadata) { MultipartInfoInitiateRequest multipartInfoInitiateRequest = MultipartInfoInitiateRequest.newBuilder().setKeyArgs( - KeyArgs.newBuilder().setVolumeName(volumeName).setKeyName(keyName) - .setBucketName(bucketName)).build(); + KeyArgs.newBuilder() + .setVolumeName(volumeName) + .setKeyName(keyName) + .setBucketName(bucketName) + .addAllMetadata(KeyValueUtil.toProtobuf(metadata)) + ) + .build(); return OMRequest.newBuilder().setClientId(UUID.randomUUID().toString()) .setCmdType(OzoneManagerProtocolProtos.Type.InitiateMultiPartUpload) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java index a4c512b25aa7..01657162310c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequest.java @@ -25,7 +25,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; @@ -62,8 +64,12 @@ public void testValidateAndUpdateCache() throws Exception { OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + Map customMetadata = new HashMap<>(); + customMetadata.put("custom-key1", "custom-value1"); + customMetadata.put("custom-key2", "custom-value2"); + OMRequest modifiedRequest = doPreExecuteInitiateMPU(volumeName, - bucketName, keyName); + bucketName, keyName, customMetadata); S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = getS3InitiateMultipartUploadReq(modifiedRequest); @@ -84,6 +90,10 @@ public void testValidateAndUpdateCache() throws Exception { assertNotNull(openMPUKeyInfo); assertNotNull(openMPUKeyInfo.getLatestVersionLocations()); assertTrue(openMPUKeyInfo.getLatestVersionLocations().isMultipartKey()); + assertNotNull(openMPUKeyInfo.getMetadata()); + assertEquals("custom-value1", openMPUKeyInfo.getMetadata().get("custom-key1")); + assertEquals("custom-value2", openMPUKeyInfo.getMetadata().get("custom-key2")); + assertNotNull(omMetadataManager.getMultipartInfoTable().get(multipartKey)); assertEquals(modifiedRequest.getInitiateMultiPartUploadRequest() diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestWithFSO.java index cbdea7572069..dd8eb00edb90 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestWithFSO.java @@ -36,7 +36,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; @@ -62,11 +64,15 @@ public void testValidateAndUpdateCache() throws Exception { OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + Map customMetadata = new HashMap<>(); + customMetadata.put("custom-key1", "custom-value1"); + customMetadata.put("custom-key2", "custom-value2"); + final long volumeId = omMetadataManager.getVolumeId(volumeName); final long bucketId = omMetadataManager.getBucketId(volumeName, bucketName); OMRequest modifiedRequest = doPreExecuteInitiateMPUWithFSO(volumeName, - bucketName, keyName); + bucketName, keyName, customMetadata); S3InitiateMultipartUploadRequest s3InitiateMultipartUploadReqFSO = getS3InitiateMultipartUploadReq(modifiedRequest); @@ -102,6 +108,9 @@ public void testValidateAndUpdateCache() throws Exception { "FileName mismatches!"); assertEquals(parentID, omKeyInfo.getParentObjectID(), "ParentId mismatches!"); + assertNotNull(omKeyInfo.getMetadata()); + assertEquals("custom-value1", omKeyInfo.getMetadata().get("custom-key1")); + assertEquals("custom-value2", omKeyInfo.getMetadata().get("custom-key2")); OmMultipartKeyInfo omMultipartKeyInfo = omMetadataManager .getMultipartInfoTable().get(multipartFileKey); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java index 16cb9b6821a1..ac26c566d4ef 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java @@ -21,10 +21,13 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.Collections; import java.util.List; +import java.util.Map; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.KeyValueUtil; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; import org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer; @@ -51,6 +54,7 @@ import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -130,9 +134,24 @@ public void stop() { */ protected OMRequest doPreExecuteInitiateMPU( String volumeName, String bucketName, String keyName) throws Exception { + return doPreExecuteInitiateMPU(volumeName, bucketName, keyName, Collections.emptyMap()); + } + + /** + * Perform preExecute of Initiate Multipart upload request for given + * volume, bucket and key name. + * @param volumeName + * @param bucketName + * @param keyName + * @param metadata + * @return OMRequest - returned from preExecute. + */ + protected OMRequest doPreExecuteInitiateMPU( + String volumeName, String bucketName, String keyName, + Map metadata) throws Exception { OMRequest omRequest = OMRequestTestUtils.createInitiateMPURequest(volumeName, bucketName, - keyName); + keyName, metadata); S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = getS3InitiateMultipartUploadReq(omRequest); @@ -147,6 +166,17 @@ protected OMRequest doPreExecuteInitiateMPU( assertThat(modifiedRequest.getInitiateMultiPartUploadRequest() .getKeyArgs().getModificationTime()).isGreaterThan(0); + if (metadata != null) { + Map modifiedKeyMetadata = KeyValueUtil.getFromProtobuf( + modifiedRequest.getInitiateMultiPartUploadRequest() + .getKeyArgs().getMetadataList()); + + for (Map.Entry entry : metadata.entrySet()) { + assertTrue(modifiedKeyMetadata.containsKey(entry.getKey())); + assertEquals(entry.getValue(), modifiedKeyMetadata.get(entry.getKey())); + } + } + return modifiedRequest; } @@ -247,9 +277,24 @@ protected OMRequest doPreExecuteCompleteMPU( */ protected OMRequest doPreExecuteInitiateMPUWithFSO( String volumeName, String bucketName, String keyName) throws Exception { + return doPreExecuteInitiateMPUWithFSO(volumeName, bucketName, keyName, Collections.emptyMap()); + } + + /** + * Perform preExecute of Initiate Multipart upload request for given + * volume, bucket and key name. + * @param volumeName + * @param bucketName + * @param keyName + * @param metadata + * @return OMRequest - returned from preExecute. + */ + protected OMRequest doPreExecuteInitiateMPUWithFSO( + String volumeName, String bucketName, String keyName, + Map metadata) throws Exception { OMRequest omRequest = OMRequestTestUtils.createInitiateMPURequest(volumeName, bucketName, - keyName); + keyName, metadata); S3InitiateMultipartUploadRequestWithFSO s3InitiateMultipartUploadRequestWithFSO = @@ -265,6 +310,16 @@ protected OMRequest doPreExecuteInitiateMPUWithFSO( .getKeyArgs().getMultipartUploadID()); assertThat(modifiedRequest.getInitiateMultiPartUploadRequest() .getKeyArgs().getModificationTime()).isGreaterThan(0); + if (metadata != null) { + Map modifiedKeyMetadata = KeyValueUtil.getFromProtobuf( + modifiedRequest.getInitiateMultiPartUploadRequest() + .getKeyArgs().getMetadataList()); + + for (Map.Entry entry : metadata.entrySet()) { + assertTrue(modifiedKeyMetadata.containsKey(entry.getKey())); + assertEquals(entry.getValue(), modifiedKeyMetadata.get(entry.getKey())); + } + } return modifiedRequest; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java index 34e32b0e182a..68ea80452de0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java @@ -26,7 +26,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import org.apache.hadoop.hdds.client.RatisReplicationConfig; @@ -72,13 +74,21 @@ public void testValidateAndUpdateCacheSuccess() throws Exception { OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager, getBucketLayout()); + Map customMetadata = new HashMap<>(); + customMetadata.put("custom-key1", "custom-value1"); + customMetadata.put("custom-key2", "custom-value2"); + String uploadId = checkValidateAndUpdateCacheSuccess( - volumeName, bucketName, keyName); + volumeName, bucketName, keyName, customMetadata); checkDeleteTableCount(volumeName, bucketName, keyName, 0, uploadId); + customMetadata.remove("custom-key1"); + customMetadata.remove("custom-key2"); + customMetadata.put("custom-key3", "custom-value3"); + // Do it twice to test overwrite uploadId = checkValidateAndUpdateCacheSuccess(volumeName, bucketName, - keyName); + keyName, customMetadata); // After overwrite, one entry must be in delete table checkDeleteTableCount(volumeName, bucketName, keyName, 1, uploadId); } @@ -106,10 +116,10 @@ public void checkDeleteTableCount(String volumeName, } private String checkValidateAndUpdateCacheSuccess(String volumeName, - String bucketName, String keyName) throws Exception { + String bucketName, String keyName, Map metadata) throws Exception { OMRequest initiateMPURequest = doPreExecuteInitiateMPU(volumeName, - bucketName, keyName); + bucketName, keyName, metadata); S3InitiateMultipartUploadRequest s3InitiateMultipartUploadRequest = getS3InitiateMultipartUploadReq(initiateMPURequest); @@ -175,6 +185,11 @@ private String checkValidateAndUpdateCacheSuccess(String volumeName, assertNotNull(multipartKeyInfo.getLatestVersionLocations()); assertTrue(multipartKeyInfo.getLatestVersionLocations() .isMultipartKey()); + if (metadata != null) { + for (Map.Entry entry : metadata.entrySet()) { + assertEquals(entry.getValue(), multipartKeyInfo.getMetadata().get(entry.getKey())); + } + } OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable() .getCacheValue(new CacheKey<>( From 8fdc308f12d32ec314eef64879cd5947aad27f16 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sun, 7 Apr 2024 20:39:50 +0800 Subject: [PATCH 14/18] combine the custom Etag metadata header logic to getCustomMetadataFromHeaders --- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 16 ++++++++++++++++ .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 12 ------------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 5810c4ec2a2f..136e47c776a5 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -19,6 +19,7 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; import javax.ws.rs.container.ContainerRequestContext; @@ -308,6 +309,20 @@ protected Map getCustomMetadataFromHeaders( customMetadata.put(mapKey, value); } } + + // If the request contains a custom metadata header "x-amz-meta-ETag", + // replace the metadata key to "etag-custom" to prevent key metadata collision with + // the ETag calculated by hashing the object when storing the key in OM table. + // The custom ETag metadata header will be rebuilt during the headObject operation. + if (customMetadata.containsKey(HttpHeaders.ETAG) + || customMetadata.containsKey(HttpHeaders.ETAG.toLowerCase())) { + String customETag = customMetadata.get(HttpHeaders.ETAG) != null ? + customMetadata.get(HttpHeaders.ETAG) : customMetadata.get(HttpHeaders.ETAG.toLowerCase()); + customMetadata.remove(HttpHeaders.ETAG); + customMetadata.remove(HttpHeaders.ETAG.toLowerCase()); + customMetadata.put(ETAG_CUSTOM, customETag); + } + return customMetadata; } @@ -321,6 +336,7 @@ protected void addCustomMetadataHeaders( } String metadataKey = entry.getKey(); if (metadataKey.equals(ETAG_CUSTOM)) { + // Rebuild the ETag custom metadata header metadataKey = ETAG.toLowerCase(); } responseBuilder diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 34538d86c958..782f393b23bc 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -289,7 +289,6 @@ public Response put( // Normal put object Map customMetadata = getCustomMetadataFromHeaders(headers.getRequestHeaders()); - replaceCustomETagMetadata(customMetadata); if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD" .equals(headers.getHeaderString("x-amz-content-sha256"))) { @@ -1323,17 +1322,6 @@ public static boolean checkCopySourceModificationTime( (lastModificationTime <= copySourceIfUnmodifiedSince); } - private void replaceCustomETagMetadata(Map customMetadata) { - if (customMetadata.containsKey(ETAG) - || customMetadata.containsKey(ETAG.toLowerCase())) { - String customETag = customMetadata.get(ETAG) != null ? - customMetadata.get(ETAG) : customMetadata.get(ETAG.toLowerCase()); - customMetadata.remove(ETAG); - customMetadata.remove(ETAG.toLowerCase()); - customMetadata.put(ETAG_CUSTOM, customETag); - } - } - @VisibleForTesting public void setOzoneConfiguration(OzoneConfiguration config) { this.ozoneConfiguration = config; From 5b0ba8a75fbfa052a022398cacb1a247763eb493 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sun, 7 Apr 2024 21:30:19 +0800 Subject: [PATCH 15/18] Remove the removed method call --- .../java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 782f393b23bc..747cfce7cacf 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -748,7 +748,6 @@ public Response initializeMultipartUpload( Map customMetadata = getCustomMetadataFromHeaders(headers.getRequestHeaders()); - replaceCustomETagMetadata(customMetadata); ReplicationConfig replicationConfig = getReplicationConfig(ozoneBucket, storageType); From ba5398ed6cac38f125db6ab50a1b5772a4e6d4d2 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 13 Apr 2024 16:00:29 +0800 Subject: [PATCH 16/18] Use Mockito and assertThat --- .../ozone/client/rpc/TestOzoneRpcClientAbstract.java | 5 +---- .../request/s3/multipart/TestS3MultipartRequest.java | 11 ++--------- .../TestS3MultipartUploadCompleteRequest.java | 4 +--- .../hadoop/ozone/s3/endpoint/TestPermissionCheck.java | 2 +- 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index 6c967f027907..8f0a20f7170f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -3692,10 +3692,7 @@ private void doMultipartUpload(OzoneBucket bucket, String keyName, byte val, Map keyMetadata = omKeyInfo.getMetadata(); assertNotNull(keyMetadata.get(ETAG)); if (customMetadata != null) { - for (Map.Entry customEntry : customMetadata.entrySet()) { - assertTrue(keyMetadata.containsKey(customEntry.getKey())); - assertEquals(customEntry.getValue(), keyMetadata.get(customEntry.getKey())); - } + assertThat(keyMetadata).containsAllEntriesOf(customMetadata); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java index ac26c566d4ef..1972fee69ba8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java @@ -54,7 +54,6 @@ import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -171,10 +170,7 @@ protected OMRequest doPreExecuteInitiateMPU( modifiedRequest.getInitiateMultiPartUploadRequest() .getKeyArgs().getMetadataList()); - for (Map.Entry entry : metadata.entrySet()) { - assertTrue(modifiedKeyMetadata.containsKey(entry.getKey())); - assertEquals(entry.getValue(), modifiedKeyMetadata.get(entry.getKey())); - } + assertThat(modifiedKeyMetadata).containsAllEntriesOf(metadata); } return modifiedRequest; @@ -315,10 +311,7 @@ protected OMRequest doPreExecuteInitiateMPUWithFSO( modifiedRequest.getInitiateMultiPartUploadRequest() .getKeyArgs().getMetadataList()); - for (Map.Entry entry : metadata.entrySet()) { - assertTrue(modifiedKeyMetadata.containsKey(entry.getKey())); - assertEquals(entry.getValue(), modifiedKeyMetadata.get(entry.getKey())); - } + assertThat(modifiedKeyMetadata).containsAllEntriesOf(metadata); } return modifiedRequest; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java index 68ea80452de0..663f2925cb16 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartUploadCompleteRequest.java @@ -186,9 +186,7 @@ private String checkValidateAndUpdateCacheSuccess(String volumeName, assertTrue(multipartKeyInfo.getLatestVersionLocations() .isMultipartKey()); if (metadata != null) { - for (Map.Entry entry : metadata.entrySet()) { - assertEquals(entry.getValue(), multipartKeyInfo.getMetadata().get(entry.getKey())); - } + assertThat(multipartKeyInfo.getMetadata()).containsAllEntriesOf(metadata); } OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable() diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index cf4dc6ac84cf..ec262cdf2158 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -42,10 +42,10 @@ import java.util.Map; import static java.net.HttpURLConnection.HTTP_FORBIDDEN; -import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.anyMap; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.isNull; From 120a9118f032dd7e05ddcdfd9186bf970e50e2f9 Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Sat, 13 Apr 2024 17:47:22 +0800 Subject: [PATCH 17/18] Only check if the custom metadata is not empty --- .../hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index 8f0a20f7170f..632076e2eefa 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -3691,7 +3691,7 @@ private void doMultipartUpload(OzoneBucket bucket, String keyName, byte val, Map keyMetadata = omKeyInfo.getMetadata(); assertNotNull(keyMetadata.get(ETAG)); - if (customMetadata != null) { + if (customMetadata != null && !customMetadata.isEmpty()) { assertThat(keyMetadata).containsAllEntriesOf(customMetadata); } } From 1de550053c12f0ec53be80120edd3d150a5ced7e Mon Sep 17 00:00:00 2001 From: Ivan Andika Date: Mon, 22 Apr 2024 09:32:37 +0800 Subject: [PATCH 18/18] Remove possible NPE in stubbed createMultipartStreamKey --- .../java/org/apache/hadoop/ozone/client/OzoneBucketStub.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 15f75e83da54..b5f37aaef3e0 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -211,8 +211,8 @@ public OzoneDataStreamOutput createMultipartStreamKey(String key, int partNumber, String uploadID) throws IOException { - String multipartUploadID = keyToMultipartUpload.get(key).getUploadId(); - if (multipartUploadID == null || !multipartUploadID.equals(uploadID)) { + MultipartInfoStub multipartInfo = keyToMultipartUpload.get(key); + if (multipartInfo == null || !multipartInfo.getUploadId().equals(uploadID)) { throw new OMException(ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR); } else { ByteBufferStreamOutput byteBufferStreamOutput =