From 202fc8f9525f569482f1b34da1b9ee560727abaa Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 15 Mar 2022 12:27:39 -0700 Subject: [PATCH 1/6] HDDS-6440. Handle custom metadata (x-amz-meta) through S3 API. --- .../apache/hadoop/ozone/client/OzoneKey.java | 38 ++++++++++++++++++ .../hadoop/ozone/client/OzoneKeyDetails.java | 12 +----- .../hadoop/ozone/client/rpc/RpcClient.java | 3 +- .../ozone/s3/endpoint/EndpointBase.java | 40 +++++++++++++++++++ .../ozone/s3/endpoint/ObjectEndpoint.java | 9 +++-- .../apache/hadoop/ozone/s3/util/S3Consts.java | 2 + 6 files changed, 89 insertions(+), 15 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index e37969d42a69..f51358b25897 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -23,6 +23,8 @@ import org.apache.hadoop.hdds.client.ReplicationType; import java.time.Instant; +import java.util.HashMap; +import java.util.Map; /** * A class that encapsulates OzoneKey. @@ -56,6 +58,8 @@ public class OzoneKey { private ReplicationConfig replicationConfig; + private Map metadata = new HashMap<>(); + /** * Constructs OzoneKey from OmKeyInfo. * @@ -76,6 +80,22 @@ public OzoneKey(String volumeName, String bucketName, ReplicationFactor.valueOf(replicationFactor)); } + @SuppressWarnings("parameternumber") + public OzoneKey(String volumeName, String bucketName, + String keyName, long size, long creationTime, + long modificationTime, ReplicationType type, + int replicationFactor, Map metadata) { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.name = keyName; + this.dataSize = size; + this.creationTime = Instant.ofEpochMilli(creationTime); + this.modificationTime = Instant.ofEpochMilli(modificationTime); + this.replicationConfig = ReplicationConfig.fromTypeAndFactor(type, + ReplicationFactor.valueOf(replicationFactor)); + this.metadata.putAll(metadata); + } + /** * Constructs OzoneKey from OmKeyInfo. * @@ -93,6 +113,20 @@ public OzoneKey(String volumeName, String bucketName, this.replicationConfig = replicationConfig; } + @SuppressWarnings("parameternumber") + public OzoneKey(String volumeName, String bucketName, + String keyName, long size, long creationTime, + long modificationTime, ReplicationConfig replicationConfig, + Map metadata) { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.name = keyName; + this.dataSize = size; + this.creationTime = Instant.ofEpochMilli(creationTime); + this.modificationTime = Instant.ofEpochMilli(modificationTime); + this.replicationConfig = replicationConfig; + this.metadata.putAll(metadata); + } /** * Returns Volume Name associated with the Key. * @@ -153,6 +187,10 @@ public Instant getModificationTime() { * @return replicationType */ + public Map getMetadata() { + return metadata; + } + @Deprecated public ReplicationType getReplicationType() { return ReplicationType diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index 7dc8008d0a27..1ccdb9c5a809 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -35,8 +35,6 @@ public class OzoneKeyDetails extends OzoneKey { */ private List ozoneKeyLocations; - private Map metadata; - private FileEncryptionInfo feInfo; /** @@ -50,9 +48,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, ReplicationType type, Map metadata, FileEncryptionInfo feInfo, int replicationFactor) { super(volumeName, bucketName, keyName, size, creationTime, - modificationTime, type, replicationFactor); + modificationTime, type, replicationFactor, metadata); this.ozoneKeyLocations = ozoneKeyLocations; - this.metadata = metadata; this.feInfo = feInfo; } @@ -68,9 +65,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, Map metadata, FileEncryptionInfo feInfo) { super(volumeName, bucketName, keyName, size, creationTime, - modificationTime, replicationConfig); + modificationTime, replicationConfig, metadata); this.ozoneKeyLocations = ozoneKeyLocations; - this.metadata = metadata; this.feInfo = feInfo; } @@ -81,10 +77,6 @@ public List getOzoneKeyLocations() { return ozoneKeyLocations; } - public Map getMetadata() { - return metadata; - } - public FileEncryptionInfo getFileEncryptionInfo() { return feInfo; } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 28efa8c91245..12dd3b9b9992 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1627,7 +1627,8 @@ public OzoneKey headObject(String volumeName, String bucketName, return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(), - keyInfo.getModificationTime(), keyInfo.getReplicationConfig()); + keyInfo.getModificationTime(), keyInfo.getReplicationConfig(), + keyInfo.getMetadata()); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index b93336374ead..0263acc66190 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -19,14 +19,24 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; + import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; @@ -35,10 +45,12 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import com.google.common.annotations.VisibleForTesting; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; +import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; /** * Basic helpers for all the REST endpoints. @@ -190,6 +202,34 @@ private Iterator iterateBuckets( } } + protected Map getCustomMetadataFromHeaders( + MultivaluedMap requestHeaders) { + Map customMetadata = new HashMap<>(); + Set customMetadataKeys = requestHeaders.keySet().stream() + .filter(k -> k.startsWith(CUSTOM_METADATA_HEADER_PREFIX)) + .collect(Collectors.toSet()); + if (!customMetadataKeys.isEmpty()) { + for (String key : customMetadataKeys) { + String mapKey = + key.substring(CUSTOM_METADATA_HEADER_PREFIX.length()); + List values = requestHeaders.get(key); + customMetadata.put(mapKey, StringUtils.join(values, ",")); + } + } + return customMetadata; + } + + protected void addCustomMetadataHeaders( + Response.ResponseBuilder responseBuilder, OzoneKey key) { + + Map metadata = key.getMetadata(); + for (Map.Entry entry : metadata.entrySet()) { + responseBuilder + .header(CUSTOM_METADATA_HEADER_PREFIX + entry.getKey(), + entry.getValue()); + } + } + @VisibleForTesting public void setClient(OzoneClient ozoneClient) { this.client = ozoneClient; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 863b1b08cd7e..766035237a59 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -47,7 +47,6 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -196,9 +195,10 @@ public Response put( // Normal put object OzoneBucket bucket = getBucket(bucketName); - + Map customMetadata = + getCustomMetadataFromHeaders(headers.getRequestHeaders()); output = bucket.createKey(keyPath, length, replicationType, - replicationFactor, new HashMap<>()); + replicationFactor, customMetadata); if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD" .equals(headers.getHeaderString("x-amz-content-sha256"))) { @@ -377,6 +377,7 @@ public Response head( .header("Content-Length", key.getDataSize()) .header("Content-Type", "binary/octet-stream"); addLastModifiedDate(response, key); + addCustomMetadataHeaders(response, key); return response.build(); } @@ -771,7 +772,7 @@ private CopyObjectResponse copyObject(String copyHeader, sourceInputStream = sourceOzoneBucket.readKey(sourceKey); destOutputStream = destOzoneBucket.createKey(destkey, sourceKeyLen, - replicationType, replicationFactor, new HashMap<>()); + replicationType, replicationFactor, sourceKeyDetails.getMetadata()); IOUtils.copy(sourceInputStream, destOutputStream); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index f891e13d7796..f620fd624c07 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -62,4 +62,6 @@ private S3Consts() { public static final String S3_XML_NAMESPACE = "http://s3.amazonaws" + ".com/doc/2006-03-01/"; + public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-"; + } From 67a99da8e0ea5c302b09c8e7f445b3ca89667523 Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 15 Mar 2022 13:15:22 -0700 Subject: [PATCH 2/6] HDDS-6440. Handle custom metadata (x-amz-meta) through S3 API. 2 --- .../dist/src/main/smoketest/s3/objectputget.robot | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 575e53826dc1..2f52dc8fa7a4 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -153,3 +153,15 @@ Zero byte file ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255 Should contain ${result} InvalidRange + +Create file with user defined metadata + Execute echo "Randomtext" > /tmp/testfile2 + Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 --body /tmp/testfile2 --metadata="custom-key1=custom-value1,custom-key2=custom-value2" + + ${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 + Should contain ${result} \"custom-key1\": \"custom-value1\" + Should contain ${result} \"custom-key2\": \"custom-value2\" + + ${result} = Execute ozone sh key info /s3v/${BUCKET}/${PREFIX}/putobject/custom-metadata/key1 + Should contain ${result} \"custom-key1\" : \"custom-value1\" + Should contain ${result} \"custom-key2\" : \"custom-value2\" \ No newline at end of file From 03be08e6018ef8df261a267b9b795149b56eaf07 Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 15 Mar 2022 14:13:53 -0700 Subject: [PATCH 3/6] Unit test fix. --- .../org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 0263acc66190..3e55d8c1c352 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -205,6 +205,10 @@ private Iterator iterateBuckets( protected Map getCustomMetadataFromHeaders( MultivaluedMap requestHeaders) { Map customMetadata = new HashMap<>(); + if (requestHeaders == null || requestHeaders.isEmpty()) { + return customMetadata; + } + Set customMetadataKeys = requestHeaders.keySet().stream() .filter(k -> k.startsWith(CUSTOM_METADATA_HEADER_PREFIX)) .collect(Collectors.toSet()); From 87280e65a3bef7a8089c6be7ae78f871e53c539a Mon Sep 17 00:00:00 2001 From: avijayanhwx <14299376+avijayanhwx@users.noreply.github.com> Date: Tue, 22 Mar 2022 14:21:22 -0700 Subject: [PATCH 4/6] Update hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java Co-authored-by: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../java/org/apache/hadoop/ozone/client/OzoneKey.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index f51358b25897..8351f3be6e90 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -118,13 +118,8 @@ public OzoneKey(String volumeName, String bucketName, String keyName, long size, long creationTime, long modificationTime, ReplicationConfig replicationConfig, Map metadata) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.name = keyName; - this.dataSize = size; - this.creationTime = Instant.ofEpochMilli(creationTime); - this.modificationTime = Instant.ofEpochMilli(modificationTime); - this.replicationConfig = replicationConfig; + this(volumeName, bucketName, keyName, size, creationTime, + modificationTime, replicationConfig); this.metadata.putAll(metadata); } /** From 44180c16c2813a56cfbf742634aa629ebcd3ec39 Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 29 Mar 2022 10:48:31 -0700 Subject: [PATCH 5/6] Max size of 2KB --- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 3e55d8c1c352..47e20b7e57b6 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -49,6 +49,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.OzoneConsts.KB; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; @@ -212,12 +214,20 @@ protected Map getCustomMetadataFromHeaders( Set customMetadataKeys = requestHeaders.keySet().stream() .filter(k -> k.startsWith(CUSTOM_METADATA_HEADER_PREFIX)) .collect(Collectors.toSet()); + long sizeInBytes = 0; if (!customMetadataKeys.isEmpty()) { for (String key : customMetadataKeys) { String mapKey = key.substring(CUSTOM_METADATA_HEADER_PREFIX.length()); List values = requestHeaders.get(key); - customMetadata.put(mapKey, StringUtils.join(values, ",")); + String value = StringUtils.join(values, ","); + sizeInBytes += mapKey.getBytes(UTF_8).length; + sizeInBytes += value.getBytes(UTF_8).length; + if (sizeInBytes > 2 * KB) { + throw new IllegalArgumentException("Illegal user defined metadata." + + " Combined size cannot exceed 2KB."); + } + customMetadata.put(mapKey, value); } } return customMetadata; From af9ee6c1b860e43cd84ce70ebcd5adf886ce5342 Mon Sep 17 00:00:00 2001 From: Aravindan Vijayan Date: Tue, 29 Mar 2022 10:59:18 -0700 Subject: [PATCH 6/6] Remove deprecated constructor. --- .../apache/hadoop/ozone/client/OzoneKey.java | 17 +---------------- .../hadoop/ozone/client/OzoneKeyDetails.java | 3 ++- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index 8351f3be6e90..a530024e33ef 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -80,22 +80,6 @@ public OzoneKey(String volumeName, String bucketName, ReplicationFactor.valueOf(replicationFactor)); } - @SuppressWarnings("parameternumber") - public OzoneKey(String volumeName, String bucketName, - String keyName, long size, long creationTime, - long modificationTime, ReplicationType type, - int replicationFactor, Map metadata) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.name = keyName; - this.dataSize = size; - this.creationTime = Instant.ofEpochMilli(creationTime); - this.modificationTime = Instant.ofEpochMilli(modificationTime); - this.replicationConfig = ReplicationConfig.fromTypeAndFactor(type, - ReplicationFactor.valueOf(replicationFactor)); - this.metadata.putAll(metadata); - } - /** * Constructs OzoneKey from OmKeyInfo. * @@ -122,6 +106,7 @@ public OzoneKey(String volumeName, String bucketName, modificationTime, replicationConfig); this.metadata.putAll(metadata); } + /** * Returns Volume Name associated with the Key. * diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index 1ccdb9c5a809..78a51d8fa79e 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -48,9 +48,10 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, ReplicationType type, Map metadata, FileEncryptionInfo feInfo, int replicationFactor) { super(volumeName, bucketName, keyName, size, creationTime, - modificationTime, type, replicationFactor, metadata); + modificationTime, type, replicationFactor); this.ozoneKeyLocations = ozoneKeyLocations; this.feInfo = feInfo; + this.getMetadata().putAll(metadata); }