diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index e37969d42a69..a530024e33ef 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -23,6 +23,8 @@ import org.apache.hadoop.hdds.client.ReplicationType; import java.time.Instant; +import java.util.HashMap; +import java.util.Map; /** * A class that encapsulates OzoneKey. @@ -56,6 +58,8 @@ public class OzoneKey { private ReplicationConfig replicationConfig; + private Map metadata = new HashMap<>(); + /** * Constructs OzoneKey from OmKeyInfo. * @@ -93,6 +97,16 @@ public OzoneKey(String volumeName, String bucketName, this.replicationConfig = replicationConfig; } + @SuppressWarnings("parameternumber") + public OzoneKey(String volumeName, String bucketName, + String keyName, long size, long creationTime, + long modificationTime, ReplicationConfig replicationConfig, + Map metadata) { + this(volumeName, bucketName, keyName, size, creationTime, + modificationTime, replicationConfig); + this.metadata.putAll(metadata); + } + /** * Returns Volume Name associated with the Key. * @@ -153,6 +167,10 @@ public Instant getModificationTime() { * @return replicationType */ + public Map getMetadata() { + return metadata; + } + @Deprecated public ReplicationType getReplicationType() { return ReplicationType diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index 7dc8008d0a27..78a51d8fa79e 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -35,8 +35,6 @@ public class OzoneKeyDetails extends OzoneKey { */ private List ozoneKeyLocations; - private Map metadata; - private FileEncryptionInfo feInfo; /** @@ -52,8 +50,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, super(volumeName, bucketName, keyName, size, creationTime, modificationTime, type, replicationFactor); this.ozoneKeyLocations = ozoneKeyLocations; - this.metadata = metadata; this.feInfo = feInfo; + this.getMetadata().putAll(metadata); } @@ -68,9 +66,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, Map metadata, FileEncryptionInfo feInfo) { super(volumeName, bucketName, keyName, size, creationTime, - modificationTime, replicationConfig); + modificationTime, replicationConfig, metadata); this.ozoneKeyLocations = ozoneKeyLocations; - this.metadata = metadata; this.feInfo = feInfo; } @@ -81,10 +78,6 @@ public List getOzoneKeyLocations() { return ozoneKeyLocations; } - public Map getMetadata() { - return metadata; - } - public FileEncryptionInfo getFileEncryptionInfo() { return feInfo; } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index e11f1abc993d..60aa063e29dc 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1638,7 +1638,8 @@ public OzoneKey headObject(String volumeName, String bucketName, return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(), - keyInfo.getModificationTime(), keyInfo.getReplicationConfig()); + keyInfo.getModificationTime(), keyInfo.getReplicationConfig(), + keyInfo.getMetadata()); } diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 575e53826dc1..2f52dc8fa7a4 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -153,3 +153,15 @@ Zero byte file ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255 Should contain ${result} InvalidRange + +Create file with user defined metadata + Execute echo "Randomtext" > /tmp/testfile2 + Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 --body /tmp/testfile2 --metadata="custom-key1=custom-value1,custom-key2=custom-value2" + + ${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 + Should contain ${result} \"custom-key1\": \"custom-value1\" + Should contain ${result} \"custom-key2\": \"custom-value2\" + + ${result} = Execute ozone sh key info /s3v/${BUCKET}/${PREFIX}/putobject/custom-metadata/key1 + Should contain ${result} \"custom-key1\" : \"custom-value1\" + Should contain ${result} \"custom-key2\" : \"custom-value2\" \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index a9411dd48f70..baccc8f6ff21 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -19,14 +19,24 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; + import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; @@ -35,11 +45,15 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import com.google.common.annotations.VisibleForTesting; + import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.OzoneConsts.KB; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; +import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; /** * Basic helpers for all the REST endpoints. @@ -191,6 +205,46 @@ private Iterator iterateBuckets( } } + protected Map getCustomMetadataFromHeaders( + MultivaluedMap requestHeaders) { + Map customMetadata = new HashMap<>(); + if (requestHeaders == null || requestHeaders.isEmpty()) { + return customMetadata; + } + + Set customMetadataKeys = requestHeaders.keySet().stream() + .filter(k -> k.startsWith(CUSTOM_METADATA_HEADER_PREFIX)) + .collect(Collectors.toSet()); + long sizeInBytes = 0; + if (!customMetadataKeys.isEmpty()) { + for (String key : customMetadataKeys) { + String mapKey = + key.substring(CUSTOM_METADATA_HEADER_PREFIX.length()); + List values = requestHeaders.get(key); + String value = StringUtils.join(values, ","); + sizeInBytes += mapKey.getBytes(UTF_8).length; + sizeInBytes += value.getBytes(UTF_8).length; + if (sizeInBytes > 2 * KB) { + throw new IllegalArgumentException("Illegal user defined metadata." + + " Combined size cannot exceed 2KB."); + } + customMetadata.put(mapKey, value); + } + } + return customMetadata; + } + + protected void addCustomMetadataHeaders( + Response.ResponseBuilder responseBuilder, OzoneKey key) { + + Map metadata = key.getMetadata(); + for (Map.Entry entry : metadata.entrySet()) { + responseBuilder + .header(CUSTOM_METADATA_HEADER_PREFIX + entry.getKey(), + entry.getValue()); + } + } + @VisibleForTesting public void setClient(OzoneClient ozoneClient) { this.client = ozoneClient; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 145578b0f637..a8a9442058c1 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -47,7 +47,6 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -197,9 +196,10 @@ public Response put( // Normal put object OzoneBucket bucket = getBucket(bucketName); - + Map customMetadata = + getCustomMetadataFromHeaders(headers.getRequestHeaders()); output = bucket.createKey(keyPath, length, replicationType, - replicationFactor, new HashMap<>()); + replicationFactor, customMetadata); if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD" .equals(headers.getHeaderString("x-amz-content-sha256"))) { @@ -391,6 +391,7 @@ public Response head( .header("Content-Length", key.getDataSize()) .header("Content-Type", "binary/octet-stream"); addLastModifiedDate(response, key); + addCustomMetadataHeaders(response, key); getMetrics().incHeadKeySuccess(); return response.build(); } @@ -800,7 +801,7 @@ private CopyObjectResponse copyObject(String copyHeader, sourceInputStream = sourceOzoneBucket.readKey(sourceKey); destOutputStream = destOzoneBucket.createKey(destkey, sourceKeyLen, - replicationType, replicationFactor, new HashMap<>()); + replicationType, replicationFactor, sourceKeyDetails.getMetadata()); IOUtils.copy(sourceInputStream, destOutputStream); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index f891e13d7796..f620fd624c07 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -62,4 +62,6 @@ private S3Consts() { public static final String S3_XML_NAMESPACE = "http://s3.amazonaws" + ".com/doc/2006-03-01/"; + public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-"; + }