diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index a3fb9e0ee9c..382b69644b9 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -391,6 +391,8 @@ private OzoneConsts() { public static final int S3_SECRET_KEY_MIN_LENGTH = 8; + public static final int S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB = 2; + //GDPR public static final String GDPR_FLAG = "gdprEnabled"; public static final String GDPR_ALGORITHM_NAME = "AES"; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index f0302c8e084..72dfb26f215 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -24,6 +24,8 @@ import org.apache.hadoop.hdds.client.ReplicationType; import java.time.Instant; +import java.util.Map; +import java.util.HashMap; /** * A class that encapsulates OzoneKey. @@ -57,6 +59,7 @@ public class OzoneKey { private ReplicationConfig replicationConfig; + private Map metadata = new HashMap<>(); /** * Constructs OzoneKey from OmKeyInfo. * @@ -67,14 +70,9 @@ public OzoneKey(String volumeName, String bucketName, String keyName, long size, long creationTime, long modificationTime, ReplicationType type, int replicationFactor) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.name = keyName; - this.dataSize = size; - this.creationTime = Instant.ofEpochMilli(creationTime); - this.modificationTime = Instant.ofEpochMilli(modificationTime); - this.replicationConfig = ReplicationConfig.fromTypeAndFactor(type, - ReplicationFactor.valueOf(replicationFactor)); + this(volumeName, bucketName, keyName, size, creationTime, modificationTime, + ReplicationConfig.fromTypeAndFactor(type, + ReplicationFactor.valueOf(replicationFactor))); } /** @@ -94,6 +92,16 @@ public OzoneKey(String volumeName, String bucketName, this.replicationConfig = replicationConfig; } + @SuppressWarnings("parameternumber") + public OzoneKey(String volumeName, String bucketName, + String keyName, long size, long creationTime, + long modificationTime, ReplicationConfig replicationConfig, + Map metadata) { + this(volumeName, bucketName, keyName, size, creationTime, + modificationTime, replicationConfig); + this.metadata.putAll(metadata); + } + /** * Returns Volume Name associated with the Key. * @@ -154,6 +162,14 @@ public Instant getModificationTime() { * @return replicationType */ + public Map getMetadata() { + return metadata; + } + + public void setMetadata(Map metadata) { + this.metadata.putAll(metadata); + } + @Deprecated @JsonIgnore public ReplicationType getReplicationType() { diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index 016c34683c6..757431bc6bf 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -39,8 +39,6 @@ public class OzoneKeyDetails extends OzoneKey { */ private List ozoneKeyLocations; - private Map metadata; - private FileEncryptionInfo feInfo; private SupplierWithIOException contentSupplier; @@ -58,8 +56,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, super(volumeName, bucketName, keyName, size, creationTime, modificationTime, type, replicationFactor); this.ozoneKeyLocations = ozoneKeyLocations; - this.metadata = metadata; this.feInfo = feInfo; + this.setMetadata(metadata); } @@ -75,9 +73,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, FileEncryptionInfo feInfo, SupplierWithIOException contentSupplier) { super(volumeName, bucketName, keyName, size, creationTime, - modificationTime, replicationConfig); + modificationTime, replicationConfig, metadata); this.ozoneKeyLocations = ozoneKeyLocations; - this.metadata = metadata; this.feInfo = feInfo; this.contentSupplier = contentSupplier; } @@ -89,10 +86,6 @@ public List getOzoneKeyLocations() { return ozoneKeyLocations; } - public Map getMetadata() { - return metadata; - } - public FileEncryptionInfo getFileEncryptionInfo() { return feInfo; } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index a588859f1cf..d944852fae7 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2008,7 +2008,8 @@ public OzoneKey headObject(String volumeName, String bucketName, return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(), - keyInfo.getModificationTime(), keyInfo.getReplicationConfig()); + keyInfo.getModificationTime(), keyInfo.getReplicationConfig(), + keyInfo.getMetadata()); } diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot index 0b03f3c75e3..60c73866f71 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot @@ -36,6 +36,11 @@ Execute AWSS3APICli and checkrc ${output} = Execute and checkrc aws s3api --endpoint-url ${ENDPOINT_URL} ${command} ${expected_error_code} [return] ${output} +Execute AWSS3APICli and ignore error + [Arguments] ${command} + ${output} = Execute And Ignore Error aws s3api --endpoint-url ${ENDPOINT_URL} ${command} + [return] ${output} + Execute AWSS3Cli [Arguments] ${command} ${output} = Execute aws s3 --endpoint-url ${ENDPOINT_URL} ${command} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 46608e73570..2f74decb617 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -159,3 +159,29 @@ Zero byte file ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255 Should contain ${result} InvalidRange + +Create file with user defined metadata + Execute echo "Randomtext" > /tmp/testfile2 + Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 --body /tmp/testfile2 --metadata="custom-key1=custom-value1,custom-key2=custom-value2" + + ${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 + Should contain ${result} \"custom-key1\": \"custom-value1\" + Should contain ${result} \"custom-key2\": \"custom-value2\" + + ${result} = Execute ozone sh key info /s3v/${BUCKET}/${PREFIX}/putobject/custom-metadata/key1 + Should contain ${result} \"custom-key1\" : \"custom-value1\" + Should contain ${result} \"custom-key2\" : \"custom-value2\" + +Create file with user defined metadata with gdpr enabled value in request + Execute echo "Randomtext" > /tmp/testfile2 + Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="gdprEnabled=true,custom-key2=custom-value2" + ${result} = Execute AWSS3ApiCli head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 + Should contain ${result} \"custom-key2\": \"custom-value2\" + Should not contain ${result} \"gdprEnabled\": \"true\" + + +Create file with user defined metadata size larger than 2 KB + Execute echo "Randomtext" > /tmp/testfile2 + ${custom_metadata_value} = Execute printf 'v%.0s' {1..3000} + ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="custom-key1=${custom_metadata_value}" + Should not contain ${result} custom-key1: ${custom_metadata_value} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index cc76f267b13..56a187fd004 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -19,14 +19,24 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.core.Context; import java.io.IOException; -import java.util.Collections; -import java.util.Iterator; +import java.util.Set; +import java.util.HashSet; +import java.util.Arrays; import java.util.Map; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Collections; import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditAction; import org.apache.hadoop.ozone.audit.AuditEventStatus; import org.apache.hadoop.ozone.audit.AuditLogger; @@ -35,6 +45,7 @@ import org.apache.hadoop.ozone.audit.Auditor; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -44,12 +55,16 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import com.google.common.annotations.VisibleForTesting; + import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; import org.apache.hadoop.ozone.s3.util.AuditUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.ozone.OzoneConsts.KB; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; +import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; /** * Basic helpers for all the REST endpoints. @@ -63,6 +78,8 @@ public abstract class EndpointBase implements Auditor { @Context private ContainerRequestContext context; + private Set excludeMetadataFields = + new HashSet<>(Arrays.asList(OzoneConsts.GDPR_FLAG)); private static final Logger LOG = LoggerFactory.getLogger(EndpointBase.class); @@ -240,6 +257,57 @@ private Iterator iterateBuckets( } } + protected Map getCustomMetadataFromHeaders( + MultivaluedMap requestHeaders) throws OS3Exception { + Map customMetadata = new HashMap<>(); + if (requestHeaders == null || requestHeaders.isEmpty()) { + return customMetadata; + } + + Set customMetadataKeys = requestHeaders.keySet().stream() + .filter(k -> { + if (k.startsWith(CUSTOM_METADATA_HEADER_PREFIX) && + !excludeMetadataFields.contains( + k.substring( + CUSTOM_METADATA_HEADER_PREFIX.length()))) { + return true; + } + return false; + }) + .collect(Collectors.toSet()); + + long sizeInBytes = 0; + if (!customMetadataKeys.isEmpty()) { + for (String key : customMetadataKeys) { + String mapKey = + key.substring(CUSTOM_METADATA_HEADER_PREFIX.length()); + List values = requestHeaders.get(key); + String value = StringUtils.join(values, ","); + sizeInBytes += mapKey.getBytes(UTF_8).length; + sizeInBytes += value.getBytes(UTF_8).length; + + if (sizeInBytes > + OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB * KB) { + throw new IllegalArgumentException("Illegal user defined metadata." + + " Combined size cannot exceed 2KB."); + } + customMetadata.put(mapKey, value); + } + } + return customMetadata; + } + + protected void addCustomMetadataHeaders( + Response.ResponseBuilder responseBuilder, OzoneKey key) { + + Map metadata = key.getMetadata(); + for (Map.Entry entry : metadata.entrySet()) { + responseBuilder + .header(CUSTOM_METADATA_HEADER_PREFIX + entry.getKey(), + entry.getValue()); + } + } + private AuditMessage.Builder auditMessageBaseBuilder(AuditAction op, Map auditMap) { AuditMessage.Builder builder = new AuditMessage.Builder() diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 307378c9f3b..04bc7677c48 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -48,10 +48,9 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.util.HashMap; +import java.util.Map; import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.OptionalLong; import org.apache.commons.lang3.StringUtils; @@ -210,13 +209,17 @@ public Response put( "Connection", "close").build(); } + // Normal put object + Map customMetadata = + getCustomMetadataFromHeaders(headers.getRequestHeaders()); + if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD" .equals(headers.getHeaderString("x-amz-content-sha256"))) { body = new SignedChunksInputStream(body); } output = getClientProtocol().createKey(volume.getName(), bucketName, - keyPath, length, replicationConfig, new HashMap<>()); + keyPath, length, replicationConfig, customMetadata); IOUtils.copy(body, output); getMetrics().incCreateKeySuccess(); @@ -470,6 +473,7 @@ public Response head( .header("Content-Length", key.getDataSize()) .header("Content-Type", "binary/octet-stream"); addLastModifiedDate(response, key); + addCustomMetadataHeaders(response, key); getMetrics().incHeadKeySuccess(); AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction, getAuditParameters())); @@ -882,10 +886,12 @@ public void setContext(ContainerRequestContext context) { void copy(OzoneVolume volume, InputStream src, long srcKeyLen, String destKey, String destBucket, - ReplicationConfig replication) throws IOException { - try (OzoneOutputStream dest = getClientProtocol().createKey( + ReplicationConfig replication, + Map metadata) throws IOException { + try (OzoneOutputStream dest = + getClientProtocol().createKey( volume.getName(), destBucket, destKey, srcKeyLen, - replication, new HashMap<>())) { + replication, metadata)) { IOUtils.copy(src, dest); } } @@ -936,7 +942,8 @@ private CopyObjectResponse copyObject(OzoneVolume volume, try (OzoneInputStream src = getClientProtocol().getKey(volume.getName(), sourceBucket, sourceKey)) { - copy(volume, src, sourceKeyLen, destkey, destBucket, replicationConfig); + copy(volume, src, sourceKeyLen, destkey, destBucket, replicationConfig, + sourceKeyDetails.getMetadata()); } final OzoneKeyDetails destKeyDetails = getClientProtocol().getKeyDetails( diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index f891e13d779..f620fd624c0 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -62,4 +62,6 @@ private S3Consts() { public static final String S3_XML_NAMESPACE = "http://s3.amazonaws" + ".com/doc/2006-03-01/"; + public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-"; + } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java new file mode 100644 index 00000000000..9c1f254b798 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + * Tests the s3 EndpointBase class methods. + */ +package org.apache.hadoop.ozone.s3.endpoint; + +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.junit.Assert; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +import javax.ws.rs.core.MultivaluedHashMap; +import javax.ws.rs.core.MultivaluedMap; + +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; + +/** + * Test methods of the EndpointBase. + */ +public class TestEndpointBase { + + /** + * Verify s3 metadata key "gdprEnabled" can't be set up directly + * from the normal client's request, + * it should be decided on the server side. + */ + @Test + public void testFilterGDPRFromCustomMetadataHeaders() + throws OS3Exception { + MultivaluedMap s3requestHeaders + = new MultivaluedHashMap<>(); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + "custom-key1", "custom-value1"); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + "custom-key2", "custom-value2"); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + OzoneConsts.GDPR_FLAG, "true"); + + EndpointBase endpointBase = new EndpointBase() { + @Override + public void init() { } + }; + + Map filteredCustomMetadata = + endpointBase.getCustomMetadataFromHeaders(s3requestHeaders); + Assert.assertTrue(filteredCustomMetadata.containsKey("custom-key1")); + Assert.assertEquals( + "custom-value1", filteredCustomMetadata.get("custom-key1")); + Assert.assertTrue(filteredCustomMetadata.containsKey("custom-key2")); + Assert.assertEquals( + "custom-value2", filteredCustomMetadata.get("custom-key2")); + Assert.assertFalse( + filteredCustomMetadata.containsKey(OzoneConsts.GDPR_FLAG)); + } + + /** + * Verify s3 request metadata size should be smaller than 2 KB. + */ + @Test + public void testCustomMetadataHeadersSizeOverbig() { + MultivaluedMap s3requestHeaders + = new MultivaluedHashMap<>(); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + "custom-key1", "custom-value1"); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + "custom-key2", "custom-value2"); + s3requestHeaders.add( + CUSTOM_METADATA_HEADER_PREFIX + "custom-key3", + new String(new byte[3000], StandardCharsets.UTF_8)); + + EndpointBase endpointBase = new EndpointBase() { + @Override + public void init() { } + }; + + Exception exception = Assertions.assertThrows( + IllegalArgumentException.class, + () -> endpointBase.getCustomMetadataFromHeaders(s3requestHeaders)); + Assert.assertEquals( + "Illegal user defined metadata. Combined size cannot exceed 2KB.", + exception.getMessage()); + } + +}