HDDS-6440. Handle custom metadata (x-amz-meta) during put-object through S3 API.

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -391,6 +391,8 @@ private OzoneConsts() {
 
   public static final int S3_SECRET_KEY_MIN_LENGTH = 8;
 
+  public static final int S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB = 2;
+
   //GDPR
   public static final String GDPR_FLAG = "gdprEnabled";
   public static final String GDPR_ALGORITHM_NAME = "AES";

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java

@@ -24,6 +24,8 @@
 import org.apache.hadoop.hdds.client.ReplicationType;
 
 import java.time.Instant;
+import java.util.Map;
+import java.util.HashMap;
 
 /**
  * A class that encapsulates OzoneKey.
@@ -57,6 +59,7 @@ public class OzoneKey {
 
   private ReplicationConfig replicationConfig;
 
+  private Map<String, String> metadata = new HashMap<>();
   /**
    * Constructs OzoneKey from OmKeyInfo.
    *
@@ -67,14 +70,9 @@ public OzoneKey(String volumeName, String bucketName,
                   String keyName, long size, long creationTime,
                   long modificationTime, ReplicationType type,
                   int replicationFactor) {
-    this.volumeName = volumeName;
-    this.bucketName = bucketName;
-    this.name = keyName;
-    this.dataSize = size;
-    this.creationTime = Instant.ofEpochMilli(creationTime);
-    this.modificationTime = Instant.ofEpochMilli(modificationTime);
-    this.replicationConfig = ReplicationConfig.fromTypeAndFactor(type,
-            ReplicationFactor.valueOf(replicationFactor));
+    this(volumeName, bucketName, keyName, size, creationTime, modificationTime,
+            ReplicationConfig.fromTypeAndFactor(type,
+                    ReplicationFactor.valueOf(replicationFactor)));
   }
 
   /**
@@ -94,6 +92,16 @@ public OzoneKey(String volumeName, String bucketName,
     this.replicationConfig = replicationConfig;
   }
 
+  @SuppressWarnings("parameternumber")
+  public OzoneKey(String volumeName, String bucketName,
+                  String keyName, long size, long creationTime,
+                  long modificationTime, ReplicationConfig replicationConfig,
+                  Map<String, String> metadata) {
+    this(volumeName, bucketName, keyName, size, creationTime,
+        modificationTime, replicationConfig);
+    this.metadata.putAll(metadata);
+  }
+
   /**
    * Returns Volume Name associated with the Key.
    *
@@ -154,6 +162,10 @@ public Instant getModificationTime() {
    * @return replicationType
    */
 
+  public Map<String, String> getMetadata() {
+    return metadata;
+  }
+
   @Deprecated
   @JsonIgnore
   public ReplicationType getReplicationType() {

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java

@@ -39,8 +39,6 @@ public class OzoneKeyDetails extends OzoneKey {
    */
   private List<OzoneKeyLocation> ozoneKeyLocations;
 
-  private Map<String, String> metadata;
-
   private FileEncryptionInfo feInfo;
 
   private SupplierWithIOException<OzoneInputStream> contentSupplier;
@@ -58,8 +56,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
     super(volumeName, bucketName, keyName, size, creationTime,
         modificationTime, type, replicationFactor);
     this.ozoneKeyLocations = ozoneKeyLocations;
-    this.metadata = metadata;
     this.feInfo = feInfo;
+    this.getMetadata().putAll(metadata);
   }
 
 
@@ -75,9 +73,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
       FileEncryptionInfo feInfo,
       SupplierWithIOException<OzoneInputStream> contentSupplier) {
     super(volumeName, bucketName, keyName, size, creationTime,
-            modificationTime, replicationConfig);
+            modificationTime, replicationConfig, metadata);
     this.ozoneKeyLocations = ozoneKeyLocations;
-    this.metadata = metadata;
     this.feInfo = feInfo;
     this.contentSupplier = contentSupplier;
   }
@@ -89,10 +86,6 @@ public List<OzoneKeyLocation> getOzoneKeyLocations() {
     return ozoneKeyLocations;
   }
 
-  public Map<String, String> getMetadata() {
-    return metadata;
-  }
-
   public FileEncryptionInfo getFileEncryptionInfo() {
     return feInfo;
   }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -2008,7 +2008,8 @@ public OzoneKey headObject(String volumeName, String bucketName,
 
     return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(),
         keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(),
-        keyInfo.getModificationTime(), keyInfo.getReplicationConfig());
+        keyInfo.getModificationTime(), keyInfo.getReplicationConfig(),
+        keyInfo.getMetadata());
 
   }
 

hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot

@@ -36,6 +36,11 @@ Execute AWSS3APICli and checkrc
     ${output} =       Execute and checkrc        aws s3api --endpoint-url ${ENDPOINT_URL} ${command}  ${expected_error_code}
     [return]          ${output}
 
+Execute AWSS3APICli and ignore error
+    [Arguments]       ${command}
+    ${output} =       Execute And Ignore Error   aws s3api --endpoint-url ${ENDPOINT_URL} ${command}
+    [return]          ${output}
+
 Execute AWSS3Cli
     [Arguments]       ${command}
     ${output} =       Execute                     aws s3 --endpoint-url ${ENDPOINT_URL} ${command}

hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot

@@ -159,3 +159,29 @@ Zero byte file
 
     ${result} =                 Execute AWSS3APICli and checkrc        get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result   255
                                 Should contain             ${result}        InvalidRange
+
+Create file with user defined metadata
+                                Execute                   echo "Randomtext" > /tmp/testfile2
+                                Execute AWSS3ApiCli       put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 --body /tmp/testfile2 --metadata="custom-key1=custom-value1,custom-key2=custom-value2"
+
+    ${result} =                 Execute AWSS3APICli       head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1
+                                Should contain            ${result}    \"custom-key1\": \"custom-value1\"
+                                Should contain            ${result}    \"custom-key2\": \"custom-value2\"
+
+    ${result} =                 Execute                   ozone sh key info /s3v/${BUCKET}/${PREFIX}/putobject/custom-metadata/key1
+                                Should contain            ${result}   \"custom-key1\" : \"custom-value1\"
+                                Should contain            ${result}   \"custom-key2\" : \"custom-value2\"
+
+Create file with user defined metadata with gdpr enabled value in request
+                                Execute                    echo "Randomtext" > /tmp/testfile2
+                                Execute AWSS3ApiCli        put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="gdprEnabled=true,custom-key2=custom-value2"
+    ${result} =                 Execute AWSS3ApiCli        head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2
+                                Should contain             ${result}   \"custom-key2\": \"custom-value2\"
+                                Should not contain         ${result}   \"gdprEnabled\": \"true\"
+
+
+Create file with user defined metadata size larger than 2 KB
+                                Execute                    echo "Randomtext" > /tmp/testfile2
+    ${custom_metadata_value} =  Execute                    printf 'v%.0s' {1..3000}
+    ${result} =                 Execute AWSS3APICli and ignore error        put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key2 --body /tmp/testfile2 --metadata="custom-key1=${custom_metadata_value}"
+                                Should not contain                          ${result}   custom-key1: ${custom_metadata_value}

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java

@@ -19,14 +19,24 @@
 
 import javax.annotation.PostConstruct;
 import javax.inject.Inject;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
 import javax.ws.rs.container.ContainerRequestContext;
 import javax.ws.rs.core.Context;
 import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Arrays;
 import java.util.Map;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Collections;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.audit.AuditAction;
 import org.apache.hadoop.ozone.audit.AuditEventStatus;
 import org.apache.hadoop.ozone.audit.AuditLogger;
@@ -35,6 +45,7 @@
 import org.apache.hadoop.ozone.audit.Auditor;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneKey;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -44,12 +55,16 @@
 import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
 
 import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
 import org.apache.hadoop.ozone.s3.util.AuditUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.ozone.OzoneConsts.KB;
 import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
 
 /**
  * Basic helpers for all the REST endpoints.
@@ -63,6 +78,8 @@ public abstract class EndpointBase implements Auditor {
   @Context
   private ContainerRequestContext context;
 
+  private Set<String> excludeMetadataFields =
+          new HashSet<>(Arrays.asList(OzoneConsts.GDPR_FLAG));
   private static final Logger LOG =
       LoggerFactory.getLogger(EndpointBase.class);
 
@@ -240,6 +257,57 @@ private Iterator<? extends OzoneBucket> iterateBuckets(
     }
   }
 
+  protected Map<String, String> getCustomMetadataFromHeaders(
+      MultivaluedMap<String, String> requestHeaders) throws OS3Exception {
+    Map<String, String> customMetadata = new HashMap<>();
+    if (requestHeaders == null || requestHeaders.isEmpty()) {
+      return customMetadata;
+    }
+
+    Set<String> customMetadataKeys = requestHeaders.keySet().stream()
+            .filter(k -> {
+              if (k.startsWith(CUSTOM_METADATA_HEADER_PREFIX) &&
+                      !excludeMetadataFields.contains(
+                        k.substring(
+                          CUSTOM_METADATA_HEADER_PREFIX.length()))) {
+                return true;
+              }
+              return false;
+            })
+            .collect(Collectors.toSet());
+
+    long sizeInBytes = 0;
+    if (!customMetadataKeys.isEmpty()) {
+      for (String key : customMetadataKeys) {
+        String mapKey =
+            key.substring(CUSTOM_METADATA_HEADER_PREFIX.length());
+        List<String> values = requestHeaders.get(key);
+        String value = StringUtils.join(values, ",");
+        sizeInBytes += mapKey.getBytes(UTF_8).length;
+        sizeInBytes += value.getBytes(UTF_8).length;
+
+        if (sizeInBytes >
+                OzoneConsts.S3_REQUEST_HEADER_METADATA_SIZE_LIMIT_KB * KB) {
+          throw new IllegalArgumentException("Illegal user defined metadata." +
+              " Combined size cannot exceed 2KB.");
+        }
+        customMetadata.put(mapKey, value);
+      }
+    }
+    return customMetadata;
+  }
+
+  protected void addCustomMetadataHeaders(
+      Response.ResponseBuilder responseBuilder, OzoneKey key) {
+
+    Map<String, String> metadata = key.getMetadata();
+    for (Map.Entry<String, String> entry : metadata.entrySet()) {
+      responseBuilder
+          .header(CUSTOM_METADATA_HEADER_PREFIX + entry.getKey(),
+              entry.getValue());
+    }
+  }
+
   private AuditMessage.Builder auditMessageBaseBuilder(AuditAction op,
       Map<String, String> auditMap) {
     AuditMessage.Builder builder = new AuditMessage.Builder()

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java

@@ -48,10 +48,9 @@
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
-import java.util.HashMap;
+import java.util.Map;
 import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.OptionalLong;
 
 import org.apache.commons.lang3.StringUtils;
@@ -210,13 +209,17 @@ public Response put(
             "Connection", "close").build();
       }
 
+      // Normal put object
+      Map<String, String> customMetadata =
+          getCustomMetadataFromHeaders(headers.getRequestHeaders());
+
       if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
           .equals(headers.getHeaderString("x-amz-content-sha256"))) {
         body = new SignedChunksInputStream(body);
       }
 
       output = getClientProtocol().createKey(volume.getName(), bucketName,
-          keyPath, length, replicationConfig, new HashMap<>());
+          keyPath, length, replicationConfig, customMetadata);
       IOUtils.copy(body, output);
 
       getMetrics().incCreateKeySuccess();
@@ -470,6 +473,7 @@ public Response head(
         .header("Content-Length", key.getDataSize())
         .header("Content-Type", "binary/octet-stream");
     addLastModifiedDate(response, key);
+    addCustomMetadataHeaders(response, key);
     getMetrics().incHeadKeySuccess();
     AUDIT.logReadSuccess(buildAuditMessageForSuccess(s3GAction,
         getAuditParameters()));
@@ -882,10 +886,12 @@ public void setContext(ContainerRequestContext context) {
 
   void copy(OzoneVolume volume, InputStream src, long srcKeyLen,
       String destKey, String destBucket,
-      ReplicationConfig replication) throws IOException {
-    try (OzoneOutputStream dest = getClientProtocol().createKey(
+      ReplicationConfig replication,
+            Map<String, String> metadata) throws IOException {
+    try (OzoneOutputStream dest =
+                 getClientProtocol().createKey(
         volume.getName(), destBucket, destKey, srcKeyLen,
-        replication, new HashMap<>())) {
+        replication, metadata)) {
       IOUtils.copy(src, dest);
     }
   }
@@ -936,7 +942,8 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
 
       try (OzoneInputStream src = getClientProtocol().getKey(volume.getName(),
           sourceBucket, sourceKey)) {
-        copy(volume, src, sourceKeyLen, destkey, destBucket, replicationConfig);
+        copy(volume, src, sourceKeyLen, destkey, destBucket, replicationConfig,
+                sourceKeyDetails.getMetadata());
       }
 
       final OzoneKeyDetails destKeyDetails = getClientProtocol().getKeyDetails(

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java

@@ -62,4 +62,6 @@ private S3Consts() {
   public static final String S3_XML_NAMESPACE = "http://s3.amazonaws" +
       ".com/doc/2006-03-01/";
 
+  public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-";
+
 }