HDDS-6440. Handle custom metadata (x-amz-meta) during put-object through S3 API.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java

@@ -23,6 +23,8 @@
 import org.apache.hadoop.hdds.client.ReplicationType;
 
 import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * A class that encapsulates OzoneKey.
@@ -56,6 +58,8 @@ public class OzoneKey {
 
   private ReplicationConfig replicationConfig;
 
+  private Map<String, String> metadata = new HashMap<>();
+
   /**
    * Constructs OzoneKey from OmKeyInfo.
    *
@@ -93,6 +97,16 @@ public OzoneKey(String volumeName, String bucketName,
     this.replicationConfig = replicationConfig;
   }
 
+  @SuppressWarnings("parameternumber")
+  public OzoneKey(String volumeName, String bucketName,
+                  String keyName, long size, long creationTime,
+                  long modificationTime, ReplicationConfig replicationConfig,
+                  Map<String, String> metadata) {
+    this(volumeName, bucketName, keyName, size, creationTime,
+        modificationTime, replicationConfig);
+    this.metadata.putAll(metadata);
+  }
+
   /**
    * Returns Volume Name associated with the Key.
    *
@@ -153,6 +167,10 @@ public Instant getModificationTime() {
    * @return replicationType
    */
 
+  public Map<String, String> getMetadata() {
+    return metadata;
+  }
+
   @Deprecated
   public ReplicationType getReplicationType() {
     return ReplicationType

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java

@@ -35,8 +35,6 @@ public class OzoneKeyDetails extends OzoneKey {
    */
   private List<OzoneKeyLocation> ozoneKeyLocations;
 
-  private Map<String, String> metadata;
-
   private FileEncryptionInfo feInfo;
 
   /**
@@ -52,8 +50,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
     super(volumeName, bucketName, keyName, size, creationTime,
         modificationTime, type, replicationFactor);
     this.ozoneKeyLocations = ozoneKeyLocations;
-    this.metadata = metadata;
     this.feInfo = feInfo;
+    this.getMetadata().putAll(metadata);
   }
 
 
@@ -68,9 +66,8 @@ public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
       Map<String, String> metadata,
       FileEncryptionInfo feInfo) {
     super(volumeName, bucketName, keyName, size, creationTime,
-            modificationTime, replicationConfig);
+            modificationTime, replicationConfig, metadata);
     this.ozoneKeyLocations = ozoneKeyLocations;
-    this.metadata = metadata;
     this.feInfo = feInfo;
   }
 
@@ -81,10 +78,6 @@ public List<OzoneKeyLocation> getOzoneKeyLocations() {
     return ozoneKeyLocations;
   }
 
-  public Map<String, String> getMetadata() {
-    return metadata;
-  }
-
   public FileEncryptionInfo getFileEncryptionInfo() {
     return feInfo;
   }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -1638,7 +1638,8 @@ public OzoneKey headObject(String volumeName, String bucketName,
 
     return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(),
         keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(),
-        keyInfo.getModificationTime(), keyInfo.getReplicationConfig());
+        keyInfo.getModificationTime(), keyInfo.getReplicationConfig(),
+        keyInfo.getMetadata());
 
   }
 

hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot

@@ -153,3 +153,15 @@ Zero byte file
 
     ${result} =                 Execute AWSS3APICli and checkrc        get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result   255
                                 Should contain             ${result}        InvalidRange
+
+Create file with user defined metadata
+                                Execute                    echo "Randomtext" > /tmp/testfile2
+                                Execute AWSS3ApiCli        put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1 --body /tmp/testfile2 --metadata="custom-key1=custom-value1,custom-key2=custom-value2"
+
+    ${result} =                 Execute AWSS3APICli       head-object --bucket ${BUCKET} --key ${PREFIX}/putobject/custom-metadata/key1
+                                Should contain            ${result}    \"custom-key1\": \"custom-value1\"
+                                Should contain            ${result}    \"custom-key2\": \"custom-value2\"
+
+    ${result} =                 Execute                   ozone sh key info /s3v/${BUCKET}/${PREFIX}/putobject/custom-metadata/key1
+                                Should contain            ${result}   \"custom-key1\" : \"custom-value1\"
+                                Should contain            ${result}   \"custom-key2\" : \"custom-value2\"

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java

@@ -19,14 +19,24 @@
 
 import javax.annotation.PostConstruct;
 import javax.inject.Inject;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneKey;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -35,11 +45,15 @@
 import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
 
 import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.ozone.OzoneConsts.KB;
 import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
 
 /**
  * Basic helpers for all the REST endpoints.
@@ -191,6 +205,46 @@ private Iterator<? extends OzoneBucket> iterateBuckets(
     }
   }
 
+  protected Map<String, String> getCustomMetadataFromHeaders(
+      MultivaluedMap<String, String> requestHeaders) {
+    Map<String, String> customMetadata = new HashMap<>();
+    if (requestHeaders == null || requestHeaders.isEmpty()) {
+      return customMetadata;
+    }
+
+    Set<String> customMetadataKeys = requestHeaders.keySet().stream()
+            .filter(k -> k.startsWith(CUSTOM_METADATA_HEADER_PREFIX))
+            .collect(Collectors.toSet());
+    long sizeInBytes = 0;
+    if (!customMetadataKeys.isEmpty()) {
+      for (String key : customMetadataKeys) {
+        String mapKey =
+            key.substring(CUSTOM_METADATA_HEADER_PREFIX.length());
+        List<String> values = requestHeaders.get(key);
+        String value = StringUtils.join(values, ",");
+        sizeInBytes += mapKey.getBytes(UTF_8).length;
+        sizeInBytes += value.getBytes(UTF_8).length;
+        if (sizeInBytes > 2 * KB) {
+          throw new IllegalArgumentException("Illegal user defined metadata." +
+              " Combined size cannot exceed 2KB.");
+        }
+        customMetadata.put(mapKey, value);
+      }
+    }
+    return customMetadata;
+  }
+
+  protected void addCustomMetadataHeaders(
+      Response.ResponseBuilder responseBuilder, OzoneKey key) {
+
+    Map<String, String> metadata = key.getMetadata();
+    for (Map.Entry<String, String> entry : metadata.entrySet()) {
+      responseBuilder
+          .header(CUSTOM_METADATA_HEADER_PREFIX + entry.getKey(),
+              entry.getValue());
+    }
+  }
+
   @VisibleForTesting
   public void setClient(OzoneClient ozoneClient) {
     this.client = ozoneClient;

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java

@@ -47,7 +47,6 @@
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -197,9 +196,10 @@ public Response put(
 
       // Normal put object
       OzoneBucket bucket = getBucket(bucketName);
-
+      Map<String, String> customMetadata =
+          getCustomMetadataFromHeaders(headers.getRequestHeaders());
       output = bucket.createKey(keyPath, length, replicationType,
-          replicationFactor, new HashMap<>());
+          replicationFactor, customMetadata);
 
       if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
           .equals(headers.getHeaderString("x-amz-content-sha256"))) {
@@ -391,6 +391,7 @@ public Response head(
         .header("Content-Length", key.getDataSize())
         .header("Content-Type", "binary/octet-stream");
     addLastModifiedDate(response, key);
+    addCustomMetadataHeaders(response, key);
     getMetrics().incHeadKeySuccess();
     return response.build();
   }
@@ -800,7 +801,7 @@ private CopyObjectResponse copyObject(String copyHeader,
       sourceInputStream = sourceOzoneBucket.readKey(sourceKey);
 
       destOutputStream = destOzoneBucket.createKey(destkey, sourceKeyLen,
-          replicationType, replicationFactor, new HashMap<>());
+          replicationType, replicationFactor, sourceKeyDetails.getMetadata());
 
       IOUtils.copy(sourceInputStream, destOutputStream);
 

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java

@@ -62,4 +62,6 @@ private S3Consts() {
   public static final String S3_XML_NAMESPACE = "http://s3.amazonaws" +
       ".com/doc/2006-03-01/";
 
+  public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-";
+
 }