HDDS-10979. Support STANDARD_IA S3 storage class to accept EC replication config.

hadoop-ozone/dist/src/main/compose/common/ec-test.sh

@@ -18,7 +18,7 @@
 start_docker_env 5
 
 ## Exclude virtual-host tests. This is tested separately as it requires additional config.
-execute_robot_test scm -v BUCKET:erasure --exclude virtual-host s3
+execute_robot_test scm -v BUCKET:erasure --exclude virtual-host --exclude ec-storage-class s3
 
 execute_robot_test scm ec/rewrite.robot
 
@@ -30,3 +30,5 @@ docker-compose up -d --no-recreate --scale datanode=3
 execute_robot_test scm -v PREFIX:${prefix} -N read-3-datanodes ec/read.robot
 docker-compose up -d --no-recreate --scale datanode=5
 execute_robot_test scm -v container:1 -v count:5 -N EC-recovery replication/wait.robot
+docker-compose up -d --no-recreate --scale datanode=9
+execute_robot_test scm s3/awss3ecstorage.robot

hadoop-ozone/dist/src/main/compose/ozone-ha/test-haproxy-s3g.sh

@@ -31,10 +31,10 @@ source "$COMPOSE_DIR/../testlib.sh"
 start_docker_env
 
 ## Exclude virtual-host tests. This is tested separately as it requires additional config.
-exclude="--exclude virtual-host"
+exclude="--exclude virtual-host --exclude ec-storage-class"
 for bucket in generated; do
   execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
   # some tests are independent of the bucket type, only need to be run once
   ## Exclude awss3virtualhost.robot
-  exclude="--exclude virtual-host --exclude no-bucket-type"
+  exclude="--exclude virtual-host --exclude ec-storage-class --exclude no-bucket-type"
 done

hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh

@@ -38,12 +38,12 @@ execute_robot_test ${SCM} basic/links.robot
 execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link ozonefs/ozonefs.robot
 
 ## Exclude virtual-host tests. This is tested separately as it requires additional config.
-exclude="--exclude virtual-host"
+exclude="--exclude virtual-host --exclude ec-storage-class"
 for bucket in generated; do
   for layout in OBJECT_STORE LEGACY FILE_SYSTEM_OPTIMIZED; do
     execute_robot_test ${SCM} -v BUCKET:${bucket} -v BUCKET_LAYOUT:${layout} -N s3-${layout}-${bucket} ${exclude} s3
     # some tests are independent of the bucket type, only need to be run once
-    exclude="--exclude virtual-host --exclude no-bucket-type"
+    exclude="--exclude virtual-host --exclude no-bucket-type --exclude ec-storage-class"
   done
 done
 

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-haproxy-s3g.sh

@@ -36,10 +36,10 @@ start_docker_env
 execute_command_in_container kms hadoop key create ${OZONE_BUCKET_KEY_NAME}
 
 ## Exclude virtual-host tests. This is tested separately as it requires additional config.
-exclude="--exclude virtual-host"
+exclude="--exclude virtual-host --exclude ec-storage-class"
 for bucket in encrypted; do
   execute_robot_test recon -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
   # some tests are independent of the bucket type, only need to be run once
   ## Exclude virtual-host.robot
-  exclude="--exclude virtual-host --exclude no-bucket-type"
+  exclude="--exclude virtual-host --exclude ec-storage-class --exclude no-bucket-type"
 done

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh

@@ -44,10 +44,10 @@ execute_robot_test s3g -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-o3fs-link o
 execute_robot_test s3g basic/links.robot
 
 ## Exclude virtual-host tests. This is tested separately as it requires additional config.
-exclude="--exclude virtual-host"
+exclude="--exclude virtual-host --exclude ec-storage-class"
 for bucket in link; do
   execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
   # some tests are independent of the bucket type, only need to be run once
   ## Exclude virtual-host.robot
-  exclude="--exclude virtual-host --exclude no-bucket-type"
+  exclude="--exclude virtual-host --exclude ec-storage-class --exclude no-bucket-type"
 done

hadoop-ozone/dist/src/main/compose/ozonesecure/test-vault.sh

@@ -31,4 +31,4 @@ export COMPOSE_FILE=docker-compose.yaml:vault.yaml
 start_docker_env
 
 ## Exclude virtual-host tests. This is tested separately as it requires additional config.
-execute_robot_test scm --exclude virtual-host s3
+execute_robot_test scm --exclude virtual-host --exclude ec-storage-class s3

hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh

@@ -43,12 +43,12 @@ execute_robot_test scm repair/bucket-encryption.robot
 execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-ofs-bucket ozonefs/ozonefs.robot
 
 ## Exclude virtual-host tests. This is tested separately as it requires additional config.
-exclude="--exclude virtual-host"
+exclude="--exclude virtual-host --exclude ec-storage-class"
 for bucket in encrypted; do
   execute_robot_test s3g -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
   # some tests are independent of the bucket type, only need to be run once
   ## Exclude virtual-host.robot
-  exclude="--exclude virtual-host --exclude no-bucket-type"
+  exclude="--exclude virtual-host --exclude ec-storage-class --exclude no-bucket-type"
 done
 
 #expects 4 pipelines, should be run before

hadoop-ozone/dist/src/main/smoketest/s3/awss3ecstorage.robot

@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       S3 gateway test with aws cli with STANDARD_IA storage class
+Library             OperatingSystem
+Library             String
+Resource            ../commonlib.robot
+Resource            commonawslib.robot
+Resource            mpu_lib.robot
+Test Timeout        5 minutes
+Suite Setup         Setup EC Multipart Tests
+Suite Teardown      Teardown EC Multipart Tests
+Test Setup          Generate random prefix
+Default Tags        ec-storage-class
+
+*** Keywords ***
+Setup EC Multipart Tests
+    Setup s3 tests
+    Create Random File KB    1023    /tmp/1mb
+
+Teardown EC Multipart Tests
+    Remove Files    /tmp/1mb
+
+*** Variables ***
+${ENDPOINT_URL}       http://s3g:9878
+${BUCKET}             generated
+
+*** Test Cases ***
+
+Put Object with STANDARD_IA storage class
+    ${file_checksum} =  Execute                    md5sum /tmp/1mb | awk '{print $1}'
+
+    ${result} =         Execute AWSS3ApiCli        put-object --bucket ${BUCKET} --key ${PREFIX}/ecKey32 --body /tmp/1mb --storage-class STANDARD_IA
+    ${eTag} =           Execute and checkrc        echo '${result}' | jq -r '.ETag'  0
+                        Should Be Equal            ${eTag}           \"${file_checksum}\"
+
+    ${result} =         Execute AWSS3ApiCli        put-object --bucket ${BUCKET} --key ${PREFIX}/ecKey63 --body /tmp/1mb --storage-class STANDARD_IA --metadata="storage-config=rs-6-3-1024k"
+    ${eTag} =           Execute and checkrc        echo '${result}' | jq -r '.ETag'  0
+                        Should Be Equal            ${eTag}           \"${file_checksum}\"
+
+Test multipart upload with STANDARD_IA storage
+    ${uploadID} =       Initiate MPU    ${BUCKET}    ${PREFIX}/ecmultipartKey32     0     --storage-class STANDARD_IA
+    ${eTag1} =          Upload MPU part    ${BUCKET}    ${PREFIX}/ecmultipartKey32    ${uploadID}    1    /tmp/1mb
+    ${result} =         Execute AWSS3APICli   list-parts --bucket ${BUCKET} --key ${PREFIX}/ecmultipartKey32 --upload-id ${uploadID}
+    ${part1} =          Execute and checkrc    echo '${result}' | jq -r '.Parts[0].ETag'  0
+                        Should Be equal       ${part1}    ${eTag1}
+                        Should contain        ${result}    STANDARD_IA
+    ${result} =         Abort MPU    ${BUCKET}    ${PREFIX}/ecmultipartKey32    ${uploadID}    0
+
+    ${uploadID} =       Initiate MPU    ${BUCKET}    ${PREFIX}/ecmultipartKey63     0     --storage-class STANDARD_IA --metadata="storage-config=rs-6-3-1024k"
+    ${eTag1} =          Upload MPU part    ${BUCKET}    ${PREFIX}/ecmultipartKey63    ${uploadID}    1    /tmp/part1
+    ${result} =         Execute AWSS3APICli   list-parts --bucket ${BUCKET} --key ${PREFIX}/ecmultipartKey63 --upload-id ${uploadID}
+    ${part1} =          Execute and checkrc    echo '${result}' | jq -r '.Parts[0].ETag'  0
+                        Should Be equal       ${part1}    ${eTag1}
+                        Should contain        ${result}    STANDARD_IA
+    ${result} =         Abort MPU    ${BUCKET}    ${PREFIX}/ecmultipartKey63    ${uploadID}    0
+
+Copy Object change storage class to STANDARD_IA
+    ${file_checksum} =  Execute                    md5sum /tmp/1mb | awk '{print $1}'
+    ${result} =         Execute AWSS3ApiCli        put-object --bucket ${BUCKET} --key ${PREFIX}/copyobject/Key1 --body /tmp/1mb
+    ${eTag} =           Execute and checkrc        echo '${result}' | jq -r '.ETag'  0
+                        Should Be Equal            ${eTag}           \"${file_checksum}\"
+
+     ${result} =         Execute AWSS3APICli        copy-object --storage-class STANDARD_IA --bucket ${BUCKET} --key ${PREFIX}/copyobject/Key1 --copy-source ${BUCKET}/${PREFIX}/copyobject/Key1
+                         Should contain             ${result}        ETag
+     ${eTag} =           Execute and checkrc        echo '${result}' | jq -r '.CopyObjectResult.ETag'  0
+                         Should Be Equal            ${eTag}           \"${file_checksum}\"
+
+     ${result} =         Execute AWSS3APICli        copy-object --storage-class STANDARD_IA --metadata="storage-config=rs-6-3-1024k" --bucket ${BUCKET} --key ${PREFIX}/copyobject/Key1 --copy-source ${BUCKET}/${PREFIX}/copyobject/Key1
+                         Should contain             ${result}        ETag
+     ${eTag} =           Execute and checkrc        echo '${result}' | jq -r '.CopyObjectResult.ETag'  0
+                         Should Be Equal            ${eTag}           \"${file_checksum}\"

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java

@@ -24,6 +24,7 @@
 import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.AWS_TAG_PREFIX;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CONFIG_HEADER;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_KEY_LENGTH_LIMIT;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_NUM_LIMIT;
@@ -97,7 +98,7 @@ public abstract class EndpointBase implements Auditor {
   private ContainerRequestContext context;
 
   private Set<String> excludeMetadataFields =
-      new HashSet<>(Arrays.asList(OzoneConsts.GDPR_FLAG));
+      new HashSet<>(Arrays.asList(OzoneConsts.GDPR_FLAG, STORAGE_CONFIG_HEADER));
   private static final Logger LOG =
       LoggerFactory.getLogger(EndpointBase.class);
 

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java

@@ -49,12 +49,14 @@
 import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_IF_MODIFIED_SINCE;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_IF_UNMODIFIED_SINCE;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_COPY_DIRECTIVE_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.CopyDirective;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.DECODED_CONTENT_LENGTH_HEADER;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.MP_PARTS_COUNT;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER_SUPPORTED_UNIT;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CONFIG_HEADER;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_COUNT_HEADER;
 import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_DIRECTIVE_HEADER;
 import static org.apache.hadoop.ozone.s3.util.S3Utils.urlDecode;
@@ -230,7 +232,7 @@ public Response put(
     boolean auditSuccess = true;
     PerformanceStringBuilder perf = new PerformanceStringBuilder();
 
-    String copyHeader = null, storageType = null;
+    String copyHeader = null, storageType = null, storageConfig = null;
     DigestInputStream digestInputStream = null;
     try {
       if (aclMarker != null) {
@@ -256,12 +258,13 @@ public Response put(
 
       copyHeader = headers.getHeaderString(COPY_SOURCE_HEADER);
       storageType = headers.getHeaderString(STORAGE_CLASS_HEADER);
+      storageConfig = headers.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + STORAGE_CONFIG_HEADER);
       boolean storageTypeDefault = StringUtils.isEmpty(storageType);
 
       // Normal put object
       OzoneBucket bucket = volume.getBucket(bucketName);
       ReplicationConfig replicationConfig =
-          getReplicationConfig(bucket, storageType);
+          getReplicationConfig(bucket, storageType, storageConfig);
 
       boolean enableEC = false;
       if ((replicationConfig != null &&
@@ -817,14 +820,15 @@ public Response initializeMultipartUpload(
     try {
       OzoneBucket ozoneBucket = getBucket(bucket);
       String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER);
+      String storageConfig = headers.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + STORAGE_CONFIG_HEADER);
 
       Map<String, String> customMetadata =
           getCustomMetadataFromHeaders(headers.getRequestHeaders());
 
       Map<String, String> tags = getTaggingFromHeaders(headers);
 
       ReplicationConfig replicationConfig =
-          getReplicationConfig(ozoneBucket, storageType);
+          getReplicationConfig(ozoneBucket, storageType, storageConfig);
 
       OmMultipartInfo multipartInfo =
           ozoneBucket.initiateMultipartUpload(key, replicationConfig, customMetadata, tags);
@@ -857,21 +861,20 @@ public Response initializeMultipartUpload(
   }
 
   private ReplicationConfig getReplicationConfig(OzoneBucket ozoneBucket,
-      String storageType) throws OS3Exception {
-    if (StringUtils.isEmpty(storageType)) {
-      S3StorageType defaultStorageType = S3StorageType.getDefault(ozoneConfiguration);
-      storageType = (defaultStorageType != null ? defaultStorageType.toString() : null);
-    }
+      String storageType, String storageConfig) throws OS3Exception {
 
     ReplicationConfig clientConfiguredReplicationConfig = null;
     String replication = ozoneConfiguration.get(OZONE_REPLICATION);
+
     if (replication != null) {
-      clientConfiguredReplicationConfig = ReplicationConfig.parse(
-          ReplicationType.valueOf(ozoneConfiguration
-              .get(OZONE_REPLICATION_TYPE, OZONE_REPLICATION_TYPE_DEFAULT)),
-          replication, ozoneConfiguration);
+      ReplicationType replicationType = ReplicationType.valueOf(
+          ozoneConfiguration.get(OZONE_REPLICATION_TYPE, OZONE_REPLICATION_TYPE_DEFAULT));
+      clientConfiguredReplicationConfig =
+          (replicationType == ReplicationType.EC) ?
+              new ECReplicationConfig(replication) : ReplicationConfig.parse(
+              replicationType, replication, ozoneConfiguration);
     }
-    return S3Utils.resolveS3ClientSideReplicationConfig(storageType,
+    return S3Utils.resolveS3ClientSideReplicationConfig(storageType, storageConfig,
         clientConfiguredReplicationConfig, ozoneBucket.getReplicationConfig());
   }
 
@@ -974,9 +977,10 @@ private Response createMultipartKey(OzoneVolume volume, String bucket,
 
       copyHeader = headers.getHeaderString(COPY_SOURCE_HEADER);
       String storageType = headers.getHeaderString(STORAGE_CLASS_HEADER);
+      String storageConfig = headers.getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + STORAGE_CONFIG_HEADER);
       final OzoneBucket ozoneBucket = volume.getBucket(bucket);
       ReplicationConfig replicationConfig =
-          getReplicationConfig(ozoneBucket, storageType);
+          getReplicationConfig(ozoneBucket, storageType, storageConfig);
 
       boolean enableEC = false;
       if ((replicationConfig != null &&

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java

@@ -150,6 +150,12 @@ public final class S3ErrorTable {
       HTTP_FORBIDDEN
   );
 
+  public static final OS3Exception INVALID_STORAGE_CLASS = new OS3Exception(
+      "InvalidStorageClass", "The storage class that you specified is not valid. " +
+      "Provide a supported storage class[STANDARD|REDUCED_REDUNDANCY|STANDARD_IA] or " +
+      "a valid custom EC storage config for if using STANDARD_IA.",
+      HTTP_BAD_REQUEST);
+
   private static Function<Exception, OS3Exception> generateInternalError =
       e -> new OS3Exception("InternalError", e.getMessage(), HTTP_INTERNAL_ERROR);
 

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java

@@ -69,6 +69,7 @@ public final class S3Consts {
   // Constants related to custom metadata
   public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-";
   public static final String CUSTOM_METADATA_COPY_DIRECTIVE_HEADER = "x-amz-metadata-directive";
+  public static final String STORAGE_CONFIG_HEADER = "storage-config";
 
   public static final String DECODED_CONTENT_LENGTH_HEADER =
       "x-amz-decoded-content-length";

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3StorageType.java

@@ -21,27 +21,29 @@
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
 
 /**
  * Maps S3 storage class values to Ozone replication values.
  */
 
 public enum S3StorageType {
 
-  REDUCED_REDUNDANCY(ReplicationType.RATIS, ReplicationFactor.ONE),
-  STANDARD(ReplicationType.RATIS, ReplicationFactor.THREE);
+  REDUCED_REDUNDANCY(ReplicationType.RATIS, ReplicationFactor.ONE, null),
+  STANDARD(ReplicationType.RATIS, ReplicationFactor.THREE, null),
+  STANDARD_IA(ReplicationType.EC, null, ECReplicationConfig.EcCodec.RS + "-3-2-1024k");
 
   private final ReplicationType type;
   private final ReplicationFactor factor;
+  private final String ecReplicationString;
 
   S3StorageType(
       ReplicationType type,
-      ReplicationFactor factor) {
+      ReplicationFactor factor,
+      String ecReplicationString) {
     this.type = type;
     this.factor = factor;
+    this.ecReplicationString = ecReplicationString;
   }
 
   public ReplicationFactor getFactor() {
@@ -52,33 +54,13 @@ public ReplicationType getType() {
     return type;
   }
 
-  /**
-   * Get default S3StorageType for a new key to be uploaded.
-   * This should align to the ozone cluster configuration.
-   * @param config OzoneConfiguration
-   * @return S3StorageType which wraps ozone replication type and factor
-   */
-  public static S3StorageType getDefault(ConfigurationSource config) {
-    String replicationString = config.get(OzoneConfigKeys.OZONE_REPLICATION);
-    ReplicationFactor configFactor;
-    if (replicationString == null) {
-      // if no config is set then let server take decision
-      return null;
-    }
-    try {
-      configFactor = ReplicationFactor.valueOf(
-          Integer.parseInt(replicationString));
-    } catch (NumberFormatException ex) {
-      // conservatively defaults to STANDARD on wrong config value
-      return STANDARD;
-    }
-    return configFactor == ReplicationFactor.ONE
-        ? REDUCED_REDUNDANCY : STANDARD;
+  public String getEcReplicationString() {
+    return ecReplicationString;
   }
 
   public static S3StorageType fromReplicationConfig(ReplicationConfig config) {
-    if (config instanceof ECReplicationConfig) {
-      return S3StorageType.STANDARD;
+    if (config.getReplicationType() == HddsProtos.ReplicationType.EC) {
+      return STANDARD_IA;
     }
     if (config.getReplicationType() == HddsProtos.ReplicationType.STAND_ALONE ||
         config.getRequiredNodes() == 1) {