HDDS-6184: EC: put command should create EC key if bucket is EC

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java

@@ -34,13 +34,15 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdds.cli.GenericCli;
 import org.apache.hadoop.hdds.cli.OzoneAdmin;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.ozone.OFSPath;
 import org.apache.hadoop.fs.ozone.OzoneFsShell;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl;
 import org.apache.hadoop.ozone.client.ObjectStore;
 import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.io.ECKeyOutputStream;
 import org.apache.hadoop.ozone.client.io.KeyOutputStream;
@@ -910,6 +912,30 @@ public void testCreateBucketWithECReplicationConfig() throws Exception {
     }
   }
 
+  @Test
+  public void testPutKeyOnBucketWithECReplicationConfig() throws Exception {
+    final String volumeName = UUID.randomUUID().toString();
+    final String bucketName = UUID.randomUUID().toString();
+    final String keyName = UUID.randomUUID().toString();
+    getVolume(volumeName);
+    String bucketPath =
+        Path.SEPARATOR + volumeName + Path.SEPARATOR + bucketName;
+    String[] args =
+        new String[] {"bucket", "create", bucketPath, "-t", "EC", "-r",
+            "rs-3-2-1024k"};
+    execute(ozoneShell, args);
+
+    args = new String[] {"key", "put", bucketPath + Path.SEPARATOR + keyName,
+        testFilePathString};
+    execute(ozoneShell, args);
+
+    OzoneKeyDetails key =
+        cluster.getClient().getObjectStore().getVolume(volumeName)
+            .getBucket(bucketName).getKey(keyName);
+    assertEquals(HddsProtos.ReplicationType.EC,
+        key.getReplicationConfig().getReplicationType());
+  }
+
   @Test
   public void testCreateBucketWithRatisReplicationConfig() throws Exception {
     final String volumeName = "volume101";

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneConfigUtil.java

@@ -44,12 +44,18 @@ public static ReplicationConfig resolveReplicationConfigPreference(
     } else {
       // type is NONE, so, let's look for the bucket defaults.
       if (bucketDefaultReplicationConfig != null) {
+        boolean hasECReplicationConfig = bucketDefaultReplicationConfig
+            .getType() == ReplicationType.EC && bucketDefaultReplicationConfig
+            .getEcReplicationConfig() != null;
         // Since Bucket defaults are available, let's inherit
         replicationConfig = ReplicationConfig.fromProto(
             ReplicationType.toProto(bucketDefaultReplicationConfig.getType()),
             ReplicationFactor
                 .toProto(bucketDefaultReplicationConfig.getFactor()),
-            bucketDefaultReplicationConfig.getEcReplicationConfig().toProto());
+            hasECReplicationConfig ?
+                bucketDefaultReplicationConfig.getEcReplicationConfig()
+                    .toProto() :
+                null);
       } else {
         // if bucket defaults also not available, then use server defaults.
         replicationConfig = omDefaultReplicationConfig;

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneConfigUtil.java

@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests the server side replication config preference logic.
+ */
+public class TestOzoneConfigUtil {
+  private ReplicationConfig ratis3ReplicationConfig =
+      new RatisReplicationConfig(HddsProtos.ReplicationFactor.THREE);
+  private HddsProtos.ReplicationType noneType = HddsProtos.ReplicationType.NONE;
+  private HddsProtos.ReplicationFactor zeroFactor =
+      HddsProtos.ReplicationFactor.ZERO;
+  private HddsProtos.ECReplicationConfig clientECReplicationConfig =
+      new ECReplicationConfig("rs-3-2-1024K").toProto();
+  private DefaultReplicationConfig bucketECConfig =
+      new DefaultReplicationConfig(
+          new ECReplicationConfig(clientECReplicationConfig));
+
+  /**
+   * Tests EC bucket defaults.
+   */
+  @Test
+  public void testResolveClientSideRepConfigWhenBucketHasEC() {
+    ReplicationConfig replicationConfig = OzoneConfigUtil
+        .resolveReplicationConfigPreference(noneType, zeroFactor,
+            clientECReplicationConfig, bucketECConfig, ratis3ReplicationConfig);
+    // Client has no preference, so we should bucket defaults as we passed.
+    Assert.assertEquals(bucketECConfig.getEcReplicationConfig(),
+        replicationConfig);
+  }
+
+  /**
+   * Tests server defaults.
+   */
+  @Test
+  public void testResolveClientSideRepConfigWithNoClientAndBucketDefaults() {
+    ReplicationConfig replicationConfig = OzoneConfigUtil
+        .resolveReplicationConfigPreference(noneType, zeroFactor,
+            clientECReplicationConfig, null, ratis3ReplicationConfig);
+    // Client has no preference, no bucket defaults, so it should return server
+    // defaults.
+    Assert.assertEquals(ratis3ReplicationConfig, replicationConfig);
+  }
+
+  /**
+   * Tests client preference of EC.
+   */
+  @Test
+  public void testResolveClientSideRepConfigWhenClientPassEC() {
+    ReplicationConfig replicationConfig = OzoneConfigUtil
+        .resolveReplicationConfigPreference(HddsProtos.ReplicationType.EC,
+            zeroFactor, clientECReplicationConfig, null,
+            ratis3ReplicationConfig);
+    // Client has preference of type EC, no bucket defaults, so it should return
+    // client preference.
+    Assert.assertEquals(new ECReplicationConfig("rs-3-2-1024K"),
+        replicationConfig);
+  }
+
+  /**
+   * Tests bucket ratis defaults.
+   */
+  @Test
+  public void testResolveClientSideRepConfigWhenBucketHasEC3() {
+    DefaultReplicationConfig ratisBucketDefaults =
+        new DefaultReplicationConfig(ReplicationType.RATIS,
+            ReplicationFactor.THREE);
+    ReplicationConfig replicationConfig = OzoneConfigUtil
+        .resolveReplicationConfigPreference(noneType, zeroFactor,
+            clientECReplicationConfig, ratisBucketDefaults,
+            ratis3ReplicationConfig);
+    // Client has no preference of type and bucket has ratis defaults, so it
+    // should return ratis.
+    Assert.assertEquals(ratisBucketDefaults.getType().name(),
+        replicationConfig.getReplicationType().name());
+    Assert.assertEquals(ratisBucketDefaults.getFactor(),
+        ReplicationFactor.valueOf(replicationConfig.getRequiredNodes()));
+  }
+
+}

hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java

@@ -147,4 +147,46 @@ static ReplicationConfig getClientConfiguredReplicationConfig(
         config.get(OZONE_REPLICATION_TYPE, OZONE_REPLICATION_TYPE_DEFAULT)),
         replication, config);
   }
+
+  /**
+   * Gets the client side replication config by checking user passed values vs
+   * client configured values.
+   * @param userPassedType - User provided replication type.
+   * @param userPassedReplication - User provided replication.
+   * @param clientSideConfig - Client side configuration.
+   * @return ReplicationConfig.
+   */
+  public static ReplicationConfig validateAndGetClientReplicationConfig(
+      ReplicationType userPassedType, String userPassedReplication,
+      OzoneConfiguration clientSideConfig) {
+    // Priority 1: User passed replication config values.
+    // Priority 2: Client side configured replication config values.
+    /* if above two are not available, we should just return null and clients
+     can pass null replication config to server. Now server will take the
+     decision of finding the replication config( either from bucket defaults
+     or server defaults). */
+    ReplicationType clientReplicationType = userPassedType;
+    String clientReplication = userPassedReplication;
+    String clientConfiguredDefaultType =
+        clientSideConfig.get(OZONE_REPLICATION_TYPE);
+    if (userPassedType == null && clientConfiguredDefaultType != null) {
+      clientReplicationType =
+          ReplicationType.valueOf(clientConfiguredDefaultType);
+    }
+
+    String clientConfiguredDefaultReplication =
+        clientSideConfig.get(OZONE_REPLICATION);
+    if (userPassedReplication == null
+        && clientConfiguredDefaultReplication != null) {
+      clientReplication = clientConfiguredDefaultReplication;
+    }
+
+    // if clientReplicationType or clientReplication is null, then we just pass
+    // replication config as null, so that server will take decision.
+    if (clientReplicationType == null || clientReplication == null) {
+      return null;
+    }
+    return ReplicationConfig
+        .parse(clientReplicationType, clientReplication, clientSideConfig);
+  }
 }

hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneClientUtils.java

@@ -19,8 +19,10 @@
 import org.apache.hadoop.hdds.client.ECReplicationConfig;
 import org.apache.hadoop.hdds.client.RatisReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -128,4 +130,97 @@ public void testResolveRepConfWhenFSPassedIsInvalidAndBucketDefaultNonEC() {
     // Configured value is ratis THREE
     Assert.assertEquals(ratis3ReplicationConfig, replicationConfig);
   }
+
+  /**
+   * Tests validateAndGetClientReplicationConfig with user passed valid config
+   * values.
+   */
+  @Test
+  public void testValidateAndGetRepConfWhenValidUserPassedValues() {
+    ReplicationConfig replicationConfig = OzoneClientUtils
+        .validateAndGetClientReplicationConfig(ReplicationType.RATIS, "1",
+            new OzoneConfiguration());
+    // Configured value is ratis ONE
+    Assert.assertEquals(ratis1ReplicationConfig, replicationConfig);
+  }
+
+  /**
+   * Tests validateAndGetClientReplicationConfig with user passed null values.
+   */
+  @Test
+  public void testValidateAndGetRepConfWhenValidUserPassedNullValues() {
+    ReplicationConfig replicationConfig = OzoneClientUtils
+        .validateAndGetClientReplicationConfig(null, null,
+            new OzoneConfiguration());
+    Assert.assertNull(replicationConfig);
+  }
+
+  /**
+   * Tests validateAndGetClientReplicationConfig with user passed null values
+   * but client config has valid values.
+   */
+  @Test
+  public void testValidateAndGetRepConfWhenValidConfigValues() {
+    OzoneConfiguration clientSideConfig = new OzoneConfiguration();
+    clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, "EC");
+    clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024K");
+    ReplicationConfig replicationConfig = OzoneClientUtils
+        .validateAndGetClientReplicationConfig(null, null, clientSideConfig);
+    Assert.assertEquals(ecReplicationConfig, replicationConfig);
+  }
+
+  /**
+   * Tests validateAndGetClientReplicationConfig with user passed null values
+   * but client config has valid values.
+   */
+  @Test
+  public void testValidateAndGetRepConfWhenNullTypeFromUser() {
+    ReplicationConfig replicationConfig = OzoneClientUtils
+        .validateAndGetClientReplicationConfig(null, "3",
+            new OzoneConfiguration());
+    Assert.assertNull(replicationConfig);
+  }
+
+  /**
+   * Tests validateAndGetClientReplicationConfig with user passed null
+   * replication but valid type.
+   */
+  @Test
+  public void testValidateAndGetRepConfWhenNullReplicationFromUser() {
+    ReplicationConfig replicationConfig = OzoneClientUtils
+        .validateAndGetClientReplicationConfig(ReplicationType.EC, null,
+            new OzoneConfiguration());
+    Assert.assertNull(replicationConfig);
+  }
+
+  /**
+   * Tests validateAndGetClientReplicationConfig with user pass null values but
+   * config has only replication configured.
+   */
+  @Test
+  public void testValidateAndGetRepConfWhenNullTypeConfigValues() {
+    OzoneConfiguration clientSideConfig = new OzoneConfiguration();
+    clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024K");
+    //By default config values are null. Let's don't set type to keep it as
+    // null.
+    ReplicationConfig replicationConfig = OzoneClientUtils
+        .validateAndGetClientReplicationConfig(null, null, clientSideConfig);
+    Assert.assertNull(replicationConfig);
+  }
+
+  /**
+   * Tests validateAndGetClientReplicationConfig with user pass null values but
+   * config has only type configured.
+   */
+  @Test
+  public void testValidateAndGetRepConfWhenNullReplicationConfigValues() {
+    OzoneConfiguration clientSideConfig = new OzoneConfiguration();
+    clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, "EC");
+    //By default config values are null. Let's don't set replication to keep it
+    // as null.
+    ReplicationConfig replicationConfig = OzoneClientUtils
+        .validateAndGetClientReplicationConfig(null, null, clientSideConfig);
+    Assert.assertNull(replicationConfig);
+  }
+
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java

@@ -27,6 +27,7 @@
 import java.util.Map;
 
 import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.fs.ozone.OzoneClientUtils;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.io.IOUtils;
@@ -40,6 +41,7 @@
 import org.apache.commons.codec.digest.DigestUtils;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT;
 import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY;
+
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
 import picocli.CommandLine.Parameters;
@@ -83,8 +85,9 @@ protected void execute(OzoneClient client, OzoneAddress address)
       }
     }
 
-    ReplicationConfig replicationConfig =
-        ReplicationConfig.parse(replicationType, replication, getConf());
+    ReplicationConfig replicationConfig = OzoneClientUtils
+        .validateAndGetClientReplicationConfig(replicationType, replication,
+            getConf());
 
     OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
     OzoneBucket bucket = vol.getBucket(bucketName);