From 7331a42f4703186ada43945e7faf68a92e5edd81 Mon Sep 17 00:00:00 2001
From: saketa <schalamchala@cloudera.com>
Date: Thu, 23 Jan 2025 23:50:05 -0800
Subject: [PATCH 01/11] HDDS-10336. Skip background pipeline creation when
 client replication config is EC. Fixed the order of resolution of replcation
 configs for buckets and keys.

---
 .../pipeline/BackgroundPipelineCreator.java   | 19 ++++++++---
 .../dist/src/main/compose/common/ec-test.sh   |  2 +-
 .../dist/src/main/compose/ozone/ec.yaml       | 33 ++++++++++++++++++
 .../src/main/compose/ozone/test-default-ec.sh | 34 +++++++++++++++++++
 .../dist/src/main/smoketest/ec/basic.robot    | 18 ++++++++--
 .../shell/bucket/CreateBucketHandler.java     |  9 +++--
 .../ozone/shell/keys/PutKeyHandler.java       |  2 +-
 7 files changed, 106 insertions(+), 11 deletions(-)
 create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ec.yaml
 create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh

diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
index 4e68605c6831..62d542e45fc0 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
@@ -44,6 +44,7 @@
 
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS;
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC;
 import static org.apache.hadoop.hdds.scm.ha.SCMService.Event.NODE_ADDRESS_UPDATE_HANDLER_TRIGGERED;
 import static org.apache.hadoop.hdds.scm.ha.SCMService.Event.UNHEALTHY_TO_HEALTHY_NODE_HANDLER_TRIGGERED;
 import static org.apache.hadoop.hdds.scm.ha.SCMService.Event.NEW_NODE_HANDLER_TRIGGERED;
@@ -132,6 +133,8 @@ public void start() {
         .setUncaughtExceptionHandler((Thread t, Throwable ex) -> {
           String message = "Terminate SCM, encounter uncaught exception"
               + " in " + threadName;
+          LOG.error("BackgroundPipelineCreator thread encountered an error. "
+                  + "Thread: {}", t.toString(), ex);
           scmContext.getScm().shutDown(message);
         })
         .build()
@@ -217,12 +220,20 @@ private void createPipelines() throws RuntimeException {
       if (factor == ReplicationFactor.ZERO) {
         continue; // Ignore it.
       }
-      final ReplicationConfig replicationConfig =
-          ReplicationConfig.fromProtoTypeAndFactor(type, factor);
-      if (skipCreation(replicationConfig, autoCreateFactorOne)) {
-        // Skip this iteration for creating pipeline
+      final ReplicationConfig replicationConfig;
+      if (type != EC) {
+        replicationConfig =
+            ReplicationConfig.fromProtoTypeAndFactor(type, factor);
+      } else if (factor == ReplicationFactor.ONE) {
+        replicationConfig =
+            ReplicationConfig.fromProtoTypeAndFactor(RATIS, factor);
+      } else {
         continue;
       }
+      if (skipCreation(replicationConfig, autoCreateFactorOne)) {
+          // Skip this iteration for creating pipeline
+          continue;
+      }
       list.add(replicationConfig);
     }
 
diff --git a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
index 04df2b2787d3..9ec0781c87b0 100755
--- a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
@@ -23,7 +23,7 @@ execute_robot_test scm -v BUCKET:erasure --exclude virtual-host s3
 execute_robot_test scm ec/rewrite.robot
 
 prefix=${RANDOM}
-execute_robot_test scm -v PREFIX:${prefix} ec/basic.robot
+execute_robot_test scm -v PREFIX:${prefix} -v OZONE_CLIENT_REPLICATION_TYPE:"${OZONE_CLIENT_REPLICATION_TYPE}" ec/basic.robot
 docker-compose up -d --no-recreate --scale datanode=4
 execute_robot_test scm -v PREFIX:${prefix} -N read-4-datanodes ec/read.robot
 docker-compose up -d --no-recreate --scale datanode=3
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ec.yaml b/hadoop-ozone/dist/src/main/compose/ozone/ec.yaml
new file mode 100644
index 000000000000..5e83092708a8
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone/ec.yaml
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+x-ec-config:
+  &ec-config
+  environment:
+    - OZONE-SITE.XML_ozone.replication.type=EC
+    - OZONE-SITE.XML_ozone.replication=RS-3-2-1024k
+
+services:
+  datanode:
+    <<: *ec-config
+  om:
+    <<: *ec-config
+  scm:
+    <<: *ec-config
+  s3g:
+    <<: *ec-config
+  recon:
+    <<: *ec-config
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh b/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh
new file mode 100644
index 000000000000..7afca25d15dd
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:EC
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=false
+export OZONE_REPLICATION_FACTOR=3
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+# Replication is EC in client side configs
+export COMPOSE_FILE=docker-compose.yaml:ec.yaml
+export OZONE_CLIENT_REPLICATION_TYPE=EC
+
+start_docker_env 5
+execute_robot_test scm -v PREFIX:${prefix} -v OZONE_CLIENT_REPLICATION_TYPE:"${OZONE_CLIENT_REPLICATION_TYPE}" ec/basic.robot
diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot b/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
index 2c8c0ff892c9..10ab045b7cca 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
@@ -64,7 +64,11 @@ Test Bucket Creation
                     Should not contain  ${result}       Failed
     ${result} =     Execute             ozone sh bucket create /${VOLUME}/default
                     Should not contain  ${result}       Failed
-                    Verify Bucket Empty Replication Config      /${VOLUME}/default
+                    IF  '${OZONE_CLIENT_REPLICATION_TYPE}' == 'EC'
+                        Verify Bucket EC Replication Config    /${VOLUME}/default    RS    3    2    1048576
+                    ELSE
+                        Verify Bucket Empty Replication Config      /${VOLUME}/default
+                    END
     ${result} =     Execute             ozone sh bucket create --replication 3 --type RATIS /${VOLUME}/ratis
                     Should not contain  ${result}       Failed
                     Verify Bucket Replica Replication Config    /${VOLUME}/ratis   RATIS   THREE
@@ -91,8 +95,16 @@ Create Key in Default Bucket
     ${file} =    Set Variable    /tmp/${size}
     Create Key    ${key}    ${file}
     Key Should Match Local File    ${key}      ${file}
-    Verify Key Replica Replication Config   ${key}     RATIS    THREE
-    Verify Key Replica Replication Config   ${dir}     RATIS    THREE
+    IF  '${OZONE_CLIENT_REPLICATION_TYPE}' == 'EC'
+        Verify Key EC Replication Config    ${key}    RS    3    2    1048576
+    ELSE
+        Verify Key Replica Replication Config   ${key}     RATIS    THREE
+    END
+    IF  '${OZONE_CLIENT_REPLICATION_TYPE}' == 'EC'
+        Verify Key EC Replication Config    ${dir}    RS    3    2    1048576
+    ELSE
+        Verify Key Replica Replication Config   ${dir}     RATIS    THREE
+    END
 
 Create Key in Ratis Bucket
     ${size} =    Set Variable    1mb
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
index 02f5e80f6eb7..e0e9081aa489 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
@@ -20,6 +20,7 @@
 import com.google.common.base.Strings;
 import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
 import org.apache.hadoop.hdds.client.OzoneQuota;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.client.BucketArgs;
@@ -128,8 +129,12 @@ public void execute(OzoneClient client, OzoneAddress address)
       }
     }
 
-    replication.fromParams(getConf()).ifPresent(config ->
-        bb.setDefaultReplicationConfig(new DefaultReplicationConfig(config)));
+    ReplicationConfig replicationConfig =
+        replication.fromParamsOrConfig(getConf());
+    if (replicationConfig != null) {
+      bb.setDefaultReplicationConfig(
+          new DefaultReplicationConfig(replicationConfig));
+    }
 
     if (!Strings.isNullOrEmpty(quotaOptions.getQuotaInBytes())) {
       bb.setQuotaInBytes(OzoneQuota.parseSpaceQuota(
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
index 30543f790749..26d47bed1891 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
@@ -90,7 +90,7 @@ protected void execute(OzoneClient client, OzoneAddress address)
     }
 
     ReplicationConfig replicationConfig =
-        replication.fromParamsOrConfig(getConf());
+        replication.fromParams(getConf()).orElse(null);
 
     OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
     OzoneBucket bucket = vol.getBucket(bucketName);

From 05ef0e3d77e78ca90ab6854a4ee80a08abf7b9ea Mon Sep 17 00:00:00 2001
From: saketa <schalamchala@cloudera.com>
Date: Fri, 24 Jan 2025 10:43:33 -0800
Subject: [PATCH 02/11] HDDS-10336. ixed checkstyle errors.

---
 .../hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
index 62d542e45fc0..91aedf5cc3e8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
@@ -231,8 +231,8 @@ private void createPipelines() throws RuntimeException {
         continue;
       }
       if (skipCreation(replicationConfig, autoCreateFactorOne)) {
-          // Skip this iteration for creating pipeline
-          continue;
+        // Skip this iteration for creating pipeline
+        continue;
       }
       list.add(replicationConfig);
     }

From 4d0c710ece4291fac5aef935f2d6e79b574eea4d Mon Sep 17 00:00:00 2001
From: saketa <schalamchala@cloudera.com>
Date: Fri, 24 Jan 2025 11:45:30 -0800
Subject: [PATCH 03/11] HDDS-10336. Fixed EC acceptance test.

---
 hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh b/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh
index 7afca25d15dd..e328657e9ce5 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh
@@ -30,5 +30,7 @@ source "$COMPOSE_DIR/../testlib.sh"
 export COMPOSE_FILE=docker-compose.yaml:ec.yaml
 export OZONE_CLIENT_REPLICATION_TYPE=EC
 
-start_docker_env 5
+docker-compose up -d --no-recreate --scale datanode=5
+wait_for_safemode_exit
 execute_robot_test scm -v PREFIX:${prefix} -v OZONE_CLIENT_REPLICATION_TYPE:"${OZONE_CLIENT_REPLICATION_TYPE}" ec/basic.robot
+docker-compose --ansi never down

From 216cad5075b642f10f9267e6d8c614c6237256e8 Mon Sep 17 00:00:00 2001
From: saketa <schalamchala@cloudera.com>
Date: Thu, 30 Jan 2025 18:17:36 -0800
Subject: [PATCH 04/11] HDDS-10336. Removing changes to bucket and key
 replication config resolution order.

---
 .../src/main/compose/ozone/test-default-ec.sh  |  2 +-
 .../dist/src/main/smoketest/ec/basic.robot     | 18 +++++++-----------
 .../shell/bucket/CreateBucketHandler.java      |  9 ++-------
 .../hadoop/ozone/shell/keys/PutKeyHandler.java |  2 +-
 4 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh b/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh
index e328657e9ce5..0788bca2fdcb 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh
@@ -32,5 +32,5 @@ export OZONE_CLIENT_REPLICATION_TYPE=EC
 
 docker-compose up -d --no-recreate --scale datanode=5
 wait_for_safemode_exit
-execute_robot_test scm -v PREFIX:${prefix} -v OZONE_CLIENT_REPLICATION_TYPE:"${OZONE_CLIENT_REPLICATION_TYPE}" ec/basic.robot
+execute_robot_test scm -v PREFIX:${RANDOM} -v OZONE_CLIENT_REPLICATION_TYPE:"${OZONE_CLIENT_REPLICATION_TYPE}" ec/basic.robot
 docker-compose --ansi never down
diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot b/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
index 10ab045b7cca..a6f5e45d2367 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
@@ -64,11 +64,7 @@ Test Bucket Creation
                     Should not contain  ${result}       Failed
     ${result} =     Execute             ozone sh bucket create /${VOLUME}/default
                     Should not contain  ${result}       Failed
-                    IF  '${OZONE_CLIENT_REPLICATION_TYPE}' == 'EC'
-                        Verify Bucket EC Replication Config    /${VOLUME}/default    RS    3    2    1048576
-                    ELSE
-                        Verify Bucket Empty Replication Config      /${VOLUME}/default
-                    END
+                    Verify Bucket Empty Replication Config      /${VOLUME}/default
     ${result} =     Execute             ozone sh bucket create --replication 3 --type RATIS /${VOLUME}/ratis
                     Should not contain  ${result}       Failed
                     Verify Bucket Replica Replication Config    /${VOLUME}/ratis   RATIS   THREE
@@ -100,11 +96,7 @@ Create Key in Default Bucket
     ELSE
         Verify Key Replica Replication Config   ${key}     RATIS    THREE
     END
-    IF  '${OZONE_CLIENT_REPLICATION_TYPE}' == 'EC'
-        Verify Key EC Replication Config    ${dir}    RS    3    2    1048576
-    ELSE
-        Verify Key Replica Replication Config   ${dir}     RATIS    THREE
-    END
+    Verify Key Replica Replication Config   ${dir}     RATIS    THREE
 
 Create Key in Ratis Bucket
     ${size} =    Set Variable    1mb
@@ -113,7 +105,11 @@ Create Key in Ratis Bucket
     ${file} =    Set Variable    /tmp/${size}
     Create Key    ${key}    ${file}
     Key Should Match Local File    ${key}      ${file}
-    Verify Key Replica Replication Config   ${key}     RATIS    THREE
+    IF  '${OZONE_CLIENT_REPLICATION_TYPE}' == 'EC'
+        Verify Key EC Replication Config    ${key}    RS    3    2    1048576
+    ELSE
+        Verify Key Replica Replication Config   ${key}     RATIS    THREE
+    END
     Verify Key Replica Replication Config   ${dir}     RATIS    THREE
 
 Create Ratis Key In EC Bucket
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
index e0e9081aa489..02f5e80f6eb7 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java
@@ -20,7 +20,6 @@
 import com.google.common.base.Strings;
 import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
 import org.apache.hadoop.hdds.client.OzoneQuota;
-import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.client.BucketArgs;
@@ -129,12 +128,8 @@ public void execute(OzoneClient client, OzoneAddress address)
       }
     }
 
-    ReplicationConfig replicationConfig =
-        replication.fromParamsOrConfig(getConf());
-    if (replicationConfig != null) {
-      bb.setDefaultReplicationConfig(
-          new DefaultReplicationConfig(replicationConfig));
-    }
+    replication.fromParams(getConf()).ifPresent(config ->
+        bb.setDefaultReplicationConfig(new DefaultReplicationConfig(config)));
 
     if (!Strings.isNullOrEmpty(quotaOptions.getQuotaInBytes())) {
       bb.setQuotaInBytes(OzoneQuota.parseSpaceQuota(
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
index 26d47bed1891..30543f790749 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/PutKeyHandler.java
@@ -90,7 +90,7 @@ protected void execute(OzoneClient client, OzoneAddress address)
     }
 
     ReplicationConfig replicationConfig =
-        replication.fromParams(getConf()).orElse(null);
+        replication.fromParamsOrConfig(getConf());
 
     OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
     OzoneBucket bucket = vol.getBucket(bucketName);

From 0599cff483adbf734981c839d2e950629d33962c Mon Sep 17 00:00:00 2001
From: saketa <schalamchala@cloudera.com>
Date: Fri, 7 Feb 2025 19:07:54 -0800
Subject: [PATCH 05/11] HDDS-10336. Added integration test. Removed acceptance
 test.

---
 .../dist/src/main/compose/common/ec-test.sh   |   2 +-
 .../dist/src/main/compose/ozone/ec.yaml       |  33 --
 .../src/main/compose/ozone/test-default-ec.sh |  36 ---
 .../dist/src/main/smoketest/ec/basic.robot    |  12 +-
 .../TestReplicationConfigPreference.java      | 285 ++++++++++++++++++
 5 files changed, 288 insertions(+), 80 deletions(-)
 delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/ec.yaml
 delete mode 100644 hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh
 create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java

diff --git a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
index 9ec0781c87b0..04df2b2787d3 100755
--- a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
+++ b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh
@@ -23,7 +23,7 @@ execute_robot_test scm -v BUCKET:erasure --exclude virtual-host s3
 execute_robot_test scm ec/rewrite.robot
 
 prefix=${RANDOM}
-execute_robot_test scm -v PREFIX:${prefix} -v OZONE_CLIENT_REPLICATION_TYPE:"${OZONE_CLIENT_REPLICATION_TYPE}" ec/basic.robot
+execute_robot_test scm -v PREFIX:${prefix} ec/basic.robot
 docker-compose up -d --no-recreate --scale datanode=4
 execute_robot_test scm -v PREFIX:${prefix} -N read-4-datanodes ec/read.robot
 docker-compose up -d --no-recreate --scale datanode=3
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/ec.yaml b/hadoop-ozone/dist/src/main/compose/ozone/ec.yaml
deleted file mode 100644
index 5e83092708a8..000000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone/ec.yaml
+++ /dev/null
@@ -1,33 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-x-ec-config:
-  &ec-config
-  environment:
-    - OZONE-SITE.XML_ozone.replication.type=EC
-    - OZONE-SITE.XML_ozone.replication=RS-3-2-1024k
-
-services:
-  datanode:
-    <<: *ec-config
-  om:
-    <<: *ec-config
-  scm:
-    <<: *ec-config
-  s3g:
-    <<: *ec-config
-  recon:
-    <<: *ec-config
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh b/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh
deleted file mode 100644
index 0788bca2fdcb..000000000000
--- a/hadoop-ozone/dist/src/main/compose/ozone/test-default-ec.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#suite:EC
-
-COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export COMPOSE_DIR
-
-export SECURITY_ENABLED=false
-export OZONE_REPLICATION_FACTOR=3
-
-# shellcheck source=/dev/null
-source "$COMPOSE_DIR/../testlib.sh"
-
-# Replication is EC in client side configs
-export COMPOSE_FILE=docker-compose.yaml:ec.yaml
-export OZONE_CLIENT_REPLICATION_TYPE=EC
-
-docker-compose up -d --no-recreate --scale datanode=5
-wait_for_safemode_exit
-execute_robot_test scm -v PREFIX:${RANDOM} -v OZONE_CLIENT_REPLICATION_TYPE:"${OZONE_CLIENT_REPLICATION_TYPE}" ec/basic.robot
-docker-compose --ansi never down
diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot b/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
index a6f5e45d2367..2c8c0ff892c9 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
@@ -91,11 +91,7 @@ Create Key in Default Bucket
     ${file} =    Set Variable    /tmp/${size}
     Create Key    ${key}    ${file}
     Key Should Match Local File    ${key}      ${file}
-    IF  '${OZONE_CLIENT_REPLICATION_TYPE}' == 'EC'
-        Verify Key EC Replication Config    ${key}    RS    3    2    1048576
-    ELSE
-        Verify Key Replica Replication Config   ${key}     RATIS    THREE
-    END
+    Verify Key Replica Replication Config   ${key}     RATIS    THREE
     Verify Key Replica Replication Config   ${dir}     RATIS    THREE
 
 Create Key in Ratis Bucket
@@ -105,11 +101,7 @@ Create Key in Ratis Bucket
     ${file} =    Set Variable    /tmp/${size}
     Create Key    ${key}    ${file}
     Key Should Match Local File    ${key}      ${file}
-    IF  '${OZONE_CLIENT_REPLICATION_TYPE}' == 'EC'
-        Verify Key EC Replication Config    ${key}    RS    3    2    1048576
-    ELSE
-        Verify Key Replica Replication Config   ${key}     RATIS    THREE
-    END
+    Verify Key Replica Replication Config   ${key}     RATIS    THREE
     Verify Key Replica Replication Config   ${dir}     RATIS    THREE
 
 Create Ratis Key In EC Bucket
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
new file mode 100644
index 000000000000..f31deecc82df
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
@@ -0,0 +1,285 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.shell;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+/**
+ * Test the order of Replication config resolution.
+ *
+ */
+@Timeout(100)
+public class TestReplicationConfigPreference {
+
+  private static MiniOzoneCluster cluster;
+  private static OzoneClient client;
+  private static String omServiceId;
+  @TempDir
+  private static java.nio.file.Path path;
+  private static File testFile;
+  private static final String DEFAULT_BUCKET = "default";
+  private static final String RATIS_BUCKET = "ratis";
+  private static final String EC_BUCKET = "ecbucket";
+  private static final String DEFAULT_KEY = "defaultkey";
+  private static final String RATIS_KEY = "ratiskey";
+  private static final String EC_KEY = "eckey";
+  private static String[] bucketList;
+  private static String[] keyList;
+  private static final ReplicationConfig RATIS_REPL_CONF =
+      ReplicationConfig.fromProtoTypeAndFactor(RATIS, THREE);
+  private static final ReplicationConfig EC_REPL_CONF = new ECReplicationConfig(
+      3, 2, ECReplicationConfig.EcCodec.RS, (int) OzoneConsts.MB);
+
+  protected static void startCluster(OzoneConfiguration ozoneConf)
+      throws Exception {
+    cluster = MiniOzoneCluster.newBuilder(ozoneConf)
+        .setNumDatanodes(5)
+        .build();
+    cluster.waitForClusterToBeReady();
+    client = cluster.newClient();
+    omServiceId = cluster.getOzoneManager().getHostname() + ":" +
+        cluster.getOzoneManager().getRpcPort();
+  }
+
+  @BeforeAll
+  public static void init() throws Exception {
+    testFile = new File(path + OZONE_URI_DELIMITER + "testFile");
+    testFile.getParentFile().mkdirs();
+    testFile.createNewFile();
+    bucketList = new String[]{DEFAULT_BUCKET, RATIS_BUCKET, EC_BUCKET};
+    keyList = new String[]{DEFAULT_KEY, RATIS_KEY, EC_KEY};
+  }
+
+  /**
+   * shutdown MiniOzoneCluster.
+   */
+  @AfterAll
+  public static void shutdown() {
+    IOUtils.closeQuietly(client);
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+
+  protected static void createAllKeys(OzoneShell ozoneShell,
+                                      String volumeName, String bucketName) {
+    ozoneShell.execute(new String[] {"key", "put", "o3://" + omServiceId
+        + "/" + volumeName + "/" + bucketName + "/" + DEFAULT_KEY,
+        testFile.getPath()});
+    ozoneShell.execute(new String[] {"key", "put", "o3://" + omServiceId
+        + "/" + volumeName + "/" + bucketName + "/" + RATIS_KEY,
+        testFile.getPath(),
+        "--type=" + RATIS_REPL_CONF.getReplicationType().name(),
+        "--replication=" + RATIS_REPL_CONF.getReplication()});
+    ozoneShell.execute(new String[] {"key", "put", "o3://" + omServiceId
+        + "/" + volumeName + "/" + bucketName + "/" + EC_KEY,
+        testFile.getPath(),
+        "--type=" + EC_REPL_CONF.getReplicationType().name(),
+        "--replication=" + EC_REPL_CONF.getReplication()});
+  }
+
+  protected static void createAllBucketsAndKeys(Map<String, String> clientConf,
+                                                String volumeName) {
+    OzoneShell ozoneShell = new OzoneShell();
+    if (clientConf != null) {
+      ozoneShell.setConfigurationOverrides(clientConf);
+    }
+    ozoneShell.execute(new String[] {"volume", "create", "o3://" + omServiceId
+        + "/" + volumeName});
+    ozoneShell.execute(new String[] {"bucket", "create", "o3://" + omServiceId
+        + "/" + volumeName + "/" + DEFAULT_BUCKET});
+    createAllKeys(ozoneShell, volumeName, DEFAULT_BUCKET);
+
+    ozoneShell = new OzoneShell();
+    if (clientConf != null) {
+      ozoneShell.setConfigurationOverrides(clientConf);
+    }
+    ozoneShell.execute(new String[] {"bucket", "create", "o3://" + omServiceId
+        + "/" + volumeName + "/" + RATIS_BUCKET,
+        "--type=" + RATIS_REPL_CONF.getReplicationType().name(),
+        "--replication=" + RATIS_REPL_CONF.getReplication()});
+    createAllKeys(ozoneShell, volumeName, RATIS_BUCKET);
+
+    ozoneShell = new OzoneShell();
+    if (clientConf != null) {
+      ozoneShell.setConfigurationOverrides(clientConf);
+    }
+    ozoneShell.execute(new String[] {"bucket", "create", "o3://" + omServiceId
+        + "/" + volumeName + "/" + EC_BUCKET,
+        "--type=" + EC_REPL_CONF.getReplicationType().name(),
+        "--replication=" + EC_REPL_CONF.getReplication()});
+    createAllKeys(ozoneShell, volumeName, EC_BUCKET);
+  }
+
+  /**
+   * Test the replication Order when different combinations of
+   * bucket and key cli replication configs, bucket replication config
+   * and server default replication configs are set.
+   * Expected Order of preference is:
+   * 1. Replication config params from CLI
+   * 2. Replication configs from client configs
+   * 2. Bucket Replication Config
+   * 3. Server Default replication config
+   */
+  public void validateReplicationOrder(String volumeName,
+                                       String clientConfigReplType,
+                                       String clientConfigReplConfig,
+                                       String serverDefaultReplType,
+                                       String serverDefaultReplConfig) throws Exception {
+
+    for (String b: bucketList) {
+      OzoneBucket bucket = client.getObjectStore().getVolume(volumeName)
+          .getBucket(b);
+      if (b.equals(DEFAULT_BUCKET)) {
+        assertNull(bucket.getReplicationConfig());
+      } else {
+        assertNotNull(bucket.getReplicationConfig());
+      }
+
+      for (String k: keyList) {
+        OzoneKeyDetails key = bucket.getKey(k);
+        if (b.equals(DEFAULT_BUCKET) && k.equals(DEFAULT_KEY)) {
+          // if replication config are not passed as CLI params during bucket and
+          // key creation then key uses client configs.
+          // If replication is not set in client configs then key uses
+          // server default replication config.
+          if (clientConfigReplConfig != null) {
+            assertEquals(clientConfigReplType, key.getReplicationConfig()
+                .getReplicationType().name());
+            assertEquals(clientConfigReplConfig, key.getReplicationConfig()
+                .getReplication());
+          } else {
+            assertEquals(serverDefaultReplType, key.getReplicationConfig()
+                .getReplicationType().name());
+            assertEquals(serverDefaultReplConfig, key.getReplicationConfig()
+                .getReplication());
+          }
+        } else if (k.equals(DEFAULT_KEY)) {
+          // if replication config are not passed as CLI params
+          // during key creation and bucket replication config is set then
+          // key uses bucket replication config
+          // unless replication config is available in client configs.
+          if (clientConfigReplConfig != null) {
+            assertEquals(clientConfigReplType, key.getReplicationConfig()
+                .getReplicationType().name());
+            assertEquals(clientConfigReplConfig, key.getReplicationConfig()
+                .getReplication());
+          } else {
+            assertEquals(bucket.getReplicationConfig(),
+                key.getReplicationConfig());
+          }
+        } else if (k.equals(RATIS_KEY)) {
+          // if client sets replication config during key creation then
+          // this replication config is used regardless of client configs or
+          // bucket replication or server defaults.
+          assertEquals(RATIS_REPL_CONF, key.getReplicationConfig());
+        } else if (k.equals(EC_KEY)) {
+          assertEquals(EC_REPL_CONF, key.getReplicationConfig());
+        }
+      }
+    }
+  }
+
+  @Test
+  public void testReplicationOrderDefaultRatisCluster() throws Exception {
+    // Starting a cluster with server default replication type RATIS/THREE.
+    shutdown();
+    startCluster(new OzoneConfiguration());
+
+    // Replication configs are not set in Client configuration.
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    createAllBucketsAndKeys(null, volumeName);
+    validateReplicationOrder(volumeName, null, null,
+        RATIS_REPL_CONF.getReplicationType().name(),
+        RATIS_REPL_CONF.getReplication());
+
+    // Replication configs are set in Client configuration.
+    Map<String, String> clientConf = new HashMap<String, String>() {{
+      put(OZONE_REPLICATION_TYPE, EC_REPL_CONF.getReplicationType().name());
+      put(OZONE_REPLICATION, EC_REPL_CONF.getReplication());
+    }};
+    volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    createAllBucketsAndKeys(clientConf, volumeName);
+    validateReplicationOrder(volumeName,
+        EC_REPL_CONF.getReplicationType().name(),
+        EC_REPL_CONF.getReplication(),
+        RATIS_REPL_CONF.getReplicationType().name(),
+        RATIS_REPL_CONF.getReplication());
+  }
+
+  @Test
+  public void testReplicationOrderDefaultECCluster() throws Exception {
+    // Starting a cluster with server default replication type EC.
+    shutdown();
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.set(OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY,
+        EC_REPL_CONF.getReplicationType().name());
+    conf.set(OZONE_SERVER_DEFAULT_REPLICATION_KEY,
+        EC_REPL_CONF.getReplication());
+    startCluster(conf);
+
+    // Replication configs are not set in Client configuration.
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    createAllBucketsAndKeys(null, volumeName);
+    validateReplicationOrder(volumeName, null, null,
+        EC_REPL_CONF.getReplicationType().name(),
+        EC_REPL_CONF.getReplication());
+
+    // Replication configs are set in Client configuration.
+    Map<String, String> clientConf = new HashMap<String, String>() {{
+      put(OZONE_REPLICATION_TYPE, RATIS_REPL_CONF.getReplicationType().name());
+      put(OZONE_REPLICATION, RATIS_REPL_CONF.getReplication());
+    }};
+    volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    createAllBucketsAndKeys(clientConf, volumeName);
+    validateReplicationOrder(volumeName,
+        RATIS_REPL_CONF.getReplicationType().name(),
+        RATIS_REPL_CONF.getReplication(),
+        EC_REPL_CONF.getReplicationType().name(),
+        EC_REPL_CONF.getReplication());
+  }
+}

From 9acc02b129e8b7e96918d91dd92883f6fbbfa3bb Mon Sep 17 00:00:00 2001
From: saketa <schalamchala@cloudera.com>
Date: Mon, 10 Feb 2025 09:52:56 -0800
Subject: [PATCH 06/11] HDDS-10336. Fixed checkstyle errors.

---
 .../TestReplicationConfigPreference.java      | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
index f31deecc82df..9d4d47d362b4 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
@@ -108,15 +108,15 @@ public static void shutdown() {
   protected static void createAllKeys(OzoneShell ozoneShell,
                                       String volumeName, String bucketName) {
     ozoneShell.execute(new String[] {"key", "put", "o3://" + omServiceId
-        + "/" + volumeName + "/" + bucketName + "/" + DEFAULT_KEY,
+          + "/" + volumeName + "/" + bucketName + "/" + DEFAULT_KEY,
         testFile.getPath()});
     ozoneShell.execute(new String[] {"key", "put", "o3://" + omServiceId
-        + "/" + volumeName + "/" + bucketName + "/" + RATIS_KEY,
+          + "/" + volumeName + "/" + bucketName + "/" + RATIS_KEY,
         testFile.getPath(),
         "--type=" + RATIS_REPL_CONF.getReplicationType().name(),
         "--replication=" + RATIS_REPL_CONF.getReplication()});
     ozoneShell.execute(new String[] {"key", "put", "o3://" + omServiceId
-        + "/" + volumeName + "/" + bucketName + "/" + EC_KEY,
+          + "/" + volumeName + "/" + bucketName + "/" + EC_KEY,
         testFile.getPath(),
         "--type=" + EC_REPL_CONF.getReplicationType().name(),
         "--replication=" + EC_REPL_CONF.getReplication()});
@@ -129,9 +129,9 @@ protected static void createAllBucketsAndKeys(Map<String, String> clientConf,
       ozoneShell.setConfigurationOverrides(clientConf);
     }
     ozoneShell.execute(new String[] {"volume", "create", "o3://" + omServiceId
-        + "/" + volumeName});
+          + "/" + volumeName});
     ozoneShell.execute(new String[] {"bucket", "create", "o3://" + omServiceId
-        + "/" + volumeName + "/" + DEFAULT_BUCKET});
+          + "/" + volumeName + "/" + DEFAULT_BUCKET});
     createAllKeys(ozoneShell, volumeName, DEFAULT_BUCKET);
 
     ozoneShell = new OzoneShell();
@@ -139,7 +139,7 @@ protected static void createAllBucketsAndKeys(Map<String, String> clientConf,
       ozoneShell.setConfigurationOverrides(clientConf);
     }
     ozoneShell.execute(new String[] {"bucket", "create", "o3://" + omServiceId
-        + "/" + volumeName + "/" + RATIS_BUCKET,
+          + "/" + volumeName + "/" + RATIS_BUCKET,
         "--type=" + RATIS_REPL_CONF.getReplicationType().name(),
         "--replication=" + RATIS_REPL_CONF.getReplication()});
     createAllKeys(ozoneShell, volumeName, RATIS_BUCKET);
@@ -149,7 +149,7 @@ protected static void createAllBucketsAndKeys(Map<String, String> clientConf,
       ozoneShell.setConfigurationOverrides(clientConf);
     }
     ozoneShell.execute(new String[] {"bucket", "create", "o3://" + omServiceId
-        + "/" + volumeName + "/" + EC_BUCKET,
+          + "/" + volumeName + "/" + EC_BUCKET,
         "--type=" + EC_REPL_CONF.getReplicationType().name(),
         "--replication=" + EC_REPL_CONF.getReplication()});
     createAllKeys(ozoneShell, volumeName, EC_BUCKET);
@@ -239,9 +239,9 @@ public void testReplicationOrderDefaultRatisCluster() throws Exception {
 
     // Replication configs are set in Client configuration.
     Map<String, String> clientConf = new HashMap<String, String>() {{
-      put(OZONE_REPLICATION_TYPE, EC_REPL_CONF.getReplicationType().name());
-      put(OZONE_REPLICATION, EC_REPL_CONF.getReplication());
-    }};
+        put(OZONE_REPLICATION_TYPE, EC_REPL_CONF.getReplicationType().name());
+        put(OZONE_REPLICATION, EC_REPL_CONF.getReplication());
+      }};
     volumeName = "volume" + RandomStringUtils.randomNumeric(5);
     createAllBucketsAndKeys(clientConf, volumeName);
     validateReplicationOrder(volumeName,
@@ -271,9 +271,9 @@ public void testReplicationOrderDefaultECCluster() throws Exception {
 
     // Replication configs are set in Client configuration.
     Map<String, String> clientConf = new HashMap<String, String>() {{
-      put(OZONE_REPLICATION_TYPE, RATIS_REPL_CONF.getReplicationType().name());
-      put(OZONE_REPLICATION, RATIS_REPL_CONF.getReplication());
-    }};
+        put(OZONE_REPLICATION_TYPE, RATIS_REPL_CONF.getReplicationType().name());
+        put(OZONE_REPLICATION, RATIS_REPL_CONF.getReplication());
+      }};
     volumeName = "volume" + RandomStringUtils.randomNumeric(5);
     createAllBucketsAndKeys(clientConf, volumeName);
     validateReplicationOrder(volumeName,

From 5e8ef288fadf3fc18ae96a34bb29dce73d79ec0e Mon Sep 17 00:00:00 2001
From: saketa <schalamchala@cloudera.com>
Date: Tue, 11 Feb 2025 15:32:59 -0800
Subject: [PATCH 07/11] HDDS-10336. Used MiniOzoneHA cluster.

---
 .../TestReplicationConfigPreference.java      | 117 +++++++++++++++---
 1 file changed, 98 insertions(+), 19 deletions(-)

diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
index 9d4d47d362b4..4c5f65824c5d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
@@ -18,20 +18,25 @@
 package org.apache.hadoop.ozone.shell;
 
 import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.cli.GenericCli;
 import org.apache.hadoop.hdds.client.ECReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.utils.IOUtils;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.api.io.TempDir;
+import picocli.CommandLine;
 
 import java.io.File;
 import java.util.HashMap;
@@ -55,6 +60,7 @@
 @Timeout(100)
 public class TestReplicationConfigPreference {
 
+  private static OzoneConfiguration conf = null;
   private static MiniOzoneCluster cluster;
   private static OzoneClient client;
   private static String omServiceId;
@@ -69,20 +75,23 @@ public class TestReplicationConfigPreference {
   private static final String EC_KEY = "eckey";
   private static String[] bucketList;
   private static String[] keyList;
+  private static int numOfOMs = 3;
   private static final ReplicationConfig RATIS_REPL_CONF =
       ReplicationConfig.fromProtoTypeAndFactor(RATIS, THREE);
   private static final ReplicationConfig EC_REPL_CONF = new ECReplicationConfig(
       3, 2, ECReplicationConfig.EcCodec.RS, (int) OzoneConsts.MB);
 
-  protected static void startCluster(OzoneConfiguration ozoneConf)
+  protected static void startCluster()
       throws Exception {
-    cluster = MiniOzoneCluster.newBuilder(ozoneConf)
-        .setNumDatanodes(5)
-        .build();
+    omServiceId = "om-service-test1";
+    MiniOzoneHAClusterImpl.Builder builder =
+        MiniOzoneCluster.newHABuilder(conf);
+    builder.setOMServiceId(omServiceId)
+        .setNumOfOzoneManagers(numOfOMs)
+        .setNumDatanodes(5);
+    cluster = builder.build();
     cluster.waitForClusterToBeReady();
     client = cluster.newClient();
-    omServiceId = cluster.getOzoneManager().getHostname() + ":" +
-        cluster.getOzoneManager().getRpcPort();
   }
 
   @BeforeAll
@@ -105,32 +114,101 @@ public static void shutdown() {
     }
   }
 
-  protected static void createAllKeys(OzoneShell ozoneShell,
-                                      String volumeName, String bucketName) {
-    ozoneShell.execute(new String[] {"key", "put", "o3://" + omServiceId
+  private String getSetConfStringFromConf(String key) {
+    return String.format("--set=%s=%s", key, conf.get(key));
+  }
+
+  private String generateSetConfString(String key, String value) {
+    return String.format("--set=%s=%s", key, value);
+  }
+
+  /**
+   * Helper function to get a String array to be fed into OzoneShell.
+   * @param numOfArgs Additional number of arguments after the HA conf string,
+   *                  this translates into the number of empty array elements
+   *                  after the HA conf string.
+   * @return String array.
+   */
+  private String[] getHASetConfStrings(int numOfArgs) {
+    assert (numOfArgs >= 0);
+    String[] res = new String[1 + 1 + numOfOMs + numOfArgs];
+    final int indexOmServiceIds = 0;
+    final int indexOmNodes = 1;
+    final int indexOmAddressStart = 2;
+
+    res[indexOmServiceIds] = getSetConfStringFromConf(
+        OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY);
+
+    String omNodesKey = ConfUtils.addKeySuffixes(
+        OMConfigKeys.OZONE_OM_NODES_KEY, omServiceId);
+    String omNodesVal = conf.get(omNodesKey);
+    res[indexOmNodes] = generateSetConfString(omNodesKey, omNodesVal);
+
+    String[] omNodesArr = omNodesVal.split(",");
+    // Sanity check
+    assert (omNodesArr.length == numOfOMs);
+    for (int i = 0; i < numOfOMs; i++) {
+      res[indexOmAddressStart + i] =
+          getSetConfStringFromConf(ConfUtils.addKeySuffixes(
+              OMConfigKeys.OZONE_OM_ADDRESS_KEY, omServiceId, omNodesArr[i]));
+    }
+
+    return res;
+  }
+
+  /**
+   * Helper function to create a new set of arguments that contains HA configs.
+   * @param existingArgs Existing arguments to be fed into OzoneShell command.
+   * @return String array.
+   */
+  private String[] getHASetConfStrings(String[] existingArgs) {
+    // Get a String array populated with HA configs first
+    String[] res = getHASetConfStrings(existingArgs.length);
+
+    int indexCopyStart = res.length - existingArgs.length;
+    // Then copy the existing args to the returned String array
+    for (int i = 0; i < existingArgs.length; i++) {
+      res[indexCopyStart + i] = existingArgs[i];
+    }
+    return res;
+  }
+
+  private void execute(GenericCli shell, String[] args) {
+    CommandLine cmd = shell.getCmd();
+
+    // Since there is no elegant way to pass Ozone config to the shell,
+    // the idea is to use 'set' to place those OM HA configs.
+    String[] argsWithHAConf = getHASetConfStrings(args);
+
+    cmd.execute(argsWithHAConf);
+  }
+
+  protected void createAllKeys(OzoneShell ozoneShell,
+      String volumeName, String bucketName) {
+    execute(ozoneShell, new String[] {"key", "put", "o3://" + omServiceId
           + "/" + volumeName + "/" + bucketName + "/" + DEFAULT_KEY,
         testFile.getPath()});
-    ozoneShell.execute(new String[] {"key", "put", "o3://" + omServiceId
+    execute(ozoneShell, new String[] {"key", "put", "o3://" + omServiceId
           + "/" + volumeName + "/" + bucketName + "/" + RATIS_KEY,
         testFile.getPath(),
         "--type=" + RATIS_REPL_CONF.getReplicationType().name(),
         "--replication=" + RATIS_REPL_CONF.getReplication()});
-    ozoneShell.execute(new String[] {"key", "put", "o3://" + omServiceId
+    execute(ozoneShell, new String[] {"key", "put", "o3://" + omServiceId
           + "/" + volumeName + "/" + bucketName + "/" + EC_KEY,
         testFile.getPath(),
         "--type=" + EC_REPL_CONF.getReplicationType().name(),
         "--replication=" + EC_REPL_CONF.getReplication()});
   }
 
-  protected static void createAllBucketsAndKeys(Map<String, String> clientConf,
+  protected void createAllBucketsAndKeys(Map<String, String> clientConf,
                                                 String volumeName) {
     OzoneShell ozoneShell = new OzoneShell();
     if (clientConf != null) {
       ozoneShell.setConfigurationOverrides(clientConf);
     }
-    ozoneShell.execute(new String[] {"volume", "create", "o3://" + omServiceId
+    execute(ozoneShell, new String[] {"volume", "create", "o3://" + omServiceId
           + "/" + volumeName});
-    ozoneShell.execute(new String[] {"bucket", "create", "o3://" + omServiceId
+    execute(ozoneShell, new String[] {"bucket", "create", "o3://" + omServiceId
           + "/" + volumeName + "/" + DEFAULT_BUCKET});
     createAllKeys(ozoneShell, volumeName, DEFAULT_BUCKET);
 
@@ -138,7 +216,7 @@ protected static void createAllBucketsAndKeys(Map<String, String> clientConf,
     if (clientConf != null) {
       ozoneShell.setConfigurationOverrides(clientConf);
     }
-    ozoneShell.execute(new String[] {"bucket", "create", "o3://" + omServiceId
+    execute(ozoneShell, new String[] {"bucket", "create", "o3://" + omServiceId
           + "/" + volumeName + "/" + RATIS_BUCKET,
         "--type=" + RATIS_REPL_CONF.getReplicationType().name(),
         "--replication=" + RATIS_REPL_CONF.getReplication()});
@@ -148,7 +226,7 @@ protected static void createAllBucketsAndKeys(Map<String, String> clientConf,
     if (clientConf != null) {
       ozoneShell.setConfigurationOverrides(clientConf);
     }
-    ozoneShell.execute(new String[] {"bucket", "create", "o3://" + omServiceId
+    execute(ozoneShell, new String[] {"bucket", "create", "o3://" + omServiceId
           + "/" + volumeName + "/" + EC_BUCKET,
         "--type=" + EC_REPL_CONF.getReplicationType().name(),
         "--replication=" + EC_REPL_CONF.getReplication()});
@@ -228,7 +306,8 @@ public void validateReplicationOrder(String volumeName,
   public void testReplicationOrderDefaultRatisCluster() throws Exception {
     // Starting a cluster with server default replication type RATIS/THREE.
     shutdown();
-    startCluster(new OzoneConfiguration());
+    conf = new OzoneConfiguration();
+    startCluster();
 
     // Replication configs are not set in Client configuration.
     String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
@@ -255,12 +334,12 @@ public void testReplicationOrderDefaultRatisCluster() throws Exception {
   public void testReplicationOrderDefaultECCluster() throws Exception {
     // Starting a cluster with server default replication type EC.
     shutdown();
-    OzoneConfiguration conf = new OzoneConfiguration();
+    conf = new OzoneConfiguration();
     conf.set(OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY,
         EC_REPL_CONF.getReplicationType().name());
     conf.set(OZONE_SERVER_DEFAULT_REPLICATION_KEY,
         EC_REPL_CONF.getReplication());
-    startCluster(conf);
+    startCluster();
 
     // Replication configs are not set in Client configuration.
     String volumeName = "volume" + RandomStringUtils.randomNumeric(5);

From c95a3fc27548b71dc40d7cc79d788e83bc2e5622 Mon Sep 17 00:00:00 2001
From: saketa <schalamchala@cloudera.com>
Date: Tue, 11 Feb 2025 15:52:23 -0800
Subject: [PATCH 08/11] HDDS-10336. Fixed findbugs error.

---
 .../hadoop/ozone/shell/TestReplicationConfigPreference.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
index 4c5f65824c5d..8dc77b36fa3d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
@@ -60,7 +60,7 @@
 @Timeout(100)
 public class TestReplicationConfigPreference {
 
-  private static OzoneConfiguration conf = null;
+  private OzoneConfiguration conf = null;
   private static MiniOzoneCluster cluster;
   private static OzoneClient client;
   private static String omServiceId;

From 9f7685a49b110602cd8ddb8724d9c4adf730e13e Mon Sep 17 00:00:00 2001
From: saketa <schalamchala@cloudera.com>
Date: Tue, 11 Feb 2025 16:13:39 -0800
Subject: [PATCH 09/11] HDDS-10336. Fixed compilation error.

---
 .../hadoop/ozone/shell/TestReplicationConfigPreference.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
index 8dc77b36fa3d..af6b194c64d0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
@@ -81,7 +81,7 @@ public class TestReplicationConfigPreference {
   private static final ReplicationConfig EC_REPL_CONF = new ECReplicationConfig(
       3, 2, ECReplicationConfig.EcCodec.RS, (int) OzoneConsts.MB);
 
-  protected static void startCluster()
+  protected void startCluster()
       throws Exception {
     omServiceId = "om-service-test1";
     MiniOzoneHAClusterImpl.Builder builder =

From 954ecb5c51a54aa860d9701c9c81bbc3db2ff61f Mon Sep 17 00:00:00 2001
From: saketa <schalamchala@cloudera.com>
Date: Tue, 11 Feb 2025 16:48:44 -0800
Subject: [PATCH 10/11] HDDS-10336. Fixed compilation errors.

---
 .../hadoop/ozone/shell/TestReplicationConfigPreference.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
index af6b194c64d0..ed7c78c782f3 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
@@ -61,9 +61,9 @@
 public class TestReplicationConfigPreference {
 
   private OzoneConfiguration conf = null;
-  private static MiniOzoneCluster cluster;
-  private static OzoneClient client;
-  private static String omServiceId;
+  private MiniOzoneCluster cluster;
+  private OzoneClient client;
+  private String omServiceId;
   @TempDir
   private static java.nio.file.Path path;
   private static File testFile;

From 816cffa936c92d9757a8c94d551aa587555af82f Mon Sep 17 00:00:00 2001
From: saketa <schalamchala@cloudera.com>
Date: Tue, 11 Feb 2025 17:10:40 -0800
Subject: [PATCH 11/11] HDDS-10336. Fixed more compilation errors.

---
 .../hadoop/ozone/shell/TestReplicationConfigPreference.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
index ed7c78c782f3..c1d4373ff228 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestReplicationConfigPreference.java
@@ -107,7 +107,7 @@ public static void init() throws Exception {
    * shutdown MiniOzoneCluster.
    */
   @AfterAll
-  public static void shutdown() {
+  public void shutdown() {
     IOUtils.closeQuietly(client);
     if (cluster != null) {
       cluster.shutdown();