HDDS-3878. Make OMHA serviceID optional if one (but only one) is defined in the config

hadoop-hdds/config/pom.xml

@@ -37,11 +37,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdds-test-utils</artifactId>
-      <scope>test</scope>
-    </dependency>
 
     <dependency>
       <groupId>junit</groupId>

hadoop-hdds/test-utils/pom.xml

@@ -33,6 +33,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   </properties>
 
   <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdds-config</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.github.spotbugs</groupId>
       <artifactId>spotbugs-annotations</artifactId>

hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/InMemoryConfiguration.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.test;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
+
+/**
+ * In memory, mutable configuration source for testing..
+ */
+public class InMemoryConfiguration implements MutableConfigurationSource {
+
+  private Map<String, String> configs = new HashMap<>();
+
+  public InMemoryConfiguration() {
+  }
+
+  public InMemoryConfiguration(String key, String value) {
+    set(key, value);
+  }
+
+  @Override
+  public String get(String key) {
+    return configs.get(key);
+  }
+
+  @Override
+  public Collection<String> getConfigKeys() {
+    return configs.keySet();
+  }
+
+  @Override
+  public char[] getPassword(String key) throws IOException {
+    return configs.get(key).toCharArray();
+  }
+
+  @Override
+  public void set(String key, String value) {
+    configs.put(key, value);
+  }
+}

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java

@@ -137,13 +137,17 @@ public static OzoneClient getRpcClient(ConfigurationSource config)
     // configuration, we don't fall back to default ozone.om.address defined
     // in ozone-default.xml.
 
-    if (OmUtils.isServiceIdsDefined(config)) {
+    String[] serviceIds = config.getTrimmedStrings(OZONE_OM_SERVICE_IDS_KEY);
+    if (serviceIds.length > 1) {
       throw new IOException("Following ServiceID's " +
           config.getTrimmedStringCollection(OZONE_OM_SERVICE_IDS_KEY) + " are" +
           " defined in the configuration. Use the method getRpcClient which " +
           "takes serviceID and configuration as param");
+    } else if (serviceIds.length == 1) {
+      return getRpcClient(getClientProtocol(config, serviceIds[0]), config);
+    } else {
+      return getRpcClient(getClientProtocol(config), config);
     }
-    return getRpcClient(getClientProtocol(config), config);
   }
 
   /**

hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml

@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+
+# reusable fragments (see https://docs.docker.com/compose/compose-file/#extension-fields)
+x-common-config:
+  &common-config
+  image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+  volumes:
+    - ../..:/opt/hadoop
+  env_file:
+    - docker-config
+
+x-replication:
+  &replication
+  OZONE-SITE.XML_ozone.replication: ${OZONE_REPLICATION_FACTOR:-1}
+
+services:
+  datanode:
+    <<: *common-config
+    ports:
+      - 9864
+      - 9882
+    environment:
+      <<: *replication
+    command: ["ozone","datanode"]
+  om1:
+    <<: *common-config
+    environment:
+      ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      <<: *replication
+    ports:
+      - 9874
+      - 9862
+    hostname: om1
+    command: ["ozone","om"]
+  om2:
+    <<: *common-config
+    environment:
+      ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      <<: *replication
+    ports:
+      - 9874
+      - 9862
+    hostname: om2
+    command: ["ozone","om"]
+  om3:
+    <<: *common-config
+    environment:
+      ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+      <<: *replication
+    ports:
+      - 9874
+      - 9862
+    hostname: om3
+    command: ["ozone","om"]
+  scm:
+    <<: *common-config
+    ports:
+      - 9876:9876
+    environment:
+      ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      OZONE-SITE.XML_hdds.scm.safemode.min.datanode: ${OZONE_SAFEMODE_MIN_DATANODES:-1}
+      <<: *replication
+    command: ["ozone","scm"]
+  s3g:
+    <<: *common-config
+    environment:
+      <<: *replication
+    ports:
+      - 9878:9878
+    command: ["ozone","s3g"]
+  recon:
+    <<: *common-config
+    ports:
+      - 9888:9888
+    environment:
+      <<: *replication
+    command: ["ozone","recon"]

hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config → hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config

@@ -14,24 +14,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
-CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.id1
-OZONE-SITE.XML_ozone.om.service.ids=id1
-OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3
-OZONE-SITE.XML_ozone.om.address.id1.om1=om1
-OZONE-SITE.XML_ozone.om.address.id1.om2=om2
-OZONE-SITE.XML_ozone.om.address.id1.om3=om3
+CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.volume1.omservice
+
+OZONE-SITE.XML_ozone.om.service.ids=omservice
+OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
+OZONE-SITE.XML_ozone.om.address.omservice.om1=om1
+OZONE-SITE.XML_ozone.om.address.omservice.om2=om2
+OZONE-SITE.XML_ozone.om.address.omservice.om3=om3
 OZONE-SITE.XML_ozone.om.ratis.enable=true
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
+
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.scm.container.size=1GB
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=3
+OZONE-SITE.XML_ozone.client.failover.max.attempts=6
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
 
-no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
+no_proxy=om1,om2,om3,scm,s3g,recon,kdc,localhost,127.0.0.1

hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/test.sh → hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh

@@ -19,18 +19,15 @@ COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 export COMPOSE_DIR
 
 export SECURITY_ENABLED=false
-export OM_SERVICE_ID="id1"
+export OZONE_REPLICATION_FACTOR=3
 
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
 
 start_docker_env
 
-execute_robot_test scm s3
-
-execute_robot_test scm freon
+execute_robot_test scm basic/ozone-shell-single.robot
 
 stop_docker_env
 
 generate_report
-