HDDS-8581. Avoid random ports in integration tests

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java

@@ -109,7 +109,6 @@ public static void initClass() throws Exception {
     final String path = GenericTestUtils.getTempPath(omId);
     java.nio.file.Path metaDirPath = java.nio.file.Paths.get(path, "om-meta");
     conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString());
-    conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
     conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, 3);
 
     conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java

@@ -23,6 +23,7 @@
 import java.util.OptionalInt;
 import java.util.UUID;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.conf.StorageUnit;
 import org.apache.hadoop.hdds.HddsConfigKeys;
@@ -310,8 +311,6 @@ abstract class Builder {
     protected Optional<String> scmId = Optional.empty();
     protected Optional<String> omId = Optional.empty();
 
-    protected Boolean randomContainerPort = true;
-    protected Boolean randomContainerStreamPort = true;
     protected Boolean enableContainerDatastream = true;
     protected Optional<String> datanodeReservedSpace = Optional.empty();
     protected Optional<Integer> chunkSize = Optional.empty();
@@ -423,18 +422,6 @@ public Builder setOmId(String id) {
       return this;
     }
 
-    /**
-     * If set to true container service will be started in a random port.
-     *
-     * @param randomPort enable random port
-     *
-     * @return MiniOzoneCluster.Builder
-     */
-    public Builder setRandomContainerPort(boolean randomPort) {
-      randomContainerPort = randomPort;
-      return this;
-    }
-
     /**
      * Sets the number of HddsDatanodes to be started as part of
      * MiniOzoneCluster.
@@ -650,4 +637,35 @@ public Builder setDnLayoutVersion(int layoutVersion) {
      */
     public abstract MiniOzoneCluster build() throws IOException;
   }
+
+  /**
+   * Helper class to get free port avoiding randomness.
+   */
+  class PortAllocator {
+
+    private static final int MIN_PORT = 15000;
+    private static final int MAX_PORT = 32000;
+    private static final AtomicInteger NEXT_PORT = new AtomicInteger(MIN_PORT);
+
+    private PortAllocator() {
+      // no instances
+    }
+
+    static synchronized int getFreePort() {
+      int port = NEXT_PORT.getAndIncrement();
+      if (port > MAX_PORT) {
+        NEXT_PORT.set(MIN_PORT);
+        port = NEXT_PORT.getAndIncrement();
+      }
+      return port;
+    }
+
+    static String localhostWithFreePort() {
+      return "127.0.0.1:" + getFreePort();
+    }
+
+    static String anyHostWithFreePort() {
+      return "0.0.0.0:" + getFreePort();
+    }
+  }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

@@ -40,6 +40,7 @@
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
 import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationTarget;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -83,23 +84,19 @@
 
 import org.apache.commons.io.FileUtils;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
-import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.RATIS;
-import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.RATIS_ADMIN;
-import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.RATIS_SERVER;
-import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.REPLICATION;
-import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.STANDALONE;
 import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
 import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY;
 import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY;
 import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY;
 import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD;
 import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION;
+import static org.apache.hadoop.ozone.MiniOzoneCluster.PortAllocator.anyHostWithFreePort;
+import static org.apache.hadoop.ozone.MiniOzoneCluster.PortAllocator.getFreePort;
+import static org.apache.hadoop.ozone.MiniOzoneCluster.PortAllocator.localhostWithFreePort;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_IPC_PORT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ADMIN_PORT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_PORT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_SERVER_PORT;
 import static org.apache.hadoop.ozone.om.OmUpgradeConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION;
 import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
@@ -399,24 +396,10 @@ private void waitForHddsDatanodeToStop(DatanodeDetails dn)
   @Override
   public void restartHddsDatanode(int i, boolean waitForDatanode)
       throws InterruptedException, TimeoutException {
-    HddsDatanodeService datanodeService = hddsDatanodes.get(i);
+    HddsDatanodeService datanodeService = hddsDatanodes.remove(i);
     stopDatanode(datanodeService);
     // ensure same ports are used across restarts.
     OzoneConfiguration config = datanodeService.getConf();
-    DatanodeDetails dn = datanodeService.getDatanodeDetails();
-    config.setBoolean(DFS_CONTAINER_IPC_RANDOM_PORT, false);
-    config.setBoolean(DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, false);
-    config.setInt(DFS_CONTAINER_IPC_PORT,
-        dn.getPort(STANDALONE).getValue());
-    config.setInt(DFS_CONTAINER_RATIS_IPC_PORT,
-        dn.getPort(RATIS).getValue());
-    config.setInt(DFS_CONTAINER_RATIS_ADMIN_PORT,
-        dn.getPort(RATIS_ADMIN).getValue());
-    config.setInt(DFS_CONTAINER_RATIS_SERVER_PORT,
-        dn.getPort(RATIS_SERVER).getValue());
-    config.setFromObject(conf.getObject(ReplicationConfig.class)
-        .setPort(dn.getPort(REPLICATION).getValue()));
-    hddsDatanodes.remove(i);
     if (waitForDatanode) {
       // wait for node to be removed from SCM healthy node list.
       waitForHddsDatanodeToStop(datanodeService.getDatanodeDetails());
@@ -847,6 +830,7 @@ protected List<HddsDatanodeService> createHddsDatanodes(
       List<HddsDatanodeService> hddsDatanodes = new ArrayList<>();
       for (int i = 0; i < numOfDatanodes; i++) {
         OzoneConfiguration dnConf = new OzoneConfiguration(conf);
+        configureDatanodePorts(dnConf);
         String datanodeBaseDir = path + "/datanode-" + Integer.toString(i);
         Path metaDir = Paths.get(datanodeBaseDir, "meta");
         List<String> dataDirs = new ArrayList<>();
@@ -903,10 +887,14 @@ private void configureLayoutVersionInDatanodes(
     }
 
     protected void configureSCM() {
-      conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
-      conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
-      conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "127.0.0.1:0");
-      conf.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "127.0.0.1:0");
+      conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY,
+          localhostWithFreePort());
+      conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
+          localhostWithFreePort());
+      conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY,
+          localhostWithFreePort());
+      conf.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY,
+          localhostWithFreePort());
       conf.setInt(ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY, numOfScmHandlers);
       conf.set(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT,
           "3s");
@@ -937,24 +925,30 @@ private void configureSCMheartbeat() {
     }
 
     private void configureOM() {
-      conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "127.0.0.1:0");
-      conf.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, "127.0.0.1:0");
+      conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, localhostWithFreePort());
+      conf.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, localhostWithFreePort());
+      conf.set(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY,
+          localhostWithFreePort());
+      conf.setInt(OMConfigKeys.OZONE_OM_RATIS_PORT_KEY, getFreePort());
       conf.setInt(OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY, numOfOmHandlers);
     }
 
     private void configureHddsDatanodes() {
-      conf.set(ScmConfigKeys.HDDS_REST_HTTP_ADDRESS_KEY, "0.0.0.0:0");
-      conf.set(HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
-      conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT,
-          randomContainerPort);
-      conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
-          randomContainerPort);
       conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED,
           enableContainerDatastream);
-      conf.setBoolean(DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT,
-          randomContainerStreamPort);
+    }
 
-      conf.setFromObject(new ReplicationConfig().setPort(0));
+    protected void configureDatanodePorts(ConfigurationTarget conf) {
+      conf.set(ScmConfigKeys.HDDS_REST_HTTP_ADDRESS_KEY,
+          anyHostWithFreePort());
+      conf.set(HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY,
+          anyHostWithFreePort());
+      conf.setInt(DFS_CONTAINER_IPC_PORT, getFreePort());
+      conf.setInt(DFS_CONTAINER_RATIS_IPC_PORT, getFreePort());
+      conf.setInt(DFS_CONTAINER_RATIS_ADMIN_PORT, getFreePort());
+      conf.setInt(DFS_CONTAINER_RATIS_SERVER_PORT, getFreePort());
+      conf.setInt(DFS_CONTAINER_RATIS_DATASTREAM_PORT, getFreePort());
+      conf.setFromObject(new ReplicationConfig().setPort(getFreePort()));
     }
 
     private void configureTrace() {
@@ -984,11 +978,12 @@ protected void configureRecon() throws IOException {
           + "/ozone_recon_derby.db");
       conf.setFromObject(dbConfig);
 
-      conf.set(OZONE_RECON_HTTP_ADDRESS_KEY, "0.0.0.0:0");
-      conf.set(OZONE_RECON_DATANODE_ADDRESS_KEY, "0.0.0.0:0");
+      conf.set(OZONE_RECON_HTTP_ADDRESS_KEY, anyHostWithFreePort());
+      conf.set(OZONE_RECON_DATANODE_ADDRESS_KEY, anyHostWithFreePort());
       conf.set(OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD, "10s");
 
       ConfigurationProvider.setConfiguration(conf);
     }
+
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterProvider.java

@@ -18,20 +18,17 @@
 package org.apache.hadoop.ozone;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
 
@@ -218,18 +215,7 @@ private Thread createClusters() {
         MiniOzoneCluster cluster = null;
         try {
           builder.setClusterId(UUID.randomUUID().toString());
-
-          OzoneConfiguration newConf = new OzoneConfiguration(conf);
-          List<Integer> portList = getFreePortList(4);
-          newConf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY,
-              "127.0.0.1:" + portList.get(0));
-          newConf.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY,
-              "127.0.0.1:" + portList.get(1));
-          newConf.set(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY,
-              "127.0.0.1:" + portList.get(2));
-          newConf.setInt(OMConfigKeys.OZONE_OM_RATIS_PORT_KEY,
-              portList.get(3));
-          builder.setConf(newConf);
+          builder.setConf(new OzoneConfiguration(conf));
 
           cluster = builder.build();
           cluster.waitForClusterToBeReady();
@@ -277,10 +263,4 @@ private void destroyRemainingClusters() {
     createdClusters.clear();
   }
 
-  private List<Integer> getFreePortList(int size) {
-    return org.apache.ratis.util.NetUtils.createLocalServerAddress(size)
-        .stream()
-        .map(inetSocketAddress -> inetSocketAddress.getPort())
-        .collect(Collectors.toList());
-  }
 }