diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java index 728ded031e44..05094b00b427 100644 --- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java +++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.time.Duration; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -102,15 +101,13 @@ public static FailureService of(String serviceName) { @SuppressWarnings("parameternumber") public MiniOzoneChaosCluster(OzoneConfiguration conf, - List ozoneManagers, List scms, - List hddsDatanodes, String omServiceID, - String scmServiceId, String clusterPath, + OMHAService omService, SCMHAService scmService, + List hddsDatanodes, String clusterPath, Set> clazzes) { - super(conf, ozoneManagers, scms, hddsDatanodes, omServiceID, scmServiceId, - clusterPath); + super(conf, omService, scmService, hddsDatanodes, clusterPath, null); this.numDatanodes = getHddsDatanodes().size(); - this.numOzoneManagers = ozoneManagers.size(); - this.numStorageContainerManagers = scms.size(); + this.numOzoneManagers = omService.getServices().size(); + this.numStorageContainerManagers = scmService.getServices().size(); this.failedOmSet = new HashSet<>(); this.failedDnSet = new HashSet<>(); @@ -304,33 +301,21 @@ public MiniOzoneChaosCluster build() throws IOException { initOMRatisConf(); } - List omList; - List scmList; + SCMHAService scmService; + OMHAService omService; try { - if (numOfSCMs > 1) { - scmList = createSCMService(); - } else { - StorageContainerManager scm = createSCM(); - scm.start(); - scmList = Arrays.asList(scm); - } - if (numOfOMs > 1) { - omList = createOMService(); - } else { - OzoneManager om = createOM(); - om.start(); - omList = Arrays.asList(om); - } + scmService = createSCMService(); + omService = createOMService(); } catch (AuthenticationException ex) { throw new IOException("Unable to build MiniOzoneCluster. ", ex); } final List hddsDatanodes = createHddsDatanodes( - scmList, null); + scmService.getActiveServices(), null); MiniOzoneChaosCluster cluster = - new MiniOzoneChaosCluster(conf, omList, scmList, hddsDatanodes, - omServiceId, scmServiceId, path, clazzes); + new MiniOzoneChaosCluster(conf, omService, scmService, hddsDatanodes, + path, clazzes); if (startDataNodes) { cluster.startHddsDatanodes(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java index b9fbde6ffd57..974912a822b7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl; +import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -167,7 +167,7 @@ public static void shutdown() { * @return the leader OM's RPC address in the MiniOzoneHACluster */ private String getLeaderOMNodeAddr() { - MiniOzoneOMHAClusterImpl haCluster = (MiniOzoneOMHAClusterImpl) cluster; + MiniOzoneHAClusterImpl haCluster = (MiniOzoneHAClusterImpl) cluster; OzoneManager omLeader = haCluster.getOMLeader(); Assert.assertNotNull("There should be a leader OM at this point.", omLeader); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index dc200508e7f9..b6d4b31ddd00 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -63,7 +63,7 @@ static Builder newBuilder(OzoneConfiguration conf) { * @return MiniOzoneCluster builder */ static Builder newOMHABuilder(OzoneConfiguration conf) { - return new MiniOzoneOMHAClusterImpl.Builder(conf); + return new MiniOzoneHAClusterImpl.Builder(conf); } static Builder newHABuilder(OzoneConfiguration conf) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index d9b5b3c5dfb5..b2a71076c9d6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -152,22 +152,6 @@ protected MiniOzoneClusterImpl(OzoneConfiguration conf, this.reconServer = reconServer; } - /** - * Creates a new MiniOzoneCluster without the OzoneManager. This is used by - * {@link MiniOzoneOMHAClusterImpl} for starting multiple OzoneManagers. - * - * @param conf - * @param scm - * @param hddsDatanodes - */ - MiniOzoneClusterImpl(OzoneConfiguration conf, StorageContainerManager scm, - List hddsDatanodes, ReconServer reconServer) { - this.conf = conf; - this.scm = scm; - this.hddsDatanodes = hddsDatanodes; - this.reconServer = reconServer; - } - /** * Creates a new MiniOzoneCluster without the OzoneManager and * StorageContainerManager. This is used by diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java index 9bfae2a66a97..e0d061f723dd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java @@ -23,6 +23,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.hadoop.hdds.ExitManager; +import org.apache.hadoop.hdds.conf.ConfigurationTarget; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; @@ -39,22 +40,30 @@ import org.apache.hadoop.ozone.recon.ReconServer; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.ozone.test.GenericTestUtils; +import org.apache.ratis.util.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Closeable; import java.io.IOException; import java.net.BindException; +import java.net.ServerSocket; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Random; +import java.util.PrimitiveIterator; +import java.util.Queue; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Function; -import java.util.stream.Collectors; +import static java.util.Collections.singletonList; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION; +import static org.apache.hadoop.ozone.OzoneTestUtils.reservePorts; import static org.apache.hadoop.ozone.om.OmUpgradeConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION; /** @@ -75,51 +84,27 @@ public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl { private int waitForClusterToBeReadyTimeout = 120000; // 2 min - private static final Random RANDOM = new Random(); private static final int RATIS_RPC_TIMEOUT = 1000; // 1 second - private static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds + public static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds /** * Creates a new MiniOzoneCluster. * * @throws IOException if there is an I/O error */ - @SuppressWarnings("checkstyle:ParameterNumber") public MiniOzoneHAClusterImpl( OzoneConfiguration conf, - List activeOMList, - List inactiveOMList, - List activeSCMList, - List inactiveSCMList, + OMHAService omhaService, + SCMHAService scmhaService, List hddsDatanodes, - String omServiceId, - String scmServiceId, String clusterPath, ReconServer reconServer) { super(conf, hddsDatanodes, reconServer); - omhaService = - new OMHAService(activeOMList, inactiveOMList, omServiceId); - scmhaService = - new SCMHAService(activeSCMList, inactiveSCMList, scmServiceId); + this.omhaService = omhaService; + this.scmhaService = scmhaService; this.clusterMetaPath = clusterPath; } - /** - * Creates a new MiniOzoneCluster with all OMs active. - * This is used by MiniOzoneChaosCluster. - */ - protected MiniOzoneHAClusterImpl( - OzoneConfiguration conf, - List omList, - List scmList, - List hddsDatanodes, - String omServiceId, - String scmServiceId, - String clusterPath) { - this(conf, omList, null, scmList, null, hddsDatanodes, - omServiceId, scmServiceId, clusterPath, null); - } - @Override public String getOMServiceId() { return omhaService.getServiceId(); @@ -155,8 +140,8 @@ public boolean isOMActive(String omNodeId) { return omhaService.isServiceActive(omNodeId); } - public boolean isSCMActive(String scmNodeId) { - return scmhaService.isServiceActive(scmNodeId); + public Iterator getInactiveSCM() { + return scmhaService.inactiveServices(); } public StorageContainerManager getSCM(String scmNodeId) { @@ -309,6 +294,13 @@ public void waitForSCMToBeReady() }, 1000, waitForClusterToBeReadyTimeout); } + @Override + public void shutdown() { + super.shutdown(); + omhaService.releasePorts(); + scmhaService.releasePorts(); + } + @Override public void stop() { for (OzoneManager ozoneManager : this.omhaService.getServices()) { @@ -343,6 +335,29 @@ public void stopOzoneManager(String omNodeId) { omhaService.deactivate(om); } + private static void configureOMPorts(ConfigurationTarget conf, + String omServiceId, String omNodeId, + ReservedPorts omPorts, ReservedPorts omRpcPorts) { + + String omAddrKey = ConfUtils.addKeySuffixes( + OMConfigKeys.OZONE_OM_ADDRESS_KEY, omServiceId, omNodeId); + String omHttpAddrKey = ConfUtils.addKeySuffixes( + OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, omServiceId, omNodeId); + String omHttpsAddrKey = ConfUtils.addKeySuffixes( + OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, omServiceId, omNodeId); + String omRatisPortKey = ConfUtils.addKeySuffixes( + OMConfigKeys.OZONE_OM_RATIS_PORT_KEY, omServiceId, omNodeId); + + PrimitiveIterator.OfInt nodePorts = omPorts.assign(omNodeId); + PrimitiveIterator.OfInt rpcPorts = omRpcPorts.assign(omNodeId); + conf.set(omAddrKey, "127.0.0.1:" + rpcPorts.nextInt()); + conf.set(omHttpAddrKey, "127.0.0.1:" + nodePorts.nextInt()); + conf.set(omHttpsAddrKey, "127.0.0.1:" + nodePorts.nextInt()); + conf.setInt(omRatisPortKey, nodePorts.nextInt()); + + omRpcPorts.release(omNodeId); + } + /** * Builder for configuring the MiniOzoneCluster to run. */ @@ -356,6 +371,15 @@ public static class Builder extends MiniOzoneClusterImpl.Builder { private List activeSCMs = new ArrayList<>(); private List inactiveSCMs = new ArrayList<>(); + // These port reservations are for servers started when the component + // (OM or SCM) is started. These are Ratis, HTTP and HTTPS. We also have + // another set of ports for RPC endpoints, which are started as soon as + // the component is created (in methods called by OzoneManager and + // StorageContainerManager constructors respectively). So we need to manage + // them separately, see initOMHAConfig() and initSCMHAConfig(). + private final ReservedPorts omPorts = new ReservedPorts(3); + private final ReservedPorts scmPorts = new ReservedPorts(3); + /** * Creates a new Builder. * @@ -365,14 +389,6 @@ public Builder(OzoneConfiguration conf) { super(conf); } - public List getActiveOMs() { - return activeOMs; - } - - public List getInactiveOMs() { - return inactiveOMs; - } - @Override public MiniOzoneCluster build() throws IOException { if (numOfActiveOMs > numOfOMs) { @@ -398,11 +414,12 @@ public MiniOzoneCluster build() throws IOException { DefaultMetricsSystem.setMiniClusterMode(true); initializeConfiguration(); initOMRatisConf(); - StorageContainerManager scm; + SCMHAService scmService; + OMHAService omService; ReconServer reconServer = null; try { - createSCMService(); - createOMService(); + scmService = createSCMService(); + omService = createOMService(); if (includeRecon) { configureRecon(); reconServer = new ReconServer(); @@ -413,11 +430,10 @@ public MiniOzoneCluster build() throws IOException { } final List hddsDatanodes = createHddsDatanodes( - activeSCMs, reconServer); + scmService.getActiveServices(), reconServer); MiniOzoneHAClusterImpl cluster = new MiniOzoneHAClusterImpl(conf, - activeOMs, inactiveOMs, activeSCMs, inactiveSCMs, - hddsDatanodes, omServiceId, scmServiceId, path, reconServer); + omService, scmService, hddsDatanodes, path, reconServer); if (startDataNodes) { cluster.startHddsDatanodes(); @@ -458,18 +474,21 @@ protected void initOMRatisConf() { /** * Start OM service with multiple OMs. */ - protected List createOMService() throws IOException, + protected OMHAService createOMService() throws IOException, AuthenticationException { + if (omServiceId == null) { + OzoneManager om = createOM(); + om.start(); + return new OMHAService(singletonList(om), null, null, null); + } List omList = Lists.newArrayList(); int retryCount = 0; - int basePort; while (true) { try { - basePort = 10000 + RANDOM.nextInt(1000) * 4; - initOMHAConfig(basePort); + initOMHAConfig(); for (int i = 1; i<= numOfOMs; i++) { // Set nodeId @@ -520,26 +539,30 @@ protected List createOMService() throws IOException, omList.clear(); ++retryCount; LOG.info("MiniOzoneHACluster port conflicts, retried {} times", - retryCount); + retryCount, e); } } - return omList; + return new OMHAService(activeOMs, inactiveOMs, omServiceId, omPorts); } /** * Start OM service with multiple OMs. */ - protected List createSCMService() + protected SCMHAService createSCMService() throws IOException, AuthenticationException { + if (scmServiceId == null) { + StorageContainerManager scm = createSCM(); + scm.start(); + return new SCMHAService(singletonList(scm), null, null, null); + } + List scmList = Lists.newArrayList(); int retryCount = 0; - int basePort = 12000; while (true) { try { - basePort = 12000 + RANDOM.nextInt(1000) * 4; - initSCMHAConfig(basePort); + initSCMHAConfig(); for (int i = 1; i<= numOfSCMs; i++) { // Set nodeId @@ -575,11 +598,11 @@ protected List createSCMService() scm.start(); activeSCMs.add(scm); LOG.info("Started SCM RPC server at {}", - scm.getClientProtocolServer()); + scm.getClientRpcAddress()); } else { inactiveSCMs.add(scm); LOG.info("Intialized SCM at {}. This SCM is currently " - + "inactive (not running).", scm.getClientProtocolServer()); + + "inactive (not running).", scm.getClientRpcAddress()); } } @@ -590,21 +613,22 @@ protected List createSCMService() scm.stop(); scm.join(); LOG.info("Stopping StorageContainerManager server at {}", - scm.getClientProtocolServer()); + scm.getClientRpcAddress()); } scmList.clear(); ++retryCount; LOG.info("MiniOzoneHACluster port conflicts, retried {} times", - retryCount); + retryCount, e); } } - return scmList; + + return new SCMHAService(activeSCMs, inactiveSCMs, scmServiceId, scmPorts); } /** * Initialize HA related configurations. */ - private void initSCMHAConfig(int basePort) throws IOException { + private void initSCMHAConfig() { // Set configurations required for starting OM HA service, because that // is the serviceID being passed to start Ozone HA cluster. // Here setting internal service and OZONE_OM_SERVICE_IDS_KEY, in this @@ -616,11 +640,14 @@ private void initSCMHAConfig(int basePort) throws IOException { StringBuilder scmNodesKeyValue = new StringBuilder(); StringBuilder scmNames = new StringBuilder(); - int port = basePort; + scmPorts.reserve(numOfSCMs); + ReservedPorts scmRpcPorts = new ReservedPorts(4); + scmRpcPorts.reserve(numOfSCMs); - for (int i = 1; i <= numOfSCMs; i++, port+=10) { + for (int i = 1; i <= numOfSCMs; i++) { String scmNodeId = SCM_NODE_ID_PREFIX + i; scmNodesKeyValue.append(",").append(scmNodeId); + String scmAddrKey = ConfUtils.addKeySuffixes( ScmConfigKeys.OZONE_SCM_ADDRESS_KEY, scmServiceId, scmNodeId); String scmHttpAddrKey = ConfUtils.addKeySuffixes( @@ -641,18 +668,32 @@ private void initSCMHAConfig(int basePort) throws IOException { String scmGrpcPortKey = ConfUtils.addKeySuffixes( ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY, scmServiceId, scmNodeId); + PrimitiveIterator.OfInt nodePorts = scmPorts.assign(scmNodeId); + PrimitiveIterator.OfInt rpcPorts = scmRpcPorts.assign(scmNodeId); conf.set(scmAddrKey, "127.0.0.1"); - conf.set(scmHttpAddrKey, "127.0.0.1:" + (port + 2)); - conf.set(scmHttpsAddrKey, "127.0.0.1:" + (port + 3)); - conf.setInt(scmRatisPortKey, port + 4); - //conf.setInt("ozone.scm.ha.ratis.bind.port", port + 4); - conf.set(dnPortKey, "127.0.0.1:" + (port + 5)); - conf.set(blockClientKey, "127.0.0.1:" + (port + 6)); - conf.set(ssClientKey, "127.0.0.1:" + (port + 7)); - conf.setInt(scmGrpcPortKey, port + 8); - scmNames.append(",").append("localhost:" + (port + 5)); - conf.set(ScmConfigKeys. - OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, "127.0.0.1:" + (port + 6)); + conf.set(scmHttpAddrKey, "127.0.0.1:" + nodePorts.nextInt()); + conf.set(scmHttpsAddrKey, "127.0.0.1:" + nodePorts.nextInt()); + + int ratisPort = nodePorts.nextInt(); + conf.setInt(scmRatisPortKey, ratisPort); + //conf.setInt("ozone.scm.ha.ratis.bind.port", ratisPort); + + int dnPort = rpcPorts.nextInt(); + conf.set(dnPortKey, "127.0.0.1:" + dnPort); + scmNames.append(",localhost:").append(dnPort); + + conf.set(ssClientKey, "127.0.0.1:" + rpcPorts.nextInt()); + conf.setInt(scmGrpcPortKey, rpcPorts.nextInt()); + + int blockPort = rpcPorts.nextInt(); + conf.set(blockClientKey, "127.0.0.1:" + blockPort); + conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, + "127.0.0.1:" + blockPort); + + if (i <= numOfActiveSCMs) { + scmPorts.release(scmNodeId); + } + scmRpcPorts.release(scmNodeId); } conf.set(scmNodesKey, scmNodesKeyValue.substring(1)); @@ -662,7 +703,7 @@ private void initSCMHAConfig(int basePort) throws IOException { /** * Initialize HA related configurations. */ - private void initOMHAConfig(int basePort) throws IOException { + private void initOMHAConfig() { // Set configurations required for starting OM HA service, because that // is the serviceID being passed to start Ozone HA cluster. // Here setting internal service and OZONE_OM_SERVICE_IDS_KEY, in this @@ -673,25 +714,19 @@ private void initOMHAConfig(int basePort) throws IOException { OMConfigKeys.OZONE_OM_NODES_KEY, omServiceId); List omNodeIds = new ArrayList<>(); - int port = basePort; + omPorts.reserve(numOfOMs); + ReservedPorts omRpcPorts = new ReservedPorts(1); + omRpcPorts.reserve(numOfOMs); - for (int i = 1; i <= numOfOMs; i++, port+=6) { + for (int i = 1; i <= numOfOMs; i++) { String omNodeId = OM_NODE_ID_PREFIX + i; omNodeIds.add(omNodeId); - String omAddrKey = ConfUtils.addKeySuffixes( - OMConfigKeys.OZONE_OM_ADDRESS_KEY, omServiceId, omNodeId); - String omHttpAddrKey = ConfUtils.addKeySuffixes( - OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, omServiceId, omNodeId); - String omHttpsAddrKey = ConfUtils.addKeySuffixes( - OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, omServiceId, omNodeId); - String omRatisPortKey = ConfUtils.addKeySuffixes( - OMConfigKeys.OZONE_OM_RATIS_PORT_KEY, omServiceId, omNodeId); - - conf.set(omAddrKey, "127.0.0.1:" + port); - conf.set(omHttpAddrKey, "127.0.0.1:" + (port + 2)); - conf.set(omHttpsAddrKey, "127.0.0.1:" + (port + 3)); - conf.setInt(omRatisPortKey, port + 4); + configureOMPorts(conf, omServiceId, omNodeId, omPorts, omRpcPorts); + + if (i <= numOfActiveOMs) { + omPorts.release(omNodeId); + } } conf.set(omNodesKey, String.join(",", omNodeIds)); @@ -729,9 +764,8 @@ public void bootstrapOzoneManager(String omNodeId, while (true) { try { - List portSet = getFreePortList(4); OzoneConfiguration newConf = addNewOMToConfig(getOMServiceId(), - omNodeId, portSet); + omNodeId); if (updateConfigs) { updateOMConfigs(newConf); @@ -755,7 +789,7 @@ public void bootstrapOzoneManager(String omNodeId, e.getCause() instanceof BindException) { ++retryCount; LOG.info("MiniOzoneHACluster port conflicts, retried {} times", - retryCount); + retryCount, e); } else { throw e; } @@ -772,30 +806,19 @@ public void bootstrapOzoneManager(String omNodeId, * Set the configs for new OMs. */ private OzoneConfiguration addNewOMToConfig(String omServiceId, - String omNodeId, List portList) { + String omNodeId) { + + ReservedPorts omPorts = omhaService.getPorts(); + omPorts.reserve(1); + ReservedPorts omRpcPorts = new ReservedPorts(1); + omRpcPorts.reserve(1); OzoneConfiguration newConf = new OzoneConfiguration(getConf()); + configureOMPorts(newConf, omServiceId, omNodeId, omPorts, omRpcPorts); + String omNodesKey = ConfUtils.addKeySuffixes( OMConfigKeys.OZONE_OM_NODES_KEY, omServiceId); - StringBuilder omNodesKeyValue = new StringBuilder(); - omNodesKeyValue.append(newConf.get(omNodesKey)) - .append(",").append(omNodeId); - - String omAddrKey = ConfUtils.addKeySuffixes( - OMConfigKeys.OZONE_OM_ADDRESS_KEY, omServiceId, omNodeId); - String omHttpAddrKey = ConfUtils.addKeySuffixes( - OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, omServiceId, omNodeId); - String omHttpsAddrKey = ConfUtils.addKeySuffixes( - OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, omServiceId, omNodeId); - String omRatisPortKey = ConfUtils.addKeySuffixes( - OMConfigKeys.OZONE_OM_RATIS_PORT_KEY, omServiceId, omNodeId); - - newConf.set(omAddrKey, "127.0.0.1:" + portList.get(0)); - newConf.set(omHttpAddrKey, "127.0.0.1:" + portList.get(1)); - newConf.set(omHttpsAddrKey, "127.0.0.1:" + portList.get(2)); - newConf.setInt(omRatisPortKey, portList.get(3)); - - newConf.set(omNodesKey, omNodesKeyValue.toString()); + newConf.set(omNodesKey, newConf.get(omNodesKey) + "," + omNodeId); return newConf; } @@ -838,9 +861,7 @@ private OzoneManager bootstrapNewOM(String nodeId, OzoneConfiguration newConf, ExitManagerForOM exitManager = new ExitManagerForOM(this, nodeId); om.setExitManagerForTesting(exitManager); omhaService.addInstance(om, false); - - om.start(); - omhaService.activate(om); + startInactiveOM(nodeId); return om; } @@ -902,6 +923,7 @@ static class MiniOzoneHAService { private List services; private String serviceId; private String serviceName; + private final ReservedPorts ports; // Active services s denote OM/SCM services which are up and running private List activeServices; @@ -911,9 +933,10 @@ static class MiniOzoneHAService { private Function serviceIdProvider; MiniOzoneHAService(String name, List activeList, - List inactiveList, String serviceId, - Function idProvider) { + List inactiveList, String serviceId, + ReservedPorts ports, Function idProvider) { this.serviceName = name; + this.ports = ports != null ? ports : new ReservedPorts(0); this.serviceMap = Maps.newHashMap(); this.serviceIdProvider = idProvider; if (activeList != null) { @@ -945,6 +968,10 @@ public List getServices() { return services; } + public void releasePorts() { + ports.releaseAll(); + } + public List getActiveServices() { return activeServices; } @@ -958,6 +985,8 @@ public void addInstance(Type t, boolean isActive) { serviceMap.put(serviceIdProvider.apply(t), t); if (isActive) { activeServices.add(t); + } else { + inactiveServices.add(t); } } @@ -975,6 +1004,10 @@ public boolean isServiceActive(String id) { return activeServices.contains(serviceMap.get(id)); } + public Iterator inactiveServices() { + return new ArrayList<>(inactiveServices).iterator(); + } + public Type getServiceByIndex(int index) { return this.services.get(index); } @@ -989,17 +1022,22 @@ public void startInactiveService(String id, if (!inactiveServices.contains(service)) { throw new IOException(serviceName + " is already active."); } else { + ports.release(id); serviceStarter.execute(service); activeServices.add(service); inactiveServices.remove(service); } } + + public ReservedPorts getPorts() { + return ports; + } } static class OMHAService extends MiniOzoneHAService { OMHAService(List activeList, List inactiveList, - String serviceId) { - super("OM", activeList, inactiveList, serviceId, + String serviceId, ReservedPorts omPorts) { + super("OM", activeList, inactiveList, serviceId, omPorts, OzoneManager::getOMNodeId); } } @@ -1007,10 +1045,10 @@ static class OMHAService extends MiniOzoneHAService { static class SCMHAService extends MiniOzoneHAService { SCMHAService(List activeList, - List inactiveList, - String serviceId) { + List inactiveList, + String serviceId, ReservedPorts scmPorts) { super("SCM", activeList, inactiveList, serviceId, - StorageContainerManager::getScmId); + scmPorts, StorageContainerManager::getSCMNodeId); } } @@ -1022,13 +1060,6 @@ public StorageContainerManager getStorageContainerManager() { return getStorageContainerManagers().get(0); } - private List getFreePortList(int size) { - return org.apache.ratis.util.NetUtils.createLocalServerAddress(size) - .stream() - .map(inetSocketAddress -> inetSocketAddress.getPort()) - .collect(Collectors.toList()); - } - private static final class ExitManagerForOM extends ExitManager { private MiniOzoneHAClusterImpl cluster; @@ -1055,4 +1086,77 @@ public void exitSystem(int status, String message, Logger log) throw new IOException(message); } } + + /** + * Reserves a number of ports for services. + */ + private static class ReservedPorts { + + private final Queue allPorts = new LinkedList<>(); + private final Map> assignedPorts = + new HashMap<>(); + private final int portsPerNode; + + ReservedPorts(int portsPerNode) { + this.portsPerNode = portsPerNode; + } + + /** + * Reserve {@code portsPerNode * nodes} ports by binding server sockets + * to random free ports. The sockets are kept open until + * {@link #release(String)} or {@link #releaseAll} is called. + */ + public void reserve(int nodes) { + Preconditions.checkState(allPorts.isEmpty()); + allPorts.addAll(reservePorts(portsPerNode * nodes)); + } + + /** + * Assign {@code portsPerNode} ports to a service identified by {@code id}. + * This set of ports should be released right before starting the service + * by calling {@link #release(String)}. + * + * @return iterator of the ports assigned + */ + public PrimitiveIterator.OfInt assign(String id) { + Preconditions.checkState(allPorts.size() >= portsPerNode); + List nodePorts = new LinkedList<>(); + for (int i = 0; i < portsPerNode; i++) { + nodePorts.add(allPorts.remove()); + } + assignedPorts.put(id, nodePorts); + LOG.debug("assign ports for {}: {}", id, nodePorts); + + return nodePorts.stream().mapToInt(ServerSocket::getLocalPort).iterator(); + } + + /** + * Release the ports assigned to the service identified by {@code id}. + * + * This closes the server sockets, making the same ports available for + * the service. Note: there is a race condition with other processes + * running on the host, but that's OK since this is for tests. + * + * If no ports are assigned to the service, this is a no-op. + */ + public void release(String id) { + List ports = assignedPorts.remove(id); + LOG.debug("release ports for {}: {}", id, ports); + if (ports != null) { + IOUtils.cleanup(LOG, ports.toArray(new Closeable[0])); + } + } + + /** + * Release all reserved ports, assigned or not. + */ + public void releaseAll() { + IOUtils.cleanup(LOG, allPorts.toArray(new Closeable[0])); + allPorts.clear(); + + for (String id : new ArrayList<>(assignedPorts.keySet())) { + release(id); + } + } + } } \ No newline at end of file diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneOMHAClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneOMHAClusterImpl.java deleted file mode 100644 index 2f2b77341481..000000000000 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneOMHAClusterImpl.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.recon.ReconServer; -import org.apache.hadoop.security.authentication.client.AuthenticationException; - -import java.io.IOException; -import java.util.List; -import java.util.Collections; - -/** - * MiniOzoneOMHAClusterImpl creates a complete in-process Ozone cluster - * with OM HA suitable for running tests. The cluster consists of a set of - * OzoneManagers, StorageContainerManager and multiple DataNodes. - */ -public final class MiniOzoneOMHAClusterImpl extends MiniOzoneHAClusterImpl { - public static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds - - /** - * Creates a new MiniOzoneOMHACluster. - * - * @throws IOException if there is an I/O error - */ - @SuppressWarnings("checkstyle:ParameterNumber") - private MiniOzoneOMHAClusterImpl( - OzoneConfiguration conf, - List activeOMList, - List inactiveOMList, - StorageContainerManager scm, - List hddsDatanodes, - String omServiceId, - String metaPath, - ReconServer reconServer) { - super(conf, activeOMList, inactiveOMList, Collections.singletonList(scm), - null, hddsDatanodes, omServiceId, null, metaPath, reconServer); - } - - /** - * Builder for configuring the MiniOzoneCluster to run. - */ - public static class Builder extends MiniOzoneHAClusterImpl.Builder { - - /** - * Creates a new Builder. - * - * @param conf configuration - */ - public Builder(OzoneConfiguration conf) { - super(conf); - } - - @Override - public MiniOzoneCluster build() throws IOException { - if (numOfActiveOMs > numOfOMs) { - throw new IllegalArgumentException("Number of active OMs cannot be " + - "more than the total number of OMs"); - } - - // If num of ActiveOMs is not set, set it to numOfOMs. - if (numOfActiveOMs == ACTIVE_OMS_NOT_SET) { - numOfActiveOMs = numOfOMs; - } - - DefaultMetricsSystem.setMiniClusterMode(true); - initializeConfiguration(); - initOMRatisConf(); - StorageContainerManager scm; - ReconServer reconServer = null; - try { - scm = createSCM(); - scm.start(); - createOMService(); - if (includeRecon) { - configureRecon(); - reconServer = new ReconServer(); - reconServer.execute(new String[] {}); - } - } catch (AuthenticationException ex) { - throw new IOException("Unable to build MiniOzoneCluster. ", ex); - } - - final List hddsDatanodes = createHddsDatanodes( - Collections.singletonList(scm), reconServer); - - MiniOzoneClusterImpl cluster = new MiniOzoneOMHAClusterImpl(conf, - getActiveOMs(), getInactiveOMs(), scm, hddsDatanodes, - omServiceId, path, reconServer); - - if (startDataNodes) { - cluster.startHddsDatanodes(); - } - return cluster; - } - } -} \ No newline at end of file diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java index 23ac5e104e4c..e4cb1a153165 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java @@ -17,7 +17,13 @@ */ package org.apache.hadoop.ozone; +import java.io.Closeable; import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hdds.client.BlockID; @@ -32,6 +38,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.ozone.test.LambdaTestUtils.VoidCallable; +import org.apache.ratis.util.IOUtils; import org.apache.ratis.util.function.CheckedConsumer; import org.junit.Assert; @@ -146,4 +153,20 @@ public static void expectOmException( Assert.assertEquals(code, ex.getResult()); } } + + public static List reservePorts(int count) { + List sockets = new ArrayList<>(count); + try { + for (int i = 0; i < count; i++) { + ServerSocket s = new ServerSocket(); + sockets.add(s); + s.setReuseAddress(true); + s.bind(new InetSocketAddress(InetAddress.getByName(null), 0), 1); + } + } catch (IOException e) { + IOUtils.cleanup(null, sockets.toArray(new Closeable[0])); + throw new UncheckedIOException(e); + } + return sockets; + } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java index 61225a9cac6c..44b635f36da3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneOMHACluster.java @@ -39,11 +39,11 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS; /** - * This class tests MiniOzoneOMHAClusterImpl. + * This class tests MiniOzoneHAClusterImpl. */ public class TestMiniOzoneOMHACluster { - private MiniOzoneOMHAClusterImpl cluster = null; + private MiniOzoneHAClusterImpl cluster = null; private OzoneConfiguration conf; private String clusterId; private String scmId; @@ -71,7 +71,7 @@ public void init() throws Exception { conf.set(OzoneConfigKeys.OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD); conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); - cluster = (MiniOzoneOMHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) + cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) .setClusterId(clusterId) .setScmId(scmId) .setOMServiceId(omServiceId) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index cd3aee41a95a..42bfd11da469 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl; +import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -58,7 +58,7 @@ @Timeout(500) public class TestOMRatisSnapshots { - private MiniOzoneOMHAClusterImpl cluster = null; + private MiniOzoneHAClusterImpl cluster = null; private ObjectStore objectStore; private OzoneConfiguration conf; private String clusterId; @@ -89,7 +89,7 @@ public void init() throws Exception { conf.setLong( OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY, SNAPSHOT_THRESHOLD); - cluster = (MiniOzoneOMHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) + cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) .setClusterId(clusterId) .setScmId(scmId) .setOMServiceId("om-service-test1") diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java index b2c466282874..5a109f7975fc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMUpgradeFinalization.java @@ -37,7 +37,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl; +import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; @@ -67,7 +67,7 @@ public class TestOMUpgradeFinalization { */ @Rule public Timeout timeout = new Timeout(300000); - private MiniOzoneOMHAClusterImpl cluster; + private MiniOzoneHAClusterImpl cluster; private OzoneManager ozoneManager; private ClientProtocol clientProtocol; private int fromLayoutVersion; @@ -100,7 +100,7 @@ public void setup() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); String omServiceId = UUID.randomUUID().toString(); - cluster = (MiniOzoneOMHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) + cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) .setClusterId(UUID.randomUUID().toString()) .setScmId(UUID.randomUUID().toString()) .setOMServiceId(omServiceId) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java index 6118a5ecaf48..b3af2f990c84 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl; +import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -69,7 +69,7 @@ */ public abstract class TestOzoneManagerHA { - private static MiniOzoneOMHAClusterImpl cluster = null; + private static MiniOzoneHAClusterImpl cluster = null; private static MiniOzoneCluster.Builder clusterBuilder = null; private static ObjectStore objectStore; private static OzoneConfiguration conf; @@ -91,7 +91,7 @@ public abstract class TestOzoneManagerHA { @Rule public Timeout timeout = Timeout.seconds(300); - public MiniOzoneOMHAClusterImpl getCluster() { + public MiniOzoneHAClusterImpl getCluster() { return cluster; } @@ -180,7 +180,7 @@ public static void init() throws Exception { .setOmId(omId) .setNumOfOzoneManagers(numOfOMs); - cluster = (MiniOzoneOMHAClusterImpl) clusterBuilder.build(); + cluster = (MiniOzoneHAClusterImpl) clusterBuilder.build(); cluster.waitForClusterToBeReady(); objectStore = OzoneClientFactory.getRpcClient(omServiceId, conf) .getObjectStore(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java index 8c085fb26604..fc4fa36eaf3d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAMetadataOnly.java @@ -62,7 +62,7 @@ import java.util.Iterator; import java.util.UUID; -import static org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl.NODE_FAILURE_TIMEOUT; +import static org.apache.hadoop.ozone.MiniOzoneHAClusterImpl.NODE_FAILURE_TIMEOUT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT; import static org.apache.ratis.metrics.RatisMetrics.RATIS_APPLICATION_NAME_METRICS; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java index a0c014a748d0..7dbacc455f54 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java @@ -43,7 +43,7 @@ import java.util.UUID; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl.NODE_FAILURE_TIMEOUT; +import static org.apache.hadoop.ozone.MiniOzoneHAClusterImpl.NODE_FAILURE_TIMEOUT; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DIRECTORY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_ALREADY_EXISTS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithFailover.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithFailover.java index ecbb6e80f997..a31ed2eb4732 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithFailover.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithFailover.java @@ -21,7 +21,7 @@ import org.junit.Assert; import org.junit.Test; -import static org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl.NODE_FAILURE_TIMEOUT; +import static org.apache.hadoop.ozone.MiniOzoneHAClusterImpl.NODE_FAILURE_TIMEOUT; /** * Test Ozone Manager operation in distributed handler scenario with failover. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java index c35b45bef033..4db0e8ef8a35 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl; +import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; @@ -47,7 +47,7 @@ */ public class TestOzoneManagerSnapshotProvider { - private MiniOzoneOMHAClusterImpl cluster = null; + private MiniOzoneHAClusterImpl cluster = null; private ObjectStore objectStore; private OzoneConfiguration conf; private String clusterId; @@ -69,7 +69,7 @@ public void init() throws Exception { omServiceId = "om-service-test1"; conf.setBoolean(OMConfigKeys.OZONE_OM_HTTP_ENABLED_KEY, true); conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true); - cluster = (MiniOzoneOMHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) + cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) .setClusterId(clusterId) .setScmId(scmId) .setOMServiceId(omServiceId) @@ -140,4 +140,4 @@ private long getDownloadedSnapshotIndex(DBCheckpoint dbCheckpoint) return trxnInfoFromCheckpoint.getTransactionIndex(); } -} \ No newline at end of file +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java index 1d3400077fd9..540f17428c40 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl; +import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; @@ -55,7 +55,7 @@ public class TestReconWithOzoneManagerHA { @Rule public Timeout timeout = Timeout.seconds(300);; - private MiniOzoneOMHAClusterImpl cluster; + private MiniOzoneHAClusterImpl cluster; private ObjectStore objectStore; private static final String OM_SERVICE_ID = "omService1"; private static final String VOL_NAME = "testrecon"; @@ -70,7 +70,7 @@ public void setup() throws Exception { dbConf.setSyncOption(true); conf.setFromObject(dbConf); - cluster = (MiniOzoneOMHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) + cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newOMHABuilder(conf) .setClusterId(UUID.randomUUID().toString()) .setScmId(UUID.randomUUID().toString()) .setOMServiceId(OM_SERVICE_ID) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java index e9a7cf337441..1932b66fa4db 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMInstallSnapshotWithHA.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.UUID; @@ -120,7 +121,7 @@ public void testInstallSnapshot() throws Exception { StorageContainerManager leaderSCM = getLeader(cluster); Assert.assertNotNull(leaderSCM); // Find the inactive SCM - String followerId = getInactiveSCM(cluster).getScmId(); + String followerId = getInactiveSCM(cluster).getSCMNodeId(); StorageContainerManager followerSCM = cluster.getSCM(followerId); // Do some transactions so that the log index increases @@ -159,7 +160,7 @@ public void testInstallSnapshot() throws Exception { public void testInstallOldCheckpointFailure() throws Exception { // Get the leader SCM StorageContainerManager leaderSCM = getLeader(cluster); - String followerId = getInactiveSCM(cluster).getScmId(); + String followerId = getInactiveSCM(cluster).getSCMNodeId(); // Find the inactive SCM StorageContainerManager followerSCM = cluster.getSCM(followerId); @@ -213,7 +214,7 @@ public void testInstallOldCheckpointFailure() throws Exception { public void testInstallCorruptedCheckpointFailure() throws Exception { StorageContainerManager leaderSCM = getLeader(cluster); // Find the inactive SCM - String followerId = getInactiveSCM(cluster).getScmId(); + String followerId = getInactiveSCM(cluster).getSCMNodeId(); StorageContainerManager followerSCM = cluster.getSCM(followerId); // Do some transactions so that the log index increases writeToIncreaseLogIndex(leaderSCM, 100); @@ -316,13 +317,10 @@ static StorageContainerManager getLeader(MiniOzoneHAClusterImpl impl) { return null; } - static StorageContainerManager getInactiveSCM(MiniOzoneHAClusterImpl impl) { - for (StorageContainerManager scm : impl.getStorageContainerManagers()) { - if (!impl.isSCMActive(scm.getScmId())) { - return scm; - } - } - return null; + private static StorageContainerManager getInactiveSCM( + MiniOzoneHAClusterImpl cluster) { + Iterator inactiveScms = cluster.getInactiveSCM(); + return inactiveScms.hasNext() ? inactiveScms.next() : null; } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index d63b3034e91c..f5259422eebf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@ -38,7 +38,7 @@ import org.apache.hadoop.fs.ozone.OzoneFsShell; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl; +import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -241,7 +241,7 @@ private void executeWithError(OzoneShell shell, String[] args, * @return the leader OM's Node ID in the MiniOzoneHACluster. */ private String getLeaderOMNodeId() { - MiniOzoneOMHAClusterImpl haCluster = (MiniOzoneOMHAClusterImpl) cluster; + MiniOzoneHAClusterImpl haCluster = (MiniOzoneHAClusterImpl) cluster; OzoneManager omLeader = haCluster.getOMLeader(); Assert.assertNotNull("There should be a leader OM at this point.", omLeader);