diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index aa082e8f0914..05ad18e9ffcf 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -27,6 +27,7 @@ import java.net.UnknownHostException; import java.nio.file.Path; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -37,6 +38,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.conf.ConfigurationException; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; @@ -56,6 +58,9 @@ import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY; import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; @@ -95,53 +100,58 @@ private HddsUtils() { * * @return Target {@code InetSocketAddress} for the SCM client endpoint. */ - public static InetSocketAddress getScmAddressForClients( + public static Collection getScmAddressForClients( ConfigurationSource conf) { - Optional host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); - if (!host.isPresent()) { - // Fallback to Ozone SCM name - host = Optional.of(getSingleSCMAddress(conf).getHostName()); - } + if (SCMHAUtils.getScmServiceId(conf) != null) { + List scmNodeInfoList = SCMNodeInfo.buildNodeInfo(conf); + Collection scmAddressList = + new HashSet<>(scmNodeInfoList.size()); + for (SCMNodeInfo scmNodeInfo : scmNodeInfoList) { + if (scmNodeInfo.getScmClientAddress() == null) { + throw new ConfigurationException("Ozone scm client address is not " + + "set for SCM service-id " + scmNodeInfo.getServiceId() + + "node-id" + scmNodeInfo.getNodeId()); + } + scmAddressList.add( + NetUtils.createSocketAddr(scmNodeInfo.getScmClientAddress())); + } + return scmAddressList; + } else { + String address = conf.getTrimmed(OZONE_SCM_CLIENT_ADDRESS_KEY); + int port = -1; + + if (address == null) { + // fall back to ozone.scm.names for non-ha + Collection scmAddresses = + conf.getTrimmedStringCollection(OZONE_SCM_NAMES); + + if (scmAddresses.isEmpty()) { + throw new ConfigurationException("Ozone scm client address is not " + + "set. Configure one of these config " + + OZONE_SCM_CLIENT_ADDRESS_KEY + ", " + OZONE_SCM_NAMES); + } - final int port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY) - .orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT); + if (scmAddresses.size() > 1) { + throw new ConfigurationException("For non-HA SCM " + OZONE_SCM_NAMES + + " should be set with single address"); + } - return NetUtils.createSocketAddr(host.get() + ":" + port); - } + address = scmAddresses.iterator().next(); - /** - * Retrieve the socket address that should be used by clients to connect - * to the SCM for block service. If - * {@link ScmConfigKeys#OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY} is not defined - * then {@link ScmConfigKeys#OZONE_SCM_CLIENT_ADDRESS_KEY} is used. If neither - * is defined then {@link ScmConfigKeys#OZONE_SCM_NAMES} is used. - * - * @return Target {@code InetSocketAddress} for the SCM block client endpoint. - * @throws IllegalArgumentException if configuration is not defined. - */ - public static InetSocketAddress getScmAddressForBlockClients( - ConfigurationSource conf) { - Optional host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); + port = conf.getInt(OZONE_SCM_CLIENT_PORT_KEY, + OZONE_SCM_CLIENT_PORT_DEFAULT); + } else { + port = getHostPort(address) + .orElse(conf.getInt(OZONE_SCM_CLIENT_PORT_KEY, + OZONE_SCM_CLIENT_PORT_DEFAULT)); + } - if (!host.isPresent()) { - // Fallback to Ozone SCM name - host = Optional.of(getSingleSCMAddress(conf).getHostName()); + return Collections.singletonList( + NetUtils.createSocketAddr(getHostName(address).get() + ":" + port)); } - - final int port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY) - .orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT); - - return NetUtils.createSocketAddr(host.get() + ":" + port); } - - /** * Retrieve the hostname, trying the supplied config keys in order. * Each config value may be absent, or if present in the format @@ -231,7 +241,7 @@ public static OptionalInt getPortNumberFromConfigKeys( * @return A collection of SCM addresses * @throws IllegalArgumentException If the configuration is invalid */ - public static Collection getSCMAddresses( + public static Collection getSCMAddressForDatanodes( ConfigurationSource conf) { // First check HA style config, if not defined fall back to OZONE_SCM_NAMES @@ -300,22 +310,6 @@ public static InetSocketAddress getReconAddresses( return NetUtils.createSocketAddr(hostname.get(), port); } - /** - * Retrieve the address of the only SCM (as currently multiple ones are not - * supported). - * - * @return SCM address - * @throws IllegalArgumentException if {@code conf} has more than one SCM - * address or it has none - */ - public static InetSocketAddress getSingleSCMAddress( - ConfigurationSource conf) { - Collection singleton = getSCMAddresses(conf); - // Preconditions.checkArgument(singleton.size() == 1, - // MULTIPLE_SCM_NOT_YET_SUPPORTED); - return singleton.iterator().next(); - } - /** * Returns the hostname for this datanode. If the hostname is not * explicitly configured in the given config, then it is determined diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java index f3191ea41327..a5e443a598d3 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java @@ -40,6 +40,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NODES_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT; @@ -141,7 +142,7 @@ public static List buildNodeInfo(ConfigurationSource conf) { String scmDatanodeAddress = getHostNameFromConfigKeys(conf, OZONE_SCM_DATANODE_ADDRESS_KEY, - OZONE_SCM_CLIENT_ADDRESS_KEY).orElse(null); + OZONE_SCM_CLIENT_ADDRESS_KEY, OZONE_SCM_NAMES).orElse(null); int scmBlockClientPort = getPortNumberFromConfigKeys(conf, OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY) @@ -199,7 +200,7 @@ private static int getPort(ConfigurationSource conf, return port.getAsInt(); } else { return conf.getInt(ConfUtils.addKeySuffixes(portKey, scmServiceId, - scmNodeId), defaultPort); + scmNodeId), conf.getInt(portKey, defaultPort)); } } diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java index b97c98ce5113..487140d1cf6a 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java @@ -32,7 +32,7 @@ import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.hadoop.test.LambdaTestUtils; -import static org.apache.hadoop.hdds.HddsUtils.getSCMAddresses; +import static org.apache.hadoop.hdds.HddsUtils.getSCMAddressForDatanodes; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; @@ -88,7 +88,7 @@ public void testGetSCMAddresses() { // Verify valid IP address setup conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "1.2.3.4"); - addresses = getSCMAddresses(conf); + addresses = getSCMAddressForDatanodes(conf); assertThat(addresses.size(), is(1)); addr = addresses.iterator().next(); assertThat(addr.getHostName(), is("1.2.3.4")); @@ -96,7 +96,7 @@ public void testGetSCMAddresses() { // Verify valid hostname setup conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1"); - addresses = getSCMAddresses(conf); + addresses = getSCMAddressForDatanodes(conf); assertThat(addresses.size(), is(1)); addr = addresses.iterator().next(); assertThat(addr.getHostName(), is("scm1")); @@ -104,7 +104,7 @@ public void testGetSCMAddresses() { // Verify valid hostname and port conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234"); - addresses = getSCMAddresses(conf); + addresses = getSCMAddressForDatanodes(conf); assertThat(addresses.size(), is(1)); addr = addresses.iterator().next(); assertThat(addr.getHostName(), is("scm1")); @@ -118,7 +118,7 @@ public void testGetSCMAddresses() { // Verify multiple hosts and port conf.setStrings( ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234,scm2:2345,scm3:3456"); - addresses = getSCMAddresses(conf); + addresses = getSCMAddressForDatanodes(conf); assertThat(addresses.size(), is(3)); it = addresses.iterator(); HashMap expected1 = new HashMap<>(hostsAndPorts); @@ -132,7 +132,7 @@ public void testGetSCMAddresses() { // Verify names with spaces conf.setStrings( ScmConfigKeys.OZONE_SCM_NAMES, " scm1:1234, scm2:2345 , scm3:3456 "); - addresses = getSCMAddresses(conf); + addresses = getSCMAddressForDatanodes(conf); assertThat(addresses.size(), is(3)); it = addresses.iterator(); HashMap expected2 = new HashMap<>(hostsAndPorts); @@ -146,7 +146,7 @@ public void testGetSCMAddresses() { // Verify empty value conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, ""); try { - getSCMAddresses(conf); + getSCMAddressForDatanodes(conf); fail("Empty value should cause an IllegalArgumentException"); } catch (Exception e) { assertTrue(e instanceof IllegalArgumentException); @@ -155,7 +155,7 @@ public void testGetSCMAddresses() { // Verify invalid hostname conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "s..x..:1234"); try { - getSCMAddresses(conf); + getSCMAddressForDatanodes(conf); fail("An invalid hostname should cause an IllegalArgumentException"); } catch (Exception e) { assertTrue(e instanceof IllegalArgumentException); @@ -164,7 +164,7 @@ public void testGetSCMAddresses() { // Verify invalid port conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm:xyz"); try { - getSCMAddresses(conf); + getSCMAddressForDatanodes(conf); fail("An invalid port should cause an IllegalArgumentException"); } catch (Exception e) { assertTrue(e instanceof IllegalArgumentException); @@ -173,7 +173,7 @@ public void testGetSCMAddresses() { // Verify a mixed case (valid and invalid value both appears) conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234, scm:xyz"); try { - getSCMAddresses(conf); + getSCMAddressForDatanodes(conf); fail("An invalid value should cause an IllegalArgumentException"); } catch (Exception e) { assertTrue(e instanceof IllegalArgumentException); @@ -201,7 +201,7 @@ public void testGetSCMAddressesWithHAConfig() { } Collection scmAddressList = - HddsUtils.getSCMAddresses(conf); + HddsUtils.getSCMAddressForDatanodes(conf); Assert.assertNotNull(scmAddressList); Assert.assertEquals(3, scmAddressList.size()); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index 975ce95e9b13..aab4fa738d5e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -172,7 +172,7 @@ private int getEndPointTaskThreadPoolSize() { int totalServerCount = reconServerCount; try { - totalServerCount += HddsUtils.getSCMAddresses(conf).size(); + totalServerCount += HddsUtils.getSCMAddressForDatanodes(conf).size(); } catch (Exception e) { LOG.error("Fail to get scm addresses", e); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java index ba898db751c3..217592ddccd2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java @@ -39,7 +39,7 @@ import com.google.common.base.Strings; import static org.apache.hadoop.hdds.HddsUtils.getReconAddresses; -import static org.apache.hadoop.hdds.HddsUtils.getSCMAddresses; +import static org.apache.hadoop.hdds.HddsUtils.getSCMAddressForDatanodes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,7 +79,7 @@ public InitDatanodeState(ConfigurationSource conf, public DatanodeStateMachine.DatanodeStates call() throws Exception { Collection addresses = null; try { - addresses = getSCMAddresses(conf); + addresses = getSCMAddressForDatanodes(conf); } catch (IllegalArgumentException e) { if(!Strings.isNullOrEmpty(e.getMessage())) { LOG.error("Failed to get SCM addresses: {}", e.getMessage()); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java index e3a255041828..bdfdfabbe79c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java @@ -69,7 +69,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL_DEFAULT; import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys; import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys; -import static org.apache.hadoop.hdds.HddsUtils.getSingleSCMAddress; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT; @@ -115,37 +114,6 @@ public static void addPBProtocol(Configuration conf, Class protocol, server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocol, service); } - /** - * Retrieve the socket address that should be used by DataNodes to connect - * to the SCM. - * - * @param conf - * @return Target {@code InetSocketAddress} for the SCM service endpoint. - */ - public static InetSocketAddress getScmAddressForDataNodes( - ConfigurationSource conf) { - // We try the following settings in decreasing priority to retrieve the - // target host. - // - OZONE_SCM_DATANODE_ADDRESS_KEY - // - OZONE_SCM_CLIENT_ADDRESS_KEY - // - OZONE_SCM_NAMES - // - Optional host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); - - if (!host.isPresent()) { - // Fallback to Ozone SCM name - host = Optional.of(getSingleSCMAddress(conf).getHostName()); - } - - final int port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY) - .orElse(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT); - - return NetUtils.createSocketAddr(host.get() + ":" + port); - } - /** * Retrieve the socket address that should be used by clients to connect * to the SCM. diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java index 39aa41e2bce8..20c046856736 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java @@ -22,10 +22,13 @@ import java.net.InetSocketAddress; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo; import org.apache.hadoop.hdds.utils.HddsServerUtil; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; + +import org.apache.hadoop.net.NetUtils; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -42,17 +45,6 @@ public class TestHddsServerUtil { @Rule public ExpectedException thrown= ExpectedException.none(); - /** - * Verify DataNode endpoint lookup failure if neither the client nor - * datanode endpoint are configured. - */ - @Test - public void testMissingScmDataNodeAddress() { - final OzoneConfiguration conf = new OzoneConfiguration(); - thrown.expect(IllegalArgumentException.class); - HddsServerUtil.getScmAddressForDataNodes(conf); - } - /** * Verify that the datanode endpoint is parsed correctly. * This tests the logic used by the DataNodes to determine which address @@ -65,7 +57,8 @@ public void testGetScmDataNodeAddress() { // First try a client address with just a host name. Verify it falls // back to the default port. conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); - InetSocketAddress addr = HddsServerUtil.getScmAddressForDataNodes(conf); + InetSocketAddress addr = NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getScmDatanodeAddress()); assertThat(addr.getHostString(), is("1.2.3.4")); assertThat(addr.getPort(), is( ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); @@ -73,10 +66,11 @@ public void testGetScmDataNodeAddress() { // Next try a client address with just a host name and port. // Verify the port is ignored and the default DataNode port is used. conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - addr = HddsServerUtil.getScmAddressForDataNodes(conf); + addr = NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getScmDatanodeAddress()); assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is( - ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); + assertThat(addr.getPort(), + is(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); // Set both OZONE_SCM_CLIENT_ADDRESS_KEY and // OZONE_SCM_DATANODE_ADDRESS_KEY. @@ -84,8 +78,8 @@ public void testGetScmDataNodeAddress() { // default. conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "5.6.7.8"); - addr = - HddsServerUtil.getScmAddressForDataNodes(conf); + addr = NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getScmDatanodeAddress()); assertThat(addr.getHostString(), is("5.6.7.8")); assertThat(addr.getPort(), is( ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); @@ -96,7 +90,8 @@ public void testGetScmDataNodeAddress() { // used. conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "5.6.7.8:200"); - addr = HddsServerUtil.getScmAddressForDataNodes(conf); + addr = NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getScmDatanodeAddress()); assertThat(addr.getHostString(), is("5.6.7.8")); assertThat(addr.getPort(), is(200)); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java index 6f8d020858d7..f7386500fd16 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtils.java @@ -24,8 +24,10 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.HddsServerUtil; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.test.PathUtils; import org.apache.commons.io.FileUtils; @@ -66,7 +68,8 @@ public void testGetDatanodeAddressWithPort() { final OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, scmHost); final InetSocketAddress address = - HddsServerUtil.getScmAddressForDataNodes(conf); + NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getScmDatanodeAddress()); assertEquals(address.getHostName(), scmHost.split(":")[0]); assertEquals(address.getPort(), Integer.parseInt(scmHost.split(":")[1])); } @@ -80,7 +83,8 @@ public void testGetDatanodeAddressWithoutPort() { final OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, scmHost); final InetSocketAddress address = - HddsServerUtil.getScmAddressForDataNodes(conf); + NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getScmDatanodeAddress()); assertEquals(scmHost, address.getHostName()); assertEquals(OZONE_SCM_DATANODE_PORT_DEFAULT, address.getPort()); } @@ -95,7 +99,8 @@ public void testDatanodeAddressFallbackToClientNoPort() { final OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmHost); final InetSocketAddress address = - HddsServerUtil.getScmAddressForDataNodes(conf); + NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getScmDatanodeAddress()); assertEquals(scmHost, address.getHostName()); assertEquals(OZONE_SCM_DATANODE_PORT_DEFAULT, address.getPort()); } @@ -112,7 +117,8 @@ public void testDatanodeAddressFallbackToClientWithPort() { final OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmHost); final InetSocketAddress address = - HddsServerUtil.getScmAddressForDataNodes(conf); + NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getScmDatanodeAddress()); assertEquals(address.getHostName(), scmHost.split(":")[0]); assertEquals(address.getPort(), OZONE_SCM_DATANODE_PORT_DEFAULT); } @@ -126,8 +132,8 @@ public void testDatanodeAddressFallbackToScmNamesNoPort() { final String scmHost = "host123"; final OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_SCM_NAMES, scmHost); - final InetSocketAddress address = - HddsServerUtil.getScmAddressForDataNodes(conf); + final InetSocketAddress address = NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getScmDatanodeAddress()); assertEquals(scmHost, address.getHostName()); assertEquals(OZONE_SCM_DATANODE_PORT_DEFAULT, address.getPort()); } @@ -144,7 +150,8 @@ public void testDatanodeAddressFallbackToScmNamesWithPort() { final OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_SCM_NAMES, scmHost); final InetSocketAddress address = - HddsServerUtil.getScmAddressForDataNodes(conf); + NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getScmDatanodeAddress()); assertEquals(address.getHostName(), scmHost.split(":")[0]); assertEquals(OZONE_SCM_DATANODE_PORT_DEFAULT, address.getPort()); } diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java index ad1f1fd64c2c..712120d223ba 100644 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java @@ -20,24 +20,36 @@ import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; import java.util.List; import org.apache.hadoop.hdds.HddsUtils; +import org.apache.hadoop.hdds.conf.ConfigurationException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; +import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.commons.lang3.StringUtils; + +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; + +import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -61,7 +73,7 @@ public class TestHddsClientUtils { @Test public void testMissingScmClientAddress() { final OzoneConfiguration conf = new OzoneConfiguration(); - thrown.expect(IllegalArgumentException.class); + thrown.expect(ConfigurationException.class); HddsUtils.getScmAddressForClients(conf); } @@ -76,16 +88,55 @@ public void testGetScmClientAddress() { // First try a client address with just a host name. Verify it falls // back to the default port. conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); - InetSocketAddress addr = HddsUtils.getScmAddressForClients(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(OZONE_SCM_CLIENT_PORT_DEFAULT)); + checkAddr(conf, "1.2.3.4", OZONE_SCM_CLIENT_PORT_DEFAULT); // Next try a client address with a host name and port. Verify both // are used correctly. conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - addr = HddsUtils.getScmAddressForClients(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(100)); + checkAddr(conf, "1.2.3.4", 100); + + } + + @Test + public void testGetScmClientAddressForHA() { + OzoneConfiguration conf = new OzoneConfiguration(); + String scmServiceId = "scmservice"; + conf.set(ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY, scmServiceId); + + String[] nodes = new String[] {"scm1", "scm2", "scm3"}; + conf.set(ScmConfigKeys.OZONE_SCM_NODES_KEY+"."+scmServiceId, + "scm1,scm2,scm3"); + conf.set(ScmConfigKeys.OZONE_SCM_NODE_ID_KEY, "scm1"); + + int port = 9880; + int i = 1; + for (String nodeId : nodes) { + conf.setInt(ConfUtils.addKeySuffixes(OZONE_SCM_CLIENT_PORT_KEY, + scmServiceId, nodeId), port); + conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, + scmServiceId, nodeId), "localhost"); + } + + Collection scmClientAddr = + HddsUtils.getScmAddressForClients(conf); + + port = 9880; + + for (InetSocketAddress scmAddr : scmClientAddr) { + assertEquals(scmAddr.getHostName(), "localhost"); + assertEquals(scmAddr.getPort(), port++); + } + + } + + private void checkAddr(OzoneConfiguration conf, String address, + int port) { + Iterator scmAddrIterator = + HddsUtils.getScmAddressForClients(conf).iterator(); + Assert.assertTrue(scmAddrIterator.hasNext()); + InetSocketAddress scmAddr = scmAddrIterator.next(); + assertThat(scmAddr.getHostString(), is(address)); + assertThat(scmAddr.getPort(), is(port)); } @Test @@ -120,59 +171,12 @@ public void testBlockClientFallbackToClientNoPort() { final String scmHost = "host123"; final OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmHost); - final InetSocketAddress address = HddsUtils.getScmAddressForBlockClients( - conf); - assertEquals(scmHost, address.getHostName()); - assertEquals(OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT, address.getPort()); - } - - @Test - @SuppressWarnings("StringSplitter") - public void testBlockClientFallbackToClientWithPort() { - // When OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY is undefined it should - // fallback to OZONE_SCM_CLIENT_ADDRESS_KEY. - // - // Verify that the OZONE_SCM_CLIENT_ADDRESS_KEY port number is ignored, - // if present. Instead we should use OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT. - final String scmHost = "host123:100"; - final OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmHost); - final InetSocketAddress address =HddsUtils.getScmAddressForBlockClients( - conf); - assertEquals(scmHost.split(":")[0], address.getHostName()); - assertEquals(OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT, address.getPort()); - } - - @Test - public void testBlockClientFallbackToScmNamesNoPort() { - // When OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY and OZONE_SCM_CLIENT_ADDRESS_KEY - // are undefined it should fallback to OZONE_SCM_NAMES. - final String scmHost = "host456"; - final OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OZONE_SCM_NAMES, scmHost); - final InetSocketAddress address = HddsUtils.getScmAddressForBlockClients( - conf); + final InetSocketAddress address = NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getBlockClientAddress()); assertEquals(scmHost, address.getHostName()); assertEquals(OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT, address.getPort()); } - @Test - @SuppressWarnings("StringSplitter") - public void testBlockClientFallbackToScmNamesWithPort() { - // When OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY and OZONE_SCM_CLIENT_ADDRESS_KEY - // are undefined it should fallback to OZONE_SCM_NAMES. - // - // Verify that the OZONE_SCM_NAMES port number is ignored, if present. - // Instead we should use OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT. - final String scmHost = "host456:200"; - final OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OZONE_SCM_NAMES, scmHost); - final InetSocketAddress address = HddsUtils.getScmAddressForBlockClients( - conf); - assertEquals(scmHost.split(":")[0], address.getHostName()); - assertEquals(OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT, address.getPort()); - } - @Test public void testClientFallbackToScmNamesNoPort() { // When OZONE_SCM_CLIENT_ADDRESS_KEY is undefined, it should fallback @@ -180,9 +184,12 @@ public void testClientFallbackToScmNamesNoPort() { final String scmHost = "host456"; final OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_SCM_NAMES, scmHost); - final InetSocketAddress address = HddsUtils.getScmAddressForClients(conf); - assertEquals(scmHost, address.getHostName()); - assertEquals(OZONE_SCM_CLIENT_PORT_DEFAULT, address.getPort()); + final Collection address = + HddsUtils.getScmAddressForClients(conf); + Assert.assertTrue(address.iterator().hasNext()); + InetSocketAddress socketAddress = address.iterator().next(); + assertEquals(scmHost, socketAddress.getHostName()); + assertEquals(OZONE_SCM_CLIENT_PORT_DEFAULT, socketAddress.getPort()); } @Test @@ -196,9 +203,30 @@ public void testClientFallbackToScmNamesWithPort() { final String scmHost = "host456:300"; final OzoneConfiguration conf = new OzoneConfiguration(); conf.set(OZONE_SCM_NAMES, scmHost); - final InetSocketAddress address = HddsUtils.getScmAddressForClients(conf); + final Collection address = + HddsUtils.getScmAddressForClients(conf); + Assert.assertTrue(address.iterator().hasNext()); + InetSocketAddress socketAddress = address.iterator().next(); + assertEquals(scmHost.split(":")[0], + socketAddress.getHostName()); + assertEquals(OZONE_SCM_CLIENT_PORT_DEFAULT, socketAddress.getPort()); + } + + @Test + @SuppressWarnings("StringSplitter") + public void testBlockClientFallbackToClientWithPort() { + // When OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY is undefined it should + // fallback to OZONE_SCM_CLIENT_ADDRESS_KEY. + // + // Verify that the OZONE_SCM_CLIENT_ADDRESS_KEY port number is ignored, + // if present. Instead we should use OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT. + final String scmHost = "host123:100"; + final OzoneConfiguration conf = new OzoneConfiguration(); + conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmHost); + final InetSocketAddress address = NetUtils.createSocketAddr( + SCMNodeInfo.buildNodeInfo(conf).get(0).getBlockClientAddress()); assertEquals(scmHost.split(":")[0], address.getHostName()); - assertEquals(OZONE_SCM_CLIENT_PORT_DEFAULT, address.getPort()); + assertEquals(OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT, address.getPort()); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java index 8a0d8db1ced4..9b0d8bcc9400 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java @@ -106,8 +106,9 @@ public void testGetServiceList() throws Exception { Assert.assertEquals(server.getHttpAddress().getPort(), omInfo.getPort(ServicePort.Type.HTTP)); + Assert.assertTrue(getScmAddressForClients(conf).iterator().hasNext()); InetSocketAddress scmAddress = - getScmAddressForClients(conf); + getScmAddressForClients(conf).iterator().next(); ServiceInfo scmInfo = serviceMap.get(HddsProtos.NodeType.SCM); Assert.assertEquals(scmAddress.getHostName(), scmInfo.getHostname()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 3c640fcd09b1..6c18e0edf5fb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2654,17 +2654,21 @@ public List getServiceList() throws IOException { services.add(omServiceInfoBuilder.build()); // For client we have to return SCM with container protocol port, - // not block protocol. - InetSocketAddress scmAddr = getScmAddressForClients( + // not block protocol. This is information is being not used by + // RpcClient, but for compatibility leaving as it is and also making sure + // that this works for SCM HA. + Collection scmAddresses = getScmAddressForClients( configuration); - ServiceInfo.Builder scmServiceInfoBuilder = ServiceInfo.newBuilder() - .setNodeType(HddsProtos.NodeType.SCM) - .setHostname(scmAddr.getHostName()) - .addServicePort(ServicePort.newBuilder() - .setType(ServicePort.Type.RPC) - .setValue(scmAddr.getPort()).build()); - services.add(scmServiceInfoBuilder.build()); + for (InetSocketAddress scmAddr : scmAddresses) { + ServiceInfo.Builder scmServiceInfoBuilder = ServiceInfo.newBuilder() + .setNodeType(HddsProtos.NodeType.SCM) + .setHostname(scmAddr.getHostName()) + .addServicePort(ServicePort.newBuilder() + .setType(ServicePort.Type.RPC) + .setValue(scmAddr.getPort()).build()); + services.add(scmServiceInfoBuilder.build()); + } metrics.incNumGetServiceLists(); // For now there is no exception that can can happen in this call, // so failure metrics is not handled. In future if there is any need to diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/conf/StorageContainerManagersCommandHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/conf/StorageContainerManagersCommandHandler.java index 8d0094945042..e626bef2958f 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/conf/StorageContainerManagersCommandHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/conf/StorageContainerManagersCommandHandler.java @@ -42,7 +42,7 @@ public class StorageContainerManagersCommandHandler implements Callable { @Override public Void call() throws Exception { Collection addresses = HddsUtils - .getSCMAddresses(OzoneConfiguration.of(tool.getConf())); + .getScmAddressForClients(OzoneConfiguration.of(tool.getConf())); for (InetSocketAddress addr : addresses) { tool.printOut(addr.getHostName());