diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 8fffcef48366..42ba7ba54f1c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -30,6 +30,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.TimeZone; import java.util.concurrent.TimeUnit; @@ -37,7 +38,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; @@ -88,6 +89,9 @@ public final class HddsUtils { "OzoneScmServiceInstance"; private static final TimeZone UTC_ZONE = TimeZone.getTimeZone("UTC"); + private static final String MULTIPLE_SCM_NOT_YET_SUPPORTED = + ScmConfigKeys.OZONE_SCM_NAMES + " must contain a single hostname." + + " Multiple SCM hosts are currently unsupported"; private static final int NO_PORT = -1; @@ -98,38 +102,22 @@ private HddsUtils() { * Retrieve the socket address that should be used by clients to connect * to the SCM. * - * @param conf - * @return Target InetSocketAddress for the SCM client endpoint. + * @return Target {@code InetSocketAddress} for the SCM client endpoint. */ public static InetSocketAddress getScmAddressForClients(Configuration conf) { Optional host = getHostNameFromConfigKeys(conf, ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); if (!host.isPresent()) { - // Fallback to Ozone SCM names. - Collection scmAddresses = getSCMAddresses(conf); - if (scmAddresses.size() > 1) { - throw new IllegalArgumentException( - ScmConfigKeys.OZONE_SCM_NAMES + - " must contain a single hostname. Multiple SCM hosts are " + - "currently unsupported"); - } - host = Optional.of(scmAddresses.iterator().next().getHostName()); + // Fallback to Ozone SCM name + host = Optional.of(getSingleSCMAddress(conf).getHostName()); } - if (!host.isPresent()) { - throw new IllegalArgumentException( - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY + " must be defined. See" - + " https://wiki.apache.org/hadoop/Ozone#Configuration for " - + "details" - + " on configuring Ozone."); - } + final int port = getPortNumberFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY) + .orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT); - final Optional port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); - - return NetUtils.createSocketAddr(host.get() + ":" + port - .orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT)); + return NetUtils.createSocketAddr(host.get() + ":" + port); } /** @@ -139,45 +127,25 @@ public static InetSocketAddress getScmAddressForClients(Configuration conf) { * then {@link ScmConfigKeys#OZONE_SCM_CLIENT_ADDRESS_KEY} is used. If neither * is defined then {@link ScmConfigKeys#OZONE_SCM_NAMES} is used. * - * @param conf - * @return Target InetSocketAddress for the SCM block client endpoint. + * @return Target {@code InetSocketAddress} for the SCM block client endpoint. * @throws IllegalArgumentException if configuration is not defined. */ public static InetSocketAddress getScmAddressForBlockClients( Configuration conf) { Optional host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY); - - if (!host.isPresent()) { - host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); - } - - if (!host.isPresent()) { - // Fallback to Ozone SCM names. - Collection scmAddresses = getSCMAddresses(conf); - if (scmAddresses.size() > 1) { - throw new IllegalArgumentException( - ScmConfigKeys.OZONE_SCM_NAMES + - " must contain a single hostname. Multiple SCM hosts are " + - "currently unsupported"); - } - host = Optional.of(scmAddresses.iterator().next().getHostName()); - } + ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, + ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); if (!host.isPresent()) { - throw new IllegalArgumentException( - ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY - + " must be defined. See" - + " https://wiki.apache.org/hadoop/Ozone#Configuration" - + " for details on configuring Ozone."); + // Fallback to Ozone SCM name + host = Optional.of(getSingleSCMAddress(conf).getHostName()); } - final Optional port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY); + final int port = getPortNumberFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY) + .orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT); - return NetUtils.createSocketAddr(host.get() + ":" + port - .orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT)); + return NetUtils.createSocketAddr(host.get() + ":" + port); } /** @@ -198,13 +166,11 @@ public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient( RetryPolicy retryPolicy = RetryPolicies.retryForeverWithFixedSleep( 1000, TimeUnit.MILLISECONDS); - SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient = - new SCMSecurityProtocolClientSideTranslatorPB( - RPC.getProtocolProxy(SCMSecurityProtocolPB.class, scmVersion, - address, UserGroupInformation.getCurrentUser(), - conf, NetUtils.getDefaultSocketFactory(conf), - Client.getRpcTimeout(conf), retryPolicy).getProxy()); - return scmSecurityClient; + return new SCMSecurityProtocolClientSideTranslatorPB( + RPC.getProtocolProxy(SCMSecurityProtocolPB.class, scmVersion, + address, UserGroupInformation.getCurrentUser(), + conf, NetUtils.getDefaultSocketFactory(conf), + Client.getRpcTimeout(conf), retryPolicy).getProxy()); } /** @@ -249,19 +215,19 @@ public static Optional getHostName(String value) { } /** - * Gets the port if there is one, throws otherwise. + * Gets the port if there is one, returns empty {@code OptionalInt} otherwise. * @param value String in host:port format. * @return Port */ - public static Optional getHostPort(String value) { + public static OptionalInt getHostPort(String value) { if ((value == null) || value.isEmpty()) { - return Optional.empty(); + return OptionalInt.empty(); } int port = HostAndPort.fromString(value).getPortOrDefault(NO_PORT); if (port == NO_PORT) { - return Optional.empty(); + return OptionalInt.empty(); } else { - return Optional.of(port); + return OptionalInt.of(port); } } @@ -277,53 +243,64 @@ public static Optional getHostPort(String value) { * @throws IllegalArgumentException if any values are not in the 'host' * or host:port format. */ - public static Optional getPortNumberFromConfigKeys( + public static OptionalInt getPortNumberFromConfigKeys( Configuration conf, String... keys) { for (final String key : keys) { final String value = conf.getTrimmed(key); - final Optional hostPort = getHostPort(value); + final OptionalInt hostPort = getHostPort(value); if (hostPort.isPresent()) { return hostPort; } } - return Optional.empty(); + return OptionalInt.empty(); } /** * Retrieve the socket addresses of all storage container managers. * - * @param conf * @return A collection of SCM addresses * @throws IllegalArgumentException If the configuration is invalid */ public static Collection getSCMAddresses( - Configuration conf) throws IllegalArgumentException { - Collection addresses = - new HashSet(); + Configuration conf) { Collection names = conf.getTrimmedStringCollection(ScmConfigKeys.OZONE_SCM_NAMES); - if (names == null || names.isEmpty()) { + if (names.isEmpty()) { throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_NAMES + " need to be a set of valid DNS names or IP addresses." - + " Null or empty address list found."); + + " Empty address list found."); } - final Optional defaultPort = Optional - .of(ScmConfigKeys.OZONE_SCM_DEFAULT_PORT); + Collection addresses = new HashSet<>(names.size()); for (String address : names) { Optional hostname = getHostName(address); if (!hostname.isPresent()) { throw new IllegalArgumentException("Invalid hostname for SCM: " - + hostname); + + address); } - Optional port = getHostPort(address); - InetSocketAddress addr = NetUtils.createSocketAddr(hostname.get(), - port.orElse(defaultPort.get())); + int port = getHostPort(address) + .orElse(ScmConfigKeys.OZONE_SCM_DEFAULT_PORT); + InetSocketAddress addr = NetUtils.createSocketAddr(hostname.get(), port); addresses.add(addr); } return addresses; } - + + /** + * Retrieve the address of the only SCM (as currently multiple ones are not + * supported). + * + * @return SCM address + * @throws IllegalArgumentException if {@code conf} has more than one SCM + * address or it has none + */ + public static InetSocketAddress getSingleSCMAddress(Configuration conf) { + Collection singleton = getSCMAddresses(conf); + Preconditions.checkArgument(singleton.size() == 1, + MULTIPLE_SCM_NOT_YET_SUPPORTED); + return singleton.iterator().next(); + } + /** * Returns the hostname for this datanode. If the hostname is not * explicitly configured in the given config, then it is determined @@ -340,9 +317,9 @@ public static String getHostName(Configuration conf) String name = conf.get(DFS_DATANODE_HOST_NAME_KEY); if (name == null) { String dnsInterface = conf.get( - CommonConfigurationKeys.HADOOP_SECURITY_DNS_INTERFACE_KEY); + CommonConfigurationKeysPublic.HADOOP_SECURITY_DNS_INTERFACE_KEY); String nameServer = conf.get( - CommonConfigurationKeys.HADOOP_SECURITY_DNS_NAMESERVER_KEY); + CommonConfigurationKeysPublic.HADOOP_SECURITY_DNS_NAMESERVER_KEY); boolean fallbackToHosts = false; if (dnsInterface == null) { @@ -399,7 +376,6 @@ public static boolean isReadOnly( * read/write data on datanode via input/output stream. * Ozone datanode uses this helper to decide which command requires block * token. - * @param cmdType * @return true if it is a cmd that block token should be checked when * security is enabled * false if block token does not apply to the command. @@ -518,51 +494,31 @@ public static long getUtcTime() { * is defined then {@link ScmConfigKeys#OZONE_SCM_NAMES} is used. * * @param conf - * @return Target InetSocketAddress for the SCM block client endpoint. - * @throws IllegalArgumentException if configuration is not defined. + * @return Target {@code InetSocketAddress} for the SCM block client endpoint. + * @throws IllegalArgumentException if configuration is not defined or invalid */ public static InetSocketAddress getScmAddressForSecurityProtocol( Configuration conf) { Optional host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY); - - if (!host.isPresent()) { - host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); - } - - if (!host.isPresent()) { - // Fallback to Ozone SCM names. - Collection scmAddresses = getSCMAddresses(conf); - if (scmAddresses.size() > 1) { - throw new IllegalArgumentException( - ScmConfigKeys.OZONE_SCM_NAMES + - " must contain a single hostname. Multiple SCM hosts are " + - "currently unsupported"); - } - host = Optional.of(scmAddresses.iterator().next().getHostName()); - } + ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY, + ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); if (!host.isPresent()) { - throw new IllegalArgumentException( - ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY - + " must be defined. See" - + " https://wiki.apache.org/hadoop/Ozone#Configuration" - + " for details on configuring Ozone."); + // Fallback to Ozone SCM name + host = Optional.of(getSingleSCMAddress(conf).getHostName()); } - final Optional port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY); + final int port = getPortNumberFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY) + .orElse(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT); - return NetUtils.createSocketAddr(host.get() + ":" + port - .orElse(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT)); + return NetUtils.createSocketAddr(host.get() + ":" + port); } /** * Initialize hadoop metrics system for Ozone servers. * @param configuration OzoneConfiguration to use. * @param serverName The logical name of the server components. - * @return */ public static MetricsSystem initializeMetrics( OzoneConfiguration configuration, String serverName) { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java index c1997d6c899b..25ca0c368074 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java @@ -29,10 +29,8 @@ import java.io.File; import java.net.InetSocketAddress; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.concurrent.TimeUnit; import static org.apache.hadoop.hdds.HddsConfigKeys @@ -76,7 +74,7 @@ private HddsServerUtil() { * to the SCM. * * @param conf - * @return Target InetSocketAddress for the SCM service endpoint. + * @return Target {@code InetSocketAddress} for the SCM service endpoint. */ public static InetSocketAddress getScmAddressForDataNodes( Configuration conf) { @@ -91,33 +89,15 @@ public static InetSocketAddress getScmAddressForDataNodes( ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); if (!host.isPresent()) { - // Fallback to Ozone SCM names. - Collection scmAddresses = getSCMAddresses(conf); - if (scmAddresses.size() > 1) { - throw new IllegalArgumentException( - ScmConfigKeys.OZONE_SCM_NAMES + - " must contain a single hostname. Multiple SCM hosts are " + - "currently unsupported"); - } - host = Optional.of(scmAddresses.iterator().next().getHostName()); - } - - if (!host.isPresent()) { - throw new IllegalArgumentException( - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY + - " must be defined. See" + - " https://wiki.apache.org/hadoop/Ozone#Configuration " - + "for details on configuring Ozone."); + // Fallback to Ozone SCM name + host = Optional.of(getSingleSCMAddress(conf).getHostName()); } - // If no port number is specified then we'll just try the defaultBindPort. - final Optional port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY); - - InetSocketAddress addr = NetUtils.createSocketAddr(host.get() + ":" + - port.orElse(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); + final int port = getPortNumberFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY) + .orElse(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT); - return addr; + return NetUtils.createSocketAddr(host.get() + ":" + port); } /** @@ -125,19 +105,19 @@ public static InetSocketAddress getScmAddressForDataNodes( * to the SCM. * * @param conf - * @return Target InetSocketAddress for the SCM client endpoint. + * @return Target {@code InetSocketAddress} for the SCM client endpoint. */ public static InetSocketAddress getScmClientBindAddress( Configuration conf) { - final Optional host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY); + final String host = getHostNameFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY) + .orElse(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT); - final Optional port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY); + final int port = getPortNumberFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY) + .orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT); - return NetUtils.createSocketAddr( - host.orElse(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT) + ":" + - port.orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT)); + return NetUtils.createSocketAddr(host + ":" + port); } /** @@ -145,20 +125,19 @@ public static InetSocketAddress getScmClientBindAddress( * to the SCM Block service. * * @param conf - * @return Target InetSocketAddress for the SCM block client endpoint. + * @return Target {@code InetSocketAddress} for the SCM block client endpoint. */ public static InetSocketAddress getScmBlockClientBindAddress( Configuration conf) { - final Optional host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY); + final String host = getHostNameFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY) + .orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT); - final Optional port = getPortNumberFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY); + final int port = getPortNumberFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY) + .orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT); - return NetUtils.createSocketAddr( - host.orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT) - + ":" - + port.orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT)); + return NetUtils.createSocketAddr(host + ":" + port); } /** @@ -166,18 +145,19 @@ public static InetSocketAddress getScmBlockClientBindAddress( * service clients. * * @param conf - * @return Target InetSocketAddress for the SCM security service. + * @return Target {@code InetSocketAddress} for the SCM security service. */ public static InetSocketAddress getScmSecurityInetAddress( Configuration conf) { - final Optional host = getHostNameFromConfigKeys(conf, - ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY); + final String host = getHostNameFromConfigKeys(conf, + ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY) + .orElse(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT); - final Optional port = getPortNumberFromConfigKeys(conf, + final OptionalInt port = getPortNumberFromConfigKeys(conf, ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY); return NetUtils.createSocketAddr( - host.orElse(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) + host + ":" + port .orElse(conf.getInt(ScmConfigKeys .OZONE_SCM_SECURITY_SERVICE_PORT_KEY, @@ -189,15 +169,14 @@ public static InetSocketAddress getScmSecurityInetAddress( * to the SCM. * * @param conf - * @return Target InetSocketAddress for the SCM service endpoint. + * @return Target {@code InetSocketAddress} for the SCM service endpoint. */ public static InetSocketAddress getScmDataNodeBindAddress( Configuration conf) { final Optional host = getHostNameFromConfigKeys(conf, ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY); - // If no port number is specified then we'll just try the defaultBindPort. - final Optional port = getPortNumberFromConfigKeys(conf, + final OptionalInt port = getPortNumberFromConfigKeys(conf, ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY); return NetUtils.createSocketAddr( @@ -319,27 +298,6 @@ public static int getContainerPort(Configuration conf) { OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT); } - - /** - * Return the list of service addresses for the Ozone SCM. This method is used - * by the DataNodes to determine the service instances to connect to. - * - * @param conf - * @return list of SCM service addresses. - */ - public static Map> - getScmServiceRpcAddresses(Configuration conf) { - - final Map serviceInstances = new HashMap<>(); - serviceInstances.put(OZONE_SCM_SERVICE_INSTANCE_ID, - getScmAddressForDataNodes(conf)); - - final Map> services = - new HashMap<>(); - services.put(OZONE_SCM_SERVICE_ID, serviceInstances); - return services; - } - public static String getOzoneDatanodeRatisDirectory(Configuration conf) { String storageDir = conf.get( OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java index 990d89dc0cb5..b6ba4f67489c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -35,6 +35,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.util.Optional; +import java.util.OptionalInt; import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys; import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys; @@ -70,15 +71,15 @@ public BaseHttpServer(Configuration conf, String name) throws IOException { if (isEnabled()) { this.httpAddress = getHttpBindAddress(); this.httpsAddress = getHttpsBindAddress(); - HttpServer2.Builder builder = null; // Avoid registering o.a.h.http.PrometheusServlet in HttpServer2. // TODO: Replace "hadoop.prometheus.endpoint.enabled" with // CommonConfigurationKeysPublic.HADOOP_PROMETHEUS_ENABLED when possible. conf.setBoolean("hadoop.prometheus.endpoint.enabled", false); - builder = DFSUtil.httpServerTemplateForNNAndJN(conf, this.httpAddress, - this.httpsAddress, name, getSpnegoPrincipal(), getKeytabFile()); + HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf, + httpAddress, httpsAddress, + name, getSpnegoPrincipal(), getKeytabFile()); final boolean xFrameEnabled = conf.getBoolean( DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED, @@ -143,7 +144,7 @@ protected InetSocketAddress getBindAddress(String bindHostKey, final Optional bindHost = getHostNameFromConfigKeys(conf, bindHostKey); - final Optional addressPort = + final OptionalInt addressPort = getPortNumberFromConfigKeys(conf, addressKey); final Optional addressHost = @@ -212,18 +213,14 @@ public void updateConnectorAddress() { httpAddress = httpServer.getConnectorAddress(connIdx++); String realAddress = NetUtils.getHostPortString(httpAddress); conf.set(getHttpAddressKey(), realAddress); - LOG.info( - String.format("HTTP server of %s is listening at http://%s", - name.toUpperCase(), realAddress)); + LOG.info("HTTP server of {} listening at http://{}", name, realAddress); } if (policy.isHttpsEnabled()) { httpsAddress = httpServer.getConnectorAddress(connIdx); String realAddress = NetUtils.getHostPortString(httpsAddress); conf.set(getHttpsAddressKey(), realAddress); - LOG.info( - String.format("HTTP server of %s is listening at https://%s", - name.toUpperCase(), realAddress)); + LOG.info("HTTPS server of {} listening at https://{}", name, realAddress); } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 22ee099ebeac..f2cb16fa36fd 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -33,6 +33,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Optional; +import java.util.OptionalInt; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -171,10 +172,8 @@ public static boolean isOmHAServiceId(Configuration conf, String serviceId) { } public static int getOmRpcPort(Configuration conf) { - // If no port number is specified then we'll just try the defaultBindPort. - final Optional port = getPortNumberFromConfigKeys(conf, - OZONE_OM_ADDRESS_KEY); - return port.orElse(OZONE_OM_PORT_DEFAULT); + return getPortNumberFromConfigKeys(conf, OZONE_OM_ADDRESS_KEY) + .orElse(OZONE_OM_PORT_DEFAULT); } /** @@ -185,17 +184,13 @@ public static int getOmRpcPort(Configuration conf) { * @return Port on which OM RPC server will listen on */ public static int getOmRpcPort(Configuration conf, String confKey) { - // If no port number is specified then we'll just try the defaultBindPort. - final Optional port = getPortNumberFromConfigKeys(conf, confKey); - return port.orElse(OZONE_OM_PORT_DEFAULT); + return getPortNumberFromConfigKeys(conf, confKey) + .orElse(OZONE_OM_PORT_DEFAULT); } public static int getOmRestPort(Configuration conf) { - // If no port number is specified then we'll just try the default - // HTTP BindPort. - final Optional port = - getPortNumberFromConfigKeys(conf, OZONE_OM_HTTP_ADDRESS_KEY); - return port.orElse(OZONE_OM_HTTP_BIND_PORT_DEFAULT); + return getPortNumberFromConfigKeys(conf, OZONE_OM_HTTP_ADDRESS_KEY) + .orElse(OZONE_OM_HTTP_BIND_PORT_DEFAULT); } /** @@ -430,7 +425,7 @@ public static String getHttpAddressForOMPeerNode(Configuration conf, final Optional bindHost = getHostNameFromConfigKeys(conf, addKeySuffixes(OZONE_OM_HTTP_BIND_HOST_KEY, omServiceId, omNodeId)); - final Optional addressPort = getPortNumberFromConfigKeys(conf, + final OptionalInt addressPort = getPortNumberFromConfigKeys(conf, addKeySuffixes(OZONE_OM_HTTP_ADDRESS_KEY, omServiceId, omNodeId)); final Optional addressHost = getHostNameFromConfigKeys(conf, @@ -453,7 +448,7 @@ public static String getHttpsAddressForOMPeerNode(Configuration conf, final Optional bindHost = getHostNameFromConfigKeys(conf, addKeySuffixes(OZONE_OM_HTTPS_BIND_HOST_KEY, omServiceId, omNodeId)); - final Optional addressPort = getPortNumberFromConfigKeys(conf, + final OptionalInt addressPort = getPortNumberFromConfigKeys(conf, addKeySuffixes(OZONE_OM_HTTPS_ADDRESS_KEY, omServiceId, omNodeId)); final Optional addressHost = getHostNameFromConfigKeys(conf, diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java index fcabc67e785e..6857cdd5303f 100644 --- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java +++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java @@ -48,6 +48,7 @@ import java.util.Collection; import java.util.Optional; +import java.util.OptionalInt; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -191,9 +192,9 @@ private String getHostFromAddress(String addr) { * @return Port number */ private int getPortFromAddress(String addr) { - Optional portOptional = getHostPort(addr); + OptionalInt portOptional = getHostPort(addr); assert(portOptional.isPresent()); - return portOptional.get(); + return portOptional.getAsInt(); } /** @@ -221,12 +222,12 @@ public void testWithQualifiedDefaultFS() throws Exception { // Expectation: Success. res = ToolRunner.run(shell, new String[] {"-ls", "/"}); // Check return value, should be 0 (success) - Assert.assertEquals(res, 0); + Assert.assertEquals(0, res); // Test case 2: ozone fs -ls o3fs:/// // Expectation: Success. fs.defaultFS is a fully qualified path. res = ToolRunner.run(shell, new String[] {"-ls", "o3fs:///"}); - Assert.assertEquals(res, 0); + Assert.assertEquals(0, res); // Test case 3: ozone fs -ls o3fs://bucket.volume/ // Expectation: Fail. Must have service id or host name when HA is enabled @@ -253,7 +254,7 @@ public void testWithQualifiedDefaultFS() throws Exception { getHostFromAddress(leaderOMNodeAddr)); res = ToolRunner.run(shell, new String[] {"-ls", qualifiedPath1}); // Note: this test case will fail if the port is not from the leader node - Assert.assertEquals(res, 0); + Assert.assertEquals(0, res); // Test case 5: ozone fs -ls o3fs://bucket.volume.om1:port/ // Expectation: Success. @@ -261,14 +262,14 @@ public void testWithQualifiedDefaultFS() throws Exception { OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName, leaderOMNodeAddr); res = ToolRunner.run(shell, new String[] {"-ls", qualifiedPath2}); - Assert.assertEquals(res, 0); + Assert.assertEquals(0, res); // Test case 6: ozone fs -ls o3fs://bucket.volume.id1/ // Expectation: Success. String qualifiedPath3 = String.format("%s://%s.%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName, omServiceId); res = ToolRunner.run(shell, new String[] {"-ls", qualifiedPath3}); - Assert.assertEquals(res, 0); + Assert.assertEquals(0, res); // Test case 7: ozone fs -ls o3fs://bucket.volume.id1:port/ // Expectation: Fail. Service ID does not use port information.