diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index ed86732be3c8..2ffa73d18c41 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -33,7 +33,9 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY; import com.google.common.base.Preconditions; import com.google.common.net.HostAndPort; @@ -73,7 +75,6 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RemoteException; @@ -84,6 +85,7 @@ import org.apache.hadoop.net.DNS; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.conf.OzoneServiceConfig; +import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.token.SecretManager; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; @@ -121,7 +123,7 @@ private HddsUtils() { public static Collection getScmAddressForClients( ConfigurationSource conf) { - if (SCMHAUtils.getScmServiceId(conf) != null) { + if (getScmServiceId(conf) != null) { List scmNodeInfoList = SCMNodeInfo.buildNodeInfo(conf); Collection scmAddressList = new HashSet<>(scmNodeInfoList.size()); @@ -284,7 +286,7 @@ public static Collection getSCMAddressForDatanodes( // First check HA style config, if not defined fall back to OZONE_SCM_NAMES - if (SCMHAUtils.getScmServiceId(conf) != null) { + if (getScmServiceId(conf) != null) { List scmNodeInfoList = SCMNodeInfo.buildNodeInfo(conf); Collection scmAddressList = new HashSet<>(scmNodeInfoList.size()); @@ -893,4 +895,51 @@ public static void reportLeak(Class clazz, String stackTrace, Logger log) { } log.warn(warning); } + + /** + * Get SCM ServiceId from OzoneConfiguration. + * @return SCM service id if defined, else null. + */ + public static String getScmServiceId(ConfigurationSource conf) { + + String localScmServiceId = conf.getTrimmed( + ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID); + + Collection scmServiceIds; + + if (localScmServiceId == null) { + // There is no default scm service id is being set, fall back to ozone + // .scm.service.ids. + scmServiceIds = conf.getTrimmedStringCollection( + OZONE_SCM_SERVICE_IDS_KEY); + if (scmServiceIds.size() > 1) { + throw new ConfigurationException("When multiple SCM Service Ids are " + + "configured," + OZONE_SCM_DEFAULT_SERVICE_ID + " need to be " + + "defined"); + } else if (scmServiceIds.size() == 1) { + localScmServiceId = scmServiceIds.iterator().next(); + } + } + return localScmServiceId; + } + + /** + * Get a collection of all scmNodeIds for the given scmServiceId. + */ + public static Collection getSCMNodeIds(ConfigurationSource conf, + String scmServiceId) { + String key = ConfUtils.addSuffix(ScmConfigKeys.OZONE_SCM_NODES_KEY, scmServiceId); + return conf.getTrimmedStringCollection(key); + } + + /** + * Get SCM Node Id list. + * @param configuration + * @return list of node ids. + */ + public static Collection getSCMNodeIds( + ConfigurationSource configuration) { + String scmServiceId = getScmServiceId(configuration); + return getSCMNodeIds(configuration, scmServiceId); + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java index d797bd0a11bd..0774581c3f11 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java @@ -41,6 +41,7 @@ import java.util.List; import java.util.OptionalInt; import net.jcip.annotations.Immutable; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.ConfigurationException; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.ozone.ha.ConfUtils; @@ -74,10 +75,10 @@ public static List buildNodeInfo(ConfigurationSource conf) { // If service Id is not defined, fall back to non-HA config. List scmNodeInfoList = new ArrayList<>(); - String scmServiceId = SCMHAUtils.getScmServiceId(conf); + String scmServiceId = HddsUtils.getScmServiceId(conf); if (scmServiceId != null) { ArrayList< String > scmNodeIds = new ArrayList<>( - SCMHAUtils.getSCMNodeIds(conf, scmServiceId)); + HddsUtils.getSCMNodeIds(conf, scmServiceId)); if (scmNodeIds.size() == 0) { throw new ConfigurationException( String.format("Configuration does not have any value set for %s " + diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 518d417f07c9..50b2f9844b4c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -55,7 +55,6 @@ import org.apache.hadoop.hdds.protocol.DatanodeID; import org.apache.hadoop.hdds.protocol.SecretKeyProtocol; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; import org.apache.hadoop.hdds.security.SecurityConfig; import org.apache.hadoop.hdds.security.symmetric.DefaultSecretKeyClient; import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; @@ -214,7 +213,7 @@ public void start() { serviceRuntimeInfo = new DNMXBeanImpl(HddsVersionInfo.HDDS_VERSION_INFO) { @Override public String getNamespace() { - return SCMHAUtils.getScmServiceId(conf); + return HddsUtils.getScmServiceId(conf); } }; serviceRuntimeInfo.setStartTime(); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java similarity index 81% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java index c4aa7300af6c..6baa1aa8554b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hdds.scm.ha; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RATIS_SNAPSHOT_DIR; import com.google.common.base.Strings; @@ -28,10 +26,8 @@ import java.io.IOException; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import org.apache.hadoop.hdds.HddsUtils; -import org.apache.hadoop.hdds.conf.ConfigurationException; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.ratis.ServerNotLeaderException; @@ -92,26 +88,6 @@ public static boolean isPrimordialSCM(ConfigurationSource conf, return primordialNode != null && (primordialNode .equals(selfNodeId) || primordialNode.equals(hostName)); } - /** - * Get a collection of all scmNodeIds for the given scmServiceId. - */ - public static Collection getSCMNodeIds(ConfigurationSource conf, - String scmServiceId) { - String key = addSuffix(ScmConfigKeys.OZONE_SCM_NODES_KEY, scmServiceId); - return conf.getTrimmedStringCollection(key); - } - - /** - * Add non empty and non null suffix to a key. - */ - private static String addSuffix(String key, String suffix) { - if (suffix == null || suffix.isEmpty()) { - return key; - } - assert !suffix.startsWith(".") : - "suffix '" + suffix + "' should not already have '.' prepended."; - return key + "." + suffix; - } /** * Get the local directory where ratis logs will be stored. @@ -150,34 +126,6 @@ public static String getSCMRatisSnapshotDirectory(ConfigurationSource conf) { return snapshotDir; } - /** - * Get SCM ServiceId from OzoneConfiguration. - * @param conf - * @return SCM service id if defined, else null. - */ - public static String getScmServiceId(ConfigurationSource conf) { - - String localScmServiceId = conf.getTrimmed( - ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID); - - Collection scmServiceIds; - - if (localScmServiceId == null) { - // There is no default scm service id is being set, fall back to ozone - // .scm.service.ids. - scmServiceIds = conf.getTrimmedStringCollection( - OZONE_SCM_SERVICE_IDS_KEY); - if (scmServiceIds.size() > 1) { - throw new ConfigurationException("When multiple SCM Service Ids are " + - "configured," + OZONE_SCM_DEFAULT_SERVICE_ID + " need to be " + - "defined"); - } else if (scmServiceIds.size() == 1) { - localScmServiceId = scmServiceIds.iterator().next(); - } - } - return localScmServiceId; - } - /** * Removes the self node from the list of nodes in the * configuration. @@ -190,7 +138,7 @@ public static OzoneConfiguration removeSelfId( OzoneConfiguration configuration, String selfId) { final OzoneConfiguration conf = new OzoneConfiguration(configuration); String scmNodesKey = ConfUtils.addKeySuffixes( - ScmConfigKeys.OZONE_SCM_NODES_KEY, getScmServiceId(conf)); + ScmConfigKeys.OZONE_SCM_NODES_KEY, HddsUtils.getScmServiceId(conf)); String scmNodes = conf.get(scmNodesKey); if (scmNodes != null) { String[] parts = scmNodes.split(","); @@ -205,17 +153,6 @@ public static OzoneConfiguration removeSelfId( return conf; } - /** - * Get SCM Node Id list. - * @param configuration - * @return list of node ids. - */ - public static Collection getSCMNodeIds( - ConfigurationSource configuration) { - String scmServiceId = getScmServiceId(configuration); - return getSCMNodeIds(configuration, scmServiceId); - } - public static Throwable unwrapException(Exception e) { Throwable cause = e.getCause(); if (e instanceof RemoteException) { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/package-info.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/package-info.java new file mode 100644 index 000000000000..9f26af19e749 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/ha/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Utility classes for SCM HA. + */ +package org.apache.hadoop.hdds.scm.ha; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java similarity index 100% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java rename to hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java index 6d6a0e04bde1..f86676fb6d23 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java @@ -43,6 +43,7 @@ import java.util.stream.Stream; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; @@ -418,7 +419,7 @@ public static List buildCAX509List(ConfigurationSource conf) th long waitDuration = conf.getTimeDuration(OZONE_SCM_CA_LIST_RETRY_INTERVAL, OZONE_SCM_CA_LIST_RETRY_INTERVAL_DEFAULT, TimeUnit.SECONDS); - Collection scmNodes = SCMHAUtils.getSCMNodeIds(conf); + Collection scmNodes = HddsUtils.getSCMNodeIds(conf); SCMSecurityProtocolClientSideTranslatorPB scmSecurityProtocolClient = HddsServerUtil.getScmSecurityClient(conf); int expectedCount = scmNodes.size() + 1; diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestSCMHAUtils.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAUtils.java similarity index 95% rename from hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestSCMHAUtils.java rename to hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAUtils.java index e263ece134d1..091fbc7a9ad6 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestSCMHAUtils.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAUtils.java @@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.Collection; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; import org.junit.jupiter.api.Test; @@ -45,7 +46,7 @@ void testRemoveSelfId() { OzoneConfiguration output = SCMHAUtils.removeSelfId(input, selfId); Collection nodesWithoutSelf = - SCMHAUtils.getSCMNodeIds(output, service); + HddsUtils.getSCMNodeIds(output, service); assertEquals(2, nodesWithoutSelf.size()); assertThat(nodesWithoutSelf).doesNotContain(selfId); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java index 83e42c954617..70365cf5c792 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHANodeDetails.java @@ -52,6 +52,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ScmUtils; @@ -178,7 +179,7 @@ public static SCMHANodeDetails loadSCMHAConfig(OzoneConfiguration conf, boolean isSCMddressSet = false; for (String serviceId : scmServiceIds) { - Collection scmNodeIds = SCMHAUtils.getSCMNodeIds(conf, serviceId); + Collection scmNodeIds = HddsUtils.getSCMNodeIds(conf, serviceId); // TODO: need to fall back to ozone.scm.names in case scm node ids are // not defined. diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 6cf267803f50..7e898acc0ed1 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -1334,7 +1334,7 @@ private static InetSocketAddress getScmAddress(SCMHANodeDetails haDetails, Preconditions.checkNotNull(scmNodeInfoList, "scmNodeInfoList is null"); InetSocketAddress scmAddress = null; - if (SCMHAUtils.getScmServiceId(conf) != null) { + if (HddsUtils.getScmServiceId(conf) != null) { for (SCMNodeInfo scmNodeInfo : scmNodeInfoList) { if (haDetails.getLocalNodeDetails().getNodeId() != null && haDetails.getLocalNodeDetails().getNodeId().equals( @@ -2137,7 +2137,7 @@ private static List> getRatisRolesException(String exceptionString) public String getPrimordialNode() { String primordialNode = SCMHAUtils.getPrimordialSCM(configuration); // primordialNode can be nodeId too . If it is then return hostname. - if (SCMHAUtils.getSCMNodeIds(configuration).contains(primordialNode)) { + if (HddsUtils.getSCMNodeIds(configuration).contains(primordialNode)) { List localAndPeerNodes = new ArrayList<>(scmHANodeDetails.getPeerNodeDetails()); localAndPeerNodes.add(getSCMHANodeDetails().getLocalNodeDetails()); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java index 02673d7614bc..57f871369006 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.client.ScmClient; -import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; import picocli.CommandLine; /** @@ -68,7 +67,7 @@ private void checkAndSetSCMAddressArg(MutableConfigurationSource conf) { if (StringUtils.isNotEmpty(scmServiceId)) { conf.set(ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID, scmServiceId); - } else if (StringUtils.isBlank(SCMHAUtils.getScmServiceId(conf))) { + } else if (StringUtils.isBlank(HddsUtils.getScmServiceId(conf))) { // Scm service id is not passed, and scm service id is not defined in // the config, assuming it should be non-HA cluster. if (!HddsUtils.getHostNameFromConfigKeys(conf,