diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java index 879b68aa30cc..3628ae0c25b0 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java @@ -76,7 +76,8 @@ public static void setRaftStorageDir(final RaftProperties properties, final ConfigurationSource conf) { String storageDir = haConf.getRatisStorageDir(); if (Strings.isNullOrEmpty(storageDir)) { - storageDir = ServerUtils.getDefaultRatisDirectory(conf); + File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); + storageDir = (new File(metaDirPath, "scm-ha")).getPath(); } RaftServerConfigKeys.setStorageDir(properties, Collections.singletonList(new File(storageDir))); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java index cbe2ce38ef41..c3259bcb5ce4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java @@ -82,9 +82,11 @@ private Object invokeLocal(Method method, Object[] args) */ private Object invokeRatis(Method method, Object[] args) throws Exception { - LOG.trace("Invoking method {} on target {}", method, ratisHandler); + long startTime = Time.monotonicNowNanos(); final SCMRatisResponse response = ratisHandler.submitRequest( SCMRatisRequest.of(requestType, method.getName(), args)); + LOG.info("Invoking method {} on target {}, cost {}us", + method, ratisHandler, (Time.monotonicNowNanos() - startTime) / 1000.0); if (response.isSuccess()) { return response.getResult(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java index 3a81d2bb9033..e9594461e7aa 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -195,10 +196,7 @@ public RaftPeerId getPeerId() { InetAddress localHost = InetAddress.getLocalHost(); // fetch hosts from ozone.scm.names - List hosts = - Arrays.stream(conf.getTrimmedStrings(ScmConfigKeys.OZONE_SCM_NAMES)) - .map(scmName -> HddsUtils.getHostName(scmName).get()) - .collect(Collectors.toList()); + List hosts = parseHosts(conf); final List raftPeers = new ArrayList<>(); for (int i = 0; i < hosts.size(); ++i) { @@ -232,5 +230,29 @@ public RaftPeerId getPeerId() { raftGroup = RaftGroup.valueOf(raftGroupId, raftPeers); } + + private List parseHosts(final ConfigurationSource conf) + throws UnknownHostException { + // fetch hosts from ozone.scm.names + List hosts = + Arrays.stream(conf.getTrimmedStrings(ScmConfigKeys.OZONE_SCM_NAMES)) + .map(scmName -> HddsUtils.getHostName(scmName).get()) + .collect(Collectors.toList()); + + // if this is not a conf for a multi-server raft cluster, + // it means we are in integration test, and need to augment + // the conf to help build a single-server raft cluster. + if (hosts.size() == 0) { + // ozone.scm.names is not set + hosts.add(InetAddress.getLocalHost().getHostName()); + } else if (hosts.size() == 1) { + // ozone.scm.names is set, yet the conf may not be usable. + hosts.set(0, InetAddress.getLocalHost().getHostName()); + } + + LOG.info("fetch hosts {} from ozone.scm.names {}.", + hosts, conf.get(ScmConfigKeys.OZONE_SCM_NAMES)); + return hosts; + } } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java index cbfda8ceb6b8..4bac431b82c5 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java @@ -475,7 +475,7 @@ public static void quasiCloseContainer(ContainerManagerV2 containerManager, public static StorageContainerManager getScmSimple(OzoneConfiguration conf) throws IOException, AuthenticationException { SCMConfigurator configurator = new SCMConfigurator(); - configurator.setSCMHAManager(MockSCMHAManager.getInstance(true)); + conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true); return StorageContainerManager.createSCM(conf, configurator); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java index a72a2570c810..7f28bddbd0b0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.junit.Assert; +import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -99,6 +100,12 @@ public void init() throws Exception { o3fs = (OzoneFileSystem) FileSystem.get(new URI(rootPath), conf); } + @After + public void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } @Test public void test() throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 340b902ddc1a..58e459725bc2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -171,6 +171,7 @@ public void waitForClusterToBeReady() final int healthy = scm.getNodeCount(HEALTHY); final boolean isNodeReady = healthy == hddsDatanodes.size(); final boolean exitSafeMode = !scm.isInSafeMode(); + final boolean checkScmLeader = scm.checkLeader(); LOG.info("{}. Got {} of {} DN Heartbeats.", isNodeReady ? "Nodes are ready" : "Waiting for nodes to be ready", @@ -178,8 +179,10 @@ public void waitForClusterToBeReady() LOG.info(exitSafeMode ? "Cluster exits safe mode" : "Waiting for cluster to exit safe mode", healthy, hddsDatanodes.size()); + LOG.info(checkScmLeader ? "SCM became leader" : + "SCM has not become leader"); - return isNodeReady && exitSafeMode; + return isNodeReady && exitSafeMode && checkScmLeader; }, 1000, waitForClusterToBeReadyTimeout); }