diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java index 1bc16974362f..8a4922b817ce 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java @@ -76,7 +76,8 @@ public static void setRaftStorageDir(final RaftProperties properties, final ConfigurationSource conf) { String storageDir = haConf.getRatisStorageDir(); if (Strings.isNullOrEmpty(storageDir)) { - storageDir = ServerUtils.getDefaultRatisDirectory(conf); + File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf); + storageDir = (new File(metaDirPath, "scm-ha")).getPath(); } RaftServerConfigKeys.setStorageDir(properties, Collections.singletonList(new File(storageDir))); @@ -100,9 +101,9 @@ private static void setRaftRpcProperties(final RaftProperties properties, Rpc.setRequestTimeout(properties, TimeDuration.valueOf( conf.getRatisRequestTimeout(), TimeUnit.MILLISECONDS)); Rpc.setTimeoutMin(properties, TimeDuration.valueOf( - conf.getRatisRequestMinTimeout(), TimeUnit.MILLISECONDS)); + conf.getLeaderElectionMinTimeout(), TimeUnit.MILLISECONDS)); Rpc.setTimeoutMax(properties, TimeDuration.valueOf( - conf.getRatisRequestMaxTimeout(), TimeUnit.MILLISECONDS)); + conf.getLeaderElectionMaxTimeout(), TimeUnit.MILLISECONDS)); Rpc.setSlownessTimeout(properties, TimeDuration.valueOf( conf.getRatisNodeFailureTimeout(), TimeUnit.MILLISECONDS)); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java index 5fbf2688b1aa..78e2cb2d084f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java @@ -203,16 +203,12 @@ public long getRatisRequestTimeout() { return ratisRequestTimeout; } - public long getRatisRequestMinTimeout() { - return ratisRequestTimeout - 1000L; - } - - public long getRatisRequestMaxTimeout() { - return ratisRequestTimeout + 1000L; + public long getLeaderElectionMinTimeout() { + return ratisLeaderElectionTimeout; } - public long getRatisLeaderElectionTimeout() { - return ratisLeaderElectionTimeout; + public long getLeaderElectionMaxTimeout() { + return ratisLeaderElectionTimeout + 200L; } public long getRatisNodeFailureTimeout() { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java index cbe2ce38ef41..c3259bcb5ce4 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java @@ -82,9 +82,11 @@ private Object invokeLocal(Method method, Object[] args) */ private Object invokeRatis(Method method, Object[] args) throws Exception { - LOG.trace("Invoking method {} on target {}", method, ratisHandler); + long startTime = Time.monotonicNowNanos(); final SCMRatisResponse response = ratisHandler.submitRequest( SCMRatisRequest.of(requestType, method.getName(), args)); + LOG.info("Invoking method {} on target {}, cost {}us", + method, ratisHandler, (Time.monotonicNowNanos() - startTime) / 1000.0); if (response.isSuccess()) { return response.getResult(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java index 3a81d2bb9033..e9594461e7aa 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -195,10 +196,7 @@ public RaftPeerId getPeerId() { InetAddress localHost = InetAddress.getLocalHost(); // fetch hosts from ozone.scm.names - List hosts = - Arrays.stream(conf.getTrimmedStrings(ScmConfigKeys.OZONE_SCM_NAMES)) - .map(scmName -> HddsUtils.getHostName(scmName).get()) - .collect(Collectors.toList()); + List hosts = parseHosts(conf); final List raftPeers = new ArrayList<>(); for (int i = 0; i < hosts.size(); ++i) { @@ -232,5 +230,29 @@ public RaftPeerId getPeerId() { raftGroup = RaftGroup.valueOf(raftGroupId, raftPeers); } + + private List parseHosts(final ConfigurationSource conf) + throws UnknownHostException { + // fetch hosts from ozone.scm.names + List hosts = + Arrays.stream(conf.getTrimmedStrings(ScmConfigKeys.OZONE_SCM_NAMES)) + .map(scmName -> HddsUtils.getHostName(scmName).get()) + .collect(Collectors.toList()); + + // if this is not a conf for a multi-server raft cluster, + // it means we are in integration test, and need to augment + // the conf to help build a single-server raft cluster. + if (hosts.size() == 0) { + // ozone.scm.names is not set + hosts.add(InetAddress.getLocalHost().getHostName()); + } else if (hosts.size() == 1) { + // ozone.scm.names is set, yet the conf may not be usable. + hosts.set(0, InetAddress.getLocalHost().getHostName()); + } + + LOG.info("fetch hosts {} from ozone.scm.names {}.", + hosts, conf.get(ScmConfigKeys.OZONE_SCM_NAMES)); + return hosts; + } } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java index 8b7d849842d0..67332bd940ba 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java @@ -181,79 +181,46 @@ public Pipeline createPipeline(ReplicationType type, ReplicationFactor factor, @Override public Pipeline getPipeline(PipelineID pipelineID) throws PipelineNotFoundException { - lock.readLock().lock(); - try { - return stateManager.getPipeline(pipelineID); - } finally { - lock.readLock().unlock(); - } + return stateManager.getPipeline(pipelineID); } @Override public boolean containsPipeline(PipelineID pipelineID) { - lock.readLock().lock(); try { getPipeline(pipelineID); return true; } catch (PipelineNotFoundException e) { return false; - } finally { - lock.readLock().unlock(); } } @Override public List getPipelines() { - lock.readLock().lock(); - try { - return stateManager.getPipelines(); - } finally { - lock.readLock().unlock(); - } + return stateManager.getPipelines(); } @Override public List getPipelines(ReplicationType type) { - lock.readLock().lock(); - try { - return stateManager.getPipelines(type); - } finally { - lock.readLock().unlock(); - } + return stateManager.getPipelines(type); } @Override public List getPipelines(ReplicationType type, ReplicationFactor factor) { - lock.readLock().lock(); - try { - return stateManager.getPipelines(type, factor); - } finally { - lock.readLock().unlock(); - } + return stateManager.getPipelines(type, factor); } @Override public List getPipelines(ReplicationType type, Pipeline.PipelineState state) { - lock.readLock().lock(); - try { - return stateManager.getPipelines(type, state); - } finally { - lock.readLock().unlock(); - } + return stateManager.getPipelines(type, state); } @Override public List getPipelines(ReplicationType type, ReplicationFactor factor, Pipeline.PipelineState state) { - lock.readLock().lock(); - try { - return stateManager.getPipelines(type, factor, state); - } finally { - lock.readLock().unlock(); - } + return stateManager.getPipelines(type, factor, state); } @Override @@ -261,46 +228,26 @@ public List getPipelines( ReplicationType type, ReplicationFactor factor, Pipeline.PipelineState state, Collection excludeDns, Collection excludePipelines) { - lock.readLock().lock(); - try { - return stateManager + return stateManager .getPipelines(type, factor, state, excludeDns, excludePipelines); - } finally { - lock.readLock().unlock(); - } } @Override public void addContainerToPipeline( PipelineID pipelineID, ContainerID containerID) throws IOException { - lock.writeLock().lock(); - try { - stateManager.addContainerToPipeline(pipelineID, containerID); - } finally { - lock.writeLock().unlock(); - } + stateManager.addContainerToPipeline(pipelineID, containerID); } @Override public void removeContainerFromPipeline( PipelineID pipelineID, ContainerID containerID) throws IOException { - lock.writeLock().lock(); - try { - stateManager.removeContainerFromPipeline(pipelineID, containerID); - } finally { - lock.writeLock().unlock(); - } + stateManager.removeContainerFromPipeline(pipelineID, containerID); } @Override public NavigableSet getContainersInPipeline( PipelineID pipelineID) throws IOException { - lock.readLock().lock(); - try { - return stateManager.getContainers(pipelineID); - } finally { - lock.readLock().unlock(); - } + return stateManager.getContainers(pipelineID); } @Override diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java index cbfda8ceb6b8..acfb41fe93f0 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java @@ -475,7 +475,6 @@ public static void quasiCloseContainer(ContainerManagerV2 containerManager, public static StorageContainerManager getScmSimple(OzoneConfiguration conf) throws IOException, AuthenticationException { SCMConfigurator configurator = new SCMConfigurator(); - configurator.setSCMHAManager(MockSCMHAManager.getInstance(true)); return StorageContainerManager.createSCM(conf, configurator); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java index a72a2570c810..7f28bddbd0b0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.junit.Assert; +import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -99,6 +100,12 @@ public void init() throws Exception { o3fs = (OzoneFileSystem) FileSystem.get(new URI(rootPath), conf); } + @After + public void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } @Test public void test() throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 340b902ddc1a..173832c6658b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -171,15 +171,18 @@ public void waitForClusterToBeReady() final int healthy = scm.getNodeCount(HEALTHY); final boolean isNodeReady = healthy == hddsDatanodes.size(); final boolean exitSafeMode = !scm.isInSafeMode(); + final boolean checkScmLeader = scm.checkLeader(); LOG.info("{}. Got {} of {} DN Heartbeats.", isNodeReady ? "Nodes are ready" : "Waiting for nodes to be ready", healthy, hddsDatanodes.size()); + LOG.info(checkScmLeader ? "SCM became leader" : + "SCM has not become leader"); LOG.info(exitSafeMode ? "Cluster exits safe mode" : "Waiting for cluster to exit safe mode", healthy, hddsDatanodes.size()); - return isNodeReady && exitSafeMode; + return isNodeReady && exitSafeMode && checkScmLeader; }, 1000, waitForClusterToBeReadyTimeout); }