diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java index 1bc16974362f..54661b3a2ba0 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java @@ -77,6 +77,10 @@ public static void setRaftStorageDir(final RaftProperties properties, String storageDir = haConf.getRatisStorageDir(); if (Strings.isNullOrEmpty(storageDir)) { storageDir = ServerUtils.getDefaultRatisDirectory(conf); + } else { + storageDir = + (new File(ServerUtils.getOzoneMetaDirPath(conf), storageDir)) + .getPath(); } RaftServerConfigKeys.setStorageDir(properties, Collections.singletonList(new File(storageDir))); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java index 5fbf2688b1aa..581ca83c73a6 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java @@ -38,7 +38,7 @@ public class SCMHAConfiguration { @Config(key = "ratis.storage.dir", type = ConfigType.STRING, - defaultValue = "", + defaultValue = "scm-ratis", tags = {OZONE, SCM, HA, RATIS}, description = "Storage directory used by SCM to write Ratis logs." ) @@ -165,6 +165,10 @@ public String getRatisStorageDir() { return ratisStorageDir; } + public void setRatisStorageDir(String dir) { + this.ratisStorageDir = dir; + } + public InetSocketAddress getRatisBindAddress() { return NetUtils.createSocketAddr(ratisBindHost, ratisBindPort); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java index 3a81d2bb9033..2240c1f6277e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java @@ -199,14 +199,14 @@ public RaftPeerId getPeerId() { Arrays.stream(conf.getTrimmedStrings(ScmConfigKeys.OZONE_SCM_NAMES)) .map(scmName -> HddsUtils.getHostName(scmName).get()) .collect(Collectors.toList()); - final List raftPeers = new ArrayList<>(); for (int i = 0; i < hosts.size(); ++i) { String nodeId = "scm" + i; RaftPeerId peerId = RaftPeerId.getRaftPeerId(nodeId); String host = hosts.get(i); - if (InetAddress.getByName(host).equals(localHost)) { + if (InetAddress.getByName(host).equals(localHost) + || host.toLowerCase().equals("localhost")) { selfPeerId = peerId; } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java index cbfda8ceb6b8..6c8f0b61ec15 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java @@ -475,7 +475,8 @@ public static void quasiCloseContainer(ContainerManagerV2 containerManager, public static StorageContainerManager getScmSimple(OzoneConfiguration conf) throws IOException, AuthenticationException { SCMConfigurator configurator = new SCMConfigurator(); - configurator.setSCMHAManager(MockSCMHAManager.getInstance(true)); + conf.set(ScmConfigKeys.OZONE_SCM_NAMES, "localhost"); + conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true); return StorageContainerManager.createSCM(conf, configurator); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java index a72a2570c810..825bb5f1148a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java @@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -99,6 +100,12 @@ public void init() throws Exception { o3fs = (OzoneFileSystem) FileSystem.get(new URI(rootPath), conf); } + @After + public void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } @Test public void test() throws Exception { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index d1d51624c0a2..8e1fe6b660f8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -307,12 +307,18 @@ abstract class Builder { protected boolean startDataNodes = true; protected CertificateClient certClient; protected int pipelineNumLimit = DEFAULT_PIPELIME_LIMIT; + protected boolean useMockSCMHAManager = false; protected Builder(OzoneConfiguration conf) { this.conf = conf; setClusterId(UUID.randomUUID().toString()); } + public Builder setUseMockSCMHAManager(boolean use) { + this.useMockSCMHAManager = use; + return this; + } + /** * Sets the cluster Id. * diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 340b902ddc1a..e61ca4bec5ca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -169,17 +169,21 @@ public void waitForClusterToBeReady() throws TimeoutException, InterruptedException { GenericTestUtils.waitFor(() -> { final int healthy = scm.getNodeCount(HEALTHY); + // When SCM HA is not enabled, scm is always leader. + final boolean checkScmLeader = scm.checkLeader(); final boolean isNodeReady = healthy == hddsDatanodes.size(); final boolean exitSafeMode = !scm.isInSafeMode(); LOG.info("{}. Got {} of {} DN Heartbeats.", isNodeReady ? "Nodes are ready" : "Waiting for nodes to be ready", healthy, hddsDatanodes.size()); + LOG.info(checkScmLeader ? "SCM became leader" : + "SCM has not become leader"); LOG.info(exitSafeMode ? "Cluster exits safe mode" : "Waiting for cluster to exit safe mode", healthy, hddsDatanodes.size()); - return isNodeReady && exitSafeMode; + return isNodeReady && exitSafeMode && checkScmLeader; }, 1000, waitForClusterToBeReadyTimeout); } @@ -631,7 +635,12 @@ protected StorageContainerManager createSCM() configureSCM(); SCMStorageConfig scmStore = new SCMStorageConfig(conf); initializeScmStorage(scmStore); - StorageContainerManager scm = TestUtils.getScmSimple(conf); + StorageContainerManager scm; + if (useMockSCMHAManager) { + scm = TestUtils.getScm(conf); + } else { + scm = TestUtils.getScmSimple(conf); + } HealthyPipelineSafeModeRule rule = scm.getScmSafeModeManager().getHealthyPipelineSafeModeRule(); if (rule != null) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java index 34f6964be94d..72fecb89967f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java @@ -140,7 +140,7 @@ public static void init() throws Exception { conf.setQuietMode(false); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10).setHbInterval(200) - .build(); + .setUseMockSCMHAManager(true).build(); cluster.waitForClusterToBeReady(); cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.ONE, 60000); //the easiest way to create an open container is creating a key diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java index 8217e3baeb09..c1b8ca70176c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestKeyInputStream.java @@ -153,6 +153,7 @@ public void init() throws Exception { .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setUseMockSCMHAManager(true) .build(); cluster.waitForClusterToBeReady(); //the easiest way to create an open container is creating a key diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java index 94930f355acd..014cd06675c6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java @@ -110,6 +110,7 @@ public void init() throws Exception { .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setUseMockSCMHAManager(true) .build(); cluster.waitForClusterToBeReady(); //the easiest way to create an open container is creating a key diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java index 75b6c8cfdcdd..b8705755990d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java @@ -121,6 +121,7 @@ public void setUp() throws Exception { cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(numOfDatanodes) + .setUseMockSCMHAManager(true) .build(); cluster.waitForClusterToBeReady(); setManagers();