diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java index 1cb8c65675f8..5fbf2688b1aa 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java @@ -53,7 +53,7 @@ public class SCMHAConfiguration { private String ratisBindHost = "0.0.0.0"; @Config(key = "ratis.bind.port", - type = ConfigType.STRING, + type = ConfigType.INT, defaultValue = "9865", tags = {OZONE, SCM, HA, RATIS}, description = "Port used by SCM for Ratis Server." @@ -78,7 +78,7 @@ public class SCMHAConfiguration { description = "The size of the raft segment used by Apache Ratis on" + " SCM. (16 KB by default)" ) - private long raftSegmentSize = 16L * 1024L; + private double raftSegmentSize = 16L * 1024L; @Config(key = "ratis.segment.preallocated.size", type = ConfigType.SIZE, @@ -87,7 +87,7 @@ public class SCMHAConfiguration { description = "The size of the buffer which is preallocated for" + " raft segment used by Apache Ratis on SCM.(16 KB by default)" ) - private long raftSegmentPreAllocatedSize = 16 * 1024; + private double raftSegmentPreAllocatedSize = 16 * 1024; @Config(key = "ratis.log.appender.queue.num-elements", type = ConfigType.INT, @@ -103,7 +103,7 @@ public class SCMHAConfiguration { tags = {SCM, OZONE, HA, RATIS}, description = "Byte limit for Raft's Log Worker queue." ) - private int raftLogAppenderQueueByteLimit = 32 * 1024 * 1024; + private double raftLogAppenderQueueByteLimit = 32 * 1024 * 1024; @Config(key = "ratis.log.purge.gap", type = ConfigType.INT, @@ -174,11 +174,11 @@ public String getRatisRpcType() { } public long getRaftSegmentSize() { - return raftSegmentSize; + return (long)raftSegmentSize; } public long getRaftSegmentPreAllocatedSize() { - return raftSegmentPreAllocatedSize; + return (long)raftSegmentPreAllocatedSize; } public int getRaftLogAppenderQueueNum() { @@ -186,7 +186,7 @@ public int getRaftLogAppenderQueueNum() { } public int getRaftLogAppenderQueueByteLimit() { - return raftLogAppenderQueueByteLimit; + return (int)raftLogAppenderQueueByteLimit; } public int getRaftLogPurgeGap() { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java index 4ab545776080..0a609c7a0f4e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java @@ -102,7 +102,11 @@ public void start(OzoneConfiguration config) pipelineTable = PIPELINES.getTable(store); + checkTableStatus(pipelineTable, PIPELINES.getName()); + containerTable = CONTAINERS.getTable(store); + + checkTableStatus(containerTable, CONTAINERS.getName()); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java index 85654aa03164..12417457e133 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java @@ -160,7 +160,7 @@ public Pipeline createPipeline(ReplicationType type, @Override public Pipeline createPipeline(ReplicationType type, ReplicationFactor factor, - List nodes) { + List nodes) { // This will mostly be used to create dummy pipeline for SimplePipelines. // We don't update the metrics for SimplePipelines. lock.writeLock().lock(); @@ -207,6 +207,7 @@ public List getPipelines() { @Override public List getPipelines(ReplicationType type) { + lock.readLock().lock(); try { return stateManager.getPipelines(type); } finally { @@ -582,6 +583,17 @@ public void allowPipelineCreation() { this.pipelineCreationAllowed.set(true); } + @VisibleForTesting + public void setPipelineProvider(ReplicationType replicationType, + PipelineProvider provider) { + pipelineFactory.setProvider(replicationType, provider); + } + + @VisibleForTesting + public StateManager getStateManager() { + return stateManager; + } + private void setBackgroundPipelineCreator( BackgroundPipelineCreator backgroundPipelineCreator) { this.backgroundPipelineCreator = backgroundPipelineCreator; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMConfigurator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMConfigurator.java index 9bbabd11ee0f..d9b511da2631 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMConfigurator.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMConfigurator.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.scm.block.BlockManager; +import org.apache.hadoop.hdds.scm.ha.SCMHAManager; import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.container.ContainerManager; @@ -51,7 +52,8 @@ * ReplicationManager replicationManager; * SCMSafeModeManager scmSafeModeManager; * CertificateServer certificateServer; - * SCMMetadata scmMetadataStore. + * SCMMetadata scmMetadataStore; + * SCMHAManager scmHAManager. * * If any of these are *not* specified then the default version of these * managers are used by SCM. @@ -67,6 +69,7 @@ public final class SCMConfigurator { private CertificateServer certificateServer; private SCMMetadataStore metadataStore; private NetworkTopology networkTopology; + private SCMHAManager scmHAManager; /** * Allows user to specify a version of Node manager to use with this SCM. @@ -148,6 +151,15 @@ public void setNetworkTopology(NetworkTopology networkTopology) { this.networkTopology = networkTopology; } + /** + * Allows user to specify a custom version of SCMHAManager to be + * used with this SCM. + * @param scmHaMgr - SCMHAManager. + */ + public void setSCMHAManager(SCMHAManager scmHaMgr) { + this.scmHAManager = scmHaMgr; + } + /** * Gets SCM Node Manager. * @return Node Manager. @@ -219,4 +231,12 @@ public SCMMetadataStore getMetadataStore() { public NetworkTopology getNetworkTopology() { return networkTopology; } + + /** + * Get SCMHAManager. + * @return SCMHAManager. + */ + public SCMHAManager getSCMHAManager() { + return scmHAManager; + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 20d0480a428b..4ba40f960a45 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -51,6 +51,8 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; import org.apache.hadoop.hdds.scm.PlacementPolicy; +import org.apache.hadoop.hdds.scm.ha.SCMHAManager; +import org.apache.hadoop.hdds.scm.ha.SCMHAManagerImpl; import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails; import org.apache.hadoop.hdds.scm.server.ratis.SCMRatisServer; @@ -94,7 +96,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineActionHandler; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.security.exception.SCMSecurityException; import org.apache.hadoop.hdds.security.x509.SecurityConfig; @@ -170,6 +172,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private final SCMStorageConfig scmStorageConfig; private SCMMetadataStore scmMetadataStore; + private SCMHAManager scmHAManager; private final EventQueue eventQueue; /* @@ -237,7 +240,7 @@ private StorageContainerManager(OzoneConfiguration conf) * @param configurator - configurator */ private StorageContainerManager(OzoneConfiguration conf, - SCMConfigurator configurator) + SCMConfigurator configurator) throws IOException, AuthenticationException { super(HddsVersionInfo.HDDS_VERSION_INFO); @@ -439,6 +442,12 @@ private void initializeSystemManagers(OzoneConfiguration conf, clusterMap = new NetworkTopologyImpl(conf); } + if (configurator.getSCMHAManager() != null) { + scmHAManager = configurator.getSCMHAManager(); + } else { + scmHAManager = new SCMHAManagerImpl(conf); + } + if(configurator.getScmNodeManager() != null) { scmNodeManager = configurator.getScmNodeManager(); } else { @@ -455,7 +464,10 @@ private void initializeSystemManagers(OzoneConfiguration conf, pipelineManager = configurator.getPipelineManager(); } else { pipelineManager = - new SCMPipelineManager(conf, scmNodeManager, + PipelineManagerV2Impl.newPipelineManager( + conf, + scmHAManager, + scmNodeManager, scmMetadataStore.getPipelineTable(), eventQueue); } @@ -825,6 +837,8 @@ public void start() throws IOException { scmRatisServer.start(); } + scmHAManager.start(); + ms = HddsServerUtil .initializeMetrics(configuration, "StorageContainerManager"); @@ -957,6 +971,12 @@ public void stop() { ms.stop(); } + try { + scmHAManager.shutdown(); + } catch (Exception ex) { + LOG.error("SCM HA Manager stop failed", ex); + } + scmSafeModeManager.stop(); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java index 4039b5a68f24..d4d11ffd6218 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java @@ -17,23 +17,16 @@ */ package org.apache.hadoop.hdds.scm; -import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.UUID; import org.apache.commons.lang3.RandomUtils; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer .NodeRegistrationContainerReport; -import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; -import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.ozone.common.Storage; -import org.apache.hadoop.security.authentication.client.AuthenticationException; /** * Stateless helper functions for Hdds tests. @@ -74,24 +67,6 @@ private HddsTestUtils() { TestUtils.getContainerReports(containers)); } - public static StorageContainerManager getScm(OzoneConfiguration conf) - throws IOException, AuthenticationException { - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0"); - conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, "127.0.0.1:0"); - conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "127.0.0.1:0"); - conf.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "127.0.0.1:0"); - SCMStorageConfig scmStore = new SCMStorageConfig(conf); - if(scmStore.getState() != Storage.StorageState.INITIALIZED) { - String clusterId = UUID.randomUUID().toString(); - String scmId = UUID.randomUUID().toString(); - scmStore.setClusterId(clusterId); - scmStore.setScmId(scmId); - // writes the version file properties - scmStore.initialize(); - } - return StorageContainerManager.createSCM(conf); - } - /** * Creates list of ContainerInfo. * diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java index 5d1ed4694680..ec80dfe4820f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java @@ -34,6 +34,7 @@ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; @@ -455,6 +456,22 @@ public static void quasiCloseContainer(ContainerManager containerManager, } + /** + * Construct and returns StorageContainerManager instance using the given + * configuration. + * + * @param conf OzoneConfiguration + * @return StorageContainerManager instance + * @throws IOException + * @throws AuthenticationException + */ + public static StorageContainerManager getScmSimple(OzoneConfiguration conf) + throws IOException, AuthenticationException { + SCMConfigurator configurator = new SCMConfigurator(); + configurator.setSCMHAManager(MockSCMHAManager.getInstance()); + return StorageContainerManager.createSCM(conf, configurator); + } + /** * Construct and returns StorageContainerManager instance using the given * configuration. The ports used by this StorageContainerManager are @@ -467,7 +484,9 @@ public static void quasiCloseContainer(ContainerManager containerManager, */ public static StorageContainerManager getScm(OzoneConfiguration conf) throws IOException, AuthenticationException { - return getScm(conf, new SCMConfigurator()); + SCMConfigurator configurator = new SCMConfigurator(); + configurator.setSCMHAManager(MockSCMHAManager.getInstance()); + return getScm(conf, configurator); } /** diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java index cf6f0ed735ff..4f8c1d69417f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java @@ -41,12 +41,14 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager; +import org.apache.hadoop.hdds.scm.ha.SCMHAManager; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; @@ -76,8 +78,9 @@ public class TestBlockManager { private StorageContainerManager scm; private SCMContainerManager mapping; private MockNodeManager nodeManager; - private SCMPipelineManager pipelineManager; + private PipelineManagerV2Impl pipelineManager; private BlockManagerImpl blockManager; + private SCMHAManager scmHAManager; private final static long DEFAULT_BLOCK_SIZE = 128 * MB; private static HddsProtos.ReplicationFactor factor; private static HddsProtos.ReplicationType type; @@ -105,14 +108,20 @@ public void setUp() throws Exception { conf.setTimeDuration(HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL, 5, TimeUnit.SECONDS); - // Override the default Node Manager in SCM with this Mock Node Manager. + // Override the default Node Manager and SCMHAManager + // in SCM with the Mock one. nodeManager = new MockNodeManager(true, 10); + scmHAManager = MockSCMHAManager.getInstance(); + eventQueue = new EventQueue(); scmMetadataStore = new SCMMetadataStoreImpl(conf); scmMetadataStore.start(conf); pipelineManager = - new SCMPipelineManager(conf, nodeManager, + PipelineManagerV2Impl.newPipelineManager( + conf, + scmHAManager, + nodeManager, scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); @@ -140,6 +149,7 @@ public void emitSafeModeStatus() { configurator.setContainerManager(containerManager); configurator.setScmSafeModeManager(safeModeManager); configurator.setMetadataStore(scmMetadataStore); + configurator.setSCMHAManager(scmHAManager); scm = TestUtils.getScm(conf, configurator); // Initialize these fields so that the tests can pass. diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java index b080ea1a8204..daa97266d8e8 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java @@ -28,11 +28,12 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; @@ -55,7 +56,7 @@ public class TestCloseContainerEventHandler { private static OzoneConfiguration configuration; private static MockNodeManager nodeManager; - private static SCMPipelineManager pipelineManager; + private static PipelineManagerV2Impl pipelineManager; private static SCMContainerManager containerManager; private static long size; private static File testDir; @@ -77,8 +78,13 @@ public static void setUp() throws Exception { scmMetadataStore = new SCMMetadataStoreImpl(configuration); pipelineManager = - new SCMPipelineManager(configuration, nodeManager, - scmMetadataStore.getPipelineTable(), eventQueue); + PipelineManagerV2Impl.newPipelineManager( + configuration, + MockSCMHAManager.getInstance(), + nodeManager, + scmMetadataStore.getPipelineTable(), + eventQueue); + pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java index 859eef7ca9a0..b8bae2225257 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java @@ -31,7 +31,6 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -107,7 +106,7 @@ private void addReplica(ContainerInfo cont, DatanodeDetails node) private ContainerInfo allocateContainer() throws IOException { - PipelineManager pipelineManager = Mockito.mock(SCMPipelineManager.class); + PipelineManager pipelineManager = Mockito.mock(PipelineManager.class); Pipeline pipeline = Pipeline.newBuilder().setState(Pipeline.PipelineState.CLOSED) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java index 12c62a956993..25650762bc59 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java @@ -42,10 +42,11 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.XceiverClientManager; +import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; @@ -66,7 +67,7 @@ public class TestSCMContainerManager { private static SCMContainerManager containerManager; private static MockNodeManager nodeManager; - private static SCMPipelineManager pipelineManager; + private static PipelineManagerV2Impl pipelineManager; private static File testDir; private static XceiverClientManager xceiverClientManager; private static Random random; @@ -92,9 +93,12 @@ public static void setUp() throws Exception { } nodeManager = new MockNodeManager(true, 10); SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(conf); - pipelineManager = - new SCMPipelineManager(conf, nodeManager, - scmMetadataStore.getPipelineTable(), new EventQueue()); + pipelineManager = PipelineManagerV2Impl.newPipelineManager( + conf, + MockSCMHAManager.getInstance(), + nodeManager, + scmMetadataStore.getPipelineTable(), + new EventQueue()); pipelineManager.allowPipelineCreation(); containerManager = new SCMContainerManager(conf, scmMetadataStore.getContainerTable(), diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java index 797709ecc5f2..7fea0c52d736 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java @@ -36,10 +36,11 @@ import org.apache.hadoop.hdds.scm.container.SCMContainerManager; import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.ozone.OzoneConsts; @@ -115,8 +116,13 @@ SCMContainerManager createContainerManager(ConfigurationSource config, EventQueue eventQueue = new EventQueue(); PipelineManager pipelineManager = - new SCMPipelineManager(config, scmNodeManager, - scmMetadataStore.getPipelineTable(), eventQueue); + PipelineManagerV2Impl.newPipelineManager( + config, + MockSCMHAManager.getInstance(), + scmNodeManager, + scmMetadataStore.getPipelineTable(), + eventQueue); + return new SCMContainerManager(config, scmMetadataStore.getContainerTable(), scmMetadataStore.getStore(), pipelineManager); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java index 6a6d3284465b..bfb211dbd97c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java @@ -40,7 +40,6 @@ .StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.StorageReportProto; -import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.ContainerManager; @@ -51,7 +50,7 @@ import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher .NodeReportFromDatanode; @@ -78,7 +77,7 @@ public class TestDeadNodeHandler { private SCMNodeManager nodeManager; private ContainerManager containerManager; private NodeReportHandler nodeReportHandler; - private SCMPipelineManager pipelineManager; + private PipelineManagerV2Impl pipelineManager; private DeadNodeHandler deadNodeHandler; private EventPublisher publisher; private EventQueue eventQueue; @@ -93,10 +92,10 @@ public void setup() throws IOException, AuthenticationException { TestDeadNodeHandler.class.getSimpleName() + UUID.randomUUID()); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); eventQueue = new EventQueue(); - scm = HddsTestUtils.getScm(conf); + scm = TestUtils.getScm(conf); nodeManager = (SCMNodeManager) scm.getScmNodeManager(); pipelineManager = - (SCMPipelineManager)scm.getPipelineManager(); + (PipelineManagerV2Impl)scm.getPipelineManager(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), conf); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index b167a38b7254..d06b641f517f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto; -import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat; @@ -137,7 +136,7 @@ OzoneConfiguration getConf() { SCMNodeManager createNodeManager(OzoneConfiguration config) throws IOException, AuthenticationException { - scm = HddsTestUtils.getScm(config); + scm = TestUtils.getScm(config); return (SCMNodeManager) scm.getScmNodeManager(); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestStatisticsUpdate.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestStatisticsUpdate.java index a6b033923fc4..0ebab870d515 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestStatisticsUpdate.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestStatisticsUpdate.java @@ -26,7 +26,6 @@ .StorageContainerDatanodeProtocolProtos.StorageReportProto; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.NodeReportProto; -import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric; @@ -68,7 +67,7 @@ public void setup() throws IOException, AuthenticationException { conf.set(ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL, "1s"); conf.set(ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL, "2s"); final EventQueue eventQueue = new EventQueue(); - final StorageContainerManager scm = HddsTestUtils.getScm(conf); + final StorageContainerManager scm = TestUtils.getScm(conf); nodeManager = scm.getScmNodeManager(); final DeadNodeHandler deadNodeHandler = new DeadNodeHandler( nodeManager, Mockito.mock(PipelineManager.class), diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java index e355877e269c..49cac8b9f034 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockRatisPipelineProvider.java @@ -34,7 +34,6 @@ public class MockRatisPipelineProvider extends RatisPipelineProvider { private boolean autoOpenPipeline; - private boolean isHealthy; public MockRatisPipelineProvider( NodeManager nodeManager, StateManager stateManager, @@ -50,14 +49,6 @@ public MockRatisPipelineProvider(NodeManager nodeManager, super(nodeManager, stateManager, conf, new EventQueue()); } - public MockRatisPipelineProvider(NodeManager nodeManager, - StateManager stateManager, - ConfigurationSource conf, - boolean isHealthy) { - super(nodeManager, stateManager, conf, new EventQueue()); - this.isHealthy = isHealthy; - } - public MockRatisPipelineProvider( NodeManager nodeManager, StateManager stateManager, ConfigurationSource conf, EventPublisher eventPublisher) { @@ -84,16 +75,18 @@ public Pipeline create(HddsProtos.ReplicationFactor factor) .setFactor(factor) .setNodes(initialPipeline.getNodes()) .build(); - if (isHealthy) { - for (DatanodeDetails datanodeDetails : initialPipeline.getNodes()) { - pipeline.reportDatanode(datanodeDetails); - } - pipeline.setLeaderId(initialPipeline.getFirstNode().getUuid()); - } return pipeline; } } + public static void markPipelineHealthy(Pipeline pipeline) + throws IOException { + for (DatanodeDetails datanodeDetails : pipeline.getNodes()) { + pipeline.reportDatanode(datanodeDetails); + } + pipeline.setLeaderId(pipeline.getFirstNode().getUuid()); + } + @Override public void shutdown() { // Do nothing. diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java index d5292e3153c0..e1f9104c9d15 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.MockNodeManager; -import org.apache.hadoop.hdds.scm.container.TestContainerManagerImpl; import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager; import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; @@ -73,7 +72,7 @@ public class TestPipelineManagerImpl { public void init() throws Exception { conf = SCMTestUtils.getConf(); testDir = GenericTestUtils.getTestDir( - TestContainerManagerImpl.class.getSimpleName() + UUID.randomUUID()); + TestPipelineManagerImpl.class.getSimpleName() + UUID.randomUUID()); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath()); dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition()); nodeManager = new MockNodeManager(true, 20); @@ -200,8 +199,8 @@ public void testRemovePipeline() throws Exception { pipelineManager.getPipeline(pipeline.getId()); fail("Pipeline should not have been retrieved"); } catch (PipelineNotFoundException e) { - // There should be no pipeline in pipelineManager. - Assert.assertEquals(0, pipelineManager.getPipelines().size()); + // There may be pipelines created by BackgroundPipelineCreator + // exist in pipelineManager, just ignore them. } pipelineManager.close(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java index 7a40e3e47c9e..f35318d9c3bc 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java @@ -31,12 +31,13 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.test.GenericTestUtils; @@ -70,8 +71,13 @@ public void testHealthyPipelineSafeModeRuleWithNoPipelines() SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config); try { - SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, scmMetadataStore.getPipelineTable(), eventQueue); + PipelineManagerV2Impl pipelineManager = + PipelineManagerV2Impl.newPipelineManager( + config, + MockSCMHAManager.getInstance(), + nodeManager, + scmMetadataStore.getPipelineTable(), + eventQueue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), config); @@ -114,13 +120,18 @@ public void testHealthyPipelineSafeModeRuleWithPipelines() throws Exception { SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config); try { - SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, scmMetadataStore.getPipelineTable(), eventQueue); + PipelineManagerV2Impl pipelineManager = + PipelineManagerV2Impl.newPipelineManager( + config, + MockSCMHAManager.getInstance(), + nodeManager, + scmMetadataStore.getPipelineTable(), + eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, - pipelineManager.getStateManager(), config, true); + pipelineManager.getStateManager(), config); pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider); @@ -135,6 +146,16 @@ public void testHealthyPipelineSafeModeRuleWithPipelines() throws Exception { pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE); + // Mark pipeline healthy + pipeline1 = pipelineManager.getPipeline(pipeline1.getId()); + MockRatisPipelineProvider.markPipelineHealthy(pipeline1); + + pipeline2 = pipelineManager.getPipeline(pipeline2.getId()); + MockRatisPipelineProvider.markPipelineHealthy(pipeline2); + + pipeline3 = pipelineManager.getPipeline(pipeline3.getId()); + MockRatisPipelineProvider.markPipelineHealthy(pipeline3); + SCMSafeModeManager scmSafeModeManager = new SCMSafeModeManager( config, containers, pipelineManager, eventQueue); @@ -190,13 +211,18 @@ public void testHealthyPipelineSafeModeRuleWithMixedPipelines() SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config); try { - SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, scmMetadataStore.getPipelineTable(), eventQueue); + PipelineManagerV2Impl pipelineManager = + PipelineManagerV2Impl.newPipelineManager( + config, + MockSCMHAManager.getInstance(), + nodeManager, + scmMetadataStore.getPipelineTable(), + eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, - pipelineManager.getStateManager(), config, true); + pipelineManager.getStateManager(), config); pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider); @@ -211,6 +237,15 @@ public void testHealthyPipelineSafeModeRuleWithMixedPipelines() pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE); + // Mark pipeline healthy + pipeline1 = pipelineManager.getPipeline(pipeline1.getId()); + MockRatisPipelineProvider.markPipelineHealthy(pipeline1); + + pipeline2 = pipelineManager.getPipeline(pipeline2.getId()); + MockRatisPipelineProvider.markPipelineHealthy(pipeline2); + + pipeline3 = pipelineManager.getPipeline(pipeline3.getId()); + MockRatisPipelineProvider.markPipelineHealthy(pipeline3); SCMSafeModeManager scmSafeModeManager = new SCMSafeModeManager( config, containers, pipelineManager, eventQueue); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java index 4e1cf6fcb2d3..5aa67a39007a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java @@ -27,12 +27,13 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl; import org.apache.hadoop.hdds.server.events.EventQueue; import org.apache.hadoop.test.GenericTestUtils; @@ -50,7 +51,7 @@ public class TestOneReplicaPipelineSafeModeRule { @Rule public TemporaryFolder folder = new TemporaryFolder(); private OneReplicaPipelineSafeModeRule rule; - private SCMPipelineManager pipelineManager; + private PipelineManagerV2Impl pipelineManager; private EventQueue eventQueue; private void setup(int nodes, int pipelineFactorThreeCount, @@ -72,10 +73,12 @@ private void setup(int nodes, int pipelineFactorThreeCount, SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(ozoneConfiguration); - pipelineManager = - new SCMPipelineManager(ozoneConfiguration, mockNodeManager, - scmMetadataStore.getPipelineTable(), - eventQueue); + pipelineManager = PipelineManagerV2Impl.newPipelineManager( + ozoneConfiguration, + MockSCMHAManager.getInstance(), + mockNodeManager, + scmMetadataStore.getPipelineTable(), + eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java index 418d945f6383..935dc7761022 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java @@ -35,13 +35,14 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus; import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; @@ -295,8 +296,13 @@ public void testFailWithIncorrectValueForHealthyPipelinePercent() OzoneConfiguration conf = createConf(100, 0.9); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); - PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, scmMetadataStore.getPipelineTable(), queue); + PipelineManager pipelineManager = + PipelineManagerV2Impl.newPipelineManager( + conf, + MockSCMHAManager.getInstance(), + mockNodeManager, + scmMetadataStore.getPipelineTable(), + queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForHealthyPipelinePercent"); @@ -313,8 +319,13 @@ public void testFailWithIncorrectValueForOneReplicaPipelinePercent() OzoneConfiguration conf = createConf(0.9, 200); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); - PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, scmMetadataStore.getPipelineTable(), queue); + PipelineManager pipelineManager = + PipelineManagerV2Impl.newPipelineManager( + conf, + MockSCMHAManager.getInstance(), + mockNodeManager, + scmMetadataStore.getPipelineTable(), + queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForOneReplicaPipelinePercent"); @@ -330,8 +341,13 @@ public void testFailWithIncorrectValueForSafeModePercent() throws Exception { OzoneConfiguration conf = createConf(0.9, 0.1); conf.setDouble(HddsConfigKeys.HDDS_SCM_SAFEMODE_THRESHOLD_PCT, -1.0); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); - PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, scmMetadataStore.getPipelineTable(), queue); + PipelineManager pipelineManager = + PipelineManagerV2Impl.newPipelineManager( + conf, + MockSCMHAManager.getInstance(), + mockNodeManager, + scmMetadataStore.getPipelineTable(), + queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForSafeModePercent"); @@ -354,18 +370,29 @@ public void testSafeModeExitRuleWithPipelineAvailabilityCheck( containers.addAll(HddsTestUtils.getContainerInfo(containerCount)); MockNodeManager mockNodeManager = new MockNodeManager(true, nodeCount); - SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, scmMetadataStore.getPipelineTable(), queue); + PipelineManagerV2Impl pipelineManager = + PipelineManagerV2Impl.newPipelineManager( + conf, + MockSCMHAManager.getInstance(), + mockNodeManager, + scmMetadataStore.getPipelineTable(), + queue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(mockNodeManager, - pipelineManager.getStateManager(), config, true); + pipelineManager.getStateManager(), config); pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider); pipelineManager.allowPipelineCreation(); - for (int i=0; i < pipelineCount; i++) { - pipelineManager.createPipeline(HddsProtos.ReplicationType.RATIS, + for (int i = 0; i < pipelineCount; i++) { + // Create pipeline + Pipeline pipeline = pipelineManager.createPipeline( + HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE); + + // Mark pipeline healthy + pipeline = pipelineManager.getPipeline(pipeline.getId()); + MockRatisPipelineProvider.markPipelineHealthy(pipeline); } for (ContainerInfo container : containers) { @@ -449,7 +476,7 @@ private void checkOpen(int expectedCount) throws Exception { 1000, 5000); } - private void firePipelineEvent(SCMPipelineManager pipelineManager, + private void firePipelineEvent(PipelineManager pipelineManager, Pipeline pipeline) throws Exception { pipelineManager.openPipeline(pipeline.getId()); queue.fireEvent(SCMEvents.OPEN_PIPELINE, @@ -571,12 +598,17 @@ public void testSafeModePipelineExitRule() throws Exception { config.setBoolean( HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); - SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, scmMetadataStore.getPipelineTable(), queue); + PipelineManagerV2Impl pipelineManager = + PipelineManagerV2Impl.newPipelineManager( + config, + MockSCMHAManager.getInstance(), + nodeManager, + scmMetadataStore.getPipelineTable(), + queue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, - pipelineManager.getStateManager(), config, true); + pipelineManager.getStateManager(), config); pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider); pipelineManager.allowPipelineCreation(); @@ -585,6 +617,9 @@ public void testSafeModePipelineExitRule() throws Exception { HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE); + pipeline = pipelineManager.getPipeline(pipeline.getId()); + MockRatisPipelineProvider.markPipelineHealthy(pipeline); + scmSafeModeManager = new SCMSafeModeManager( config, containers, pipelineManager, queue); @@ -625,13 +660,17 @@ public void testPipelinesNotCreatedUntilPreCheckPasses() config.setBoolean( HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); - SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, scmMetadataStore.getPipelineTable(), queue); - + PipelineManagerV2Impl pipelineManager = + PipelineManagerV2Impl.newPipelineManager( + config, + MockSCMHAManager.getInstance(), + nodeManager, + scmMetadataStore.getPipelineTable(), + queue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, - pipelineManager.getStateManager(), config, true); + pipelineManager.getStateManager(), config); pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider); @@ -668,6 +707,11 @@ public void testPipelinesNotCreatedUntilPreCheckPasses() Pipeline pipeline = pipelineManager.createPipeline( HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE); + + // Mark pipeline healthy + pipeline = pipelineManager.getPipeline(pipeline.getId()); + MockRatisPipelineProvider.markPipelineHealthy(pipeline); + firePipelineEvent(pipelineManager, pipeline); queue.processAll(5000); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java index 349e705956df..f4553abd736c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos; import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocolServerSideTranslatorPB; @@ -58,6 +59,7 @@ public void setUp() throws Exception { File dir = GenericTestUtils.getRandomizedTestDir(); config.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.toString()); SCMConfigurator configurator = new SCMConfigurator(); + configurator.setSCMHAManager(MockSCMHAManager.getInstance()); scm = TestUtils.getScm(config, configurator); scm.start(); scm.exitSafeMode(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java index 40799655b44e..8a233aeca13e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMRatisServer.java @@ -20,8 +20,8 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; @@ -72,7 +72,7 @@ public void init() throws Exception { // Standalone SCM Ratis server initSCM(); - scm = HddsTestUtils.getScm(conf); + scm = TestUtils.getScm(conf); scm.start(); scmRatisServer = scm.getScmRatisServer(); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMStateMachine.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMStateMachine.java index 69bc5bd93b68..0eddbdec0441 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMStateMachine.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/ratis/TestSCMStateMachine.java @@ -17,8 +17,8 @@ package org.apache.hadoop.hdds.scm.server.ratis; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.junit.After; @@ -56,7 +56,7 @@ public void init() throws Exception { scmId = UUID.randomUUID().toString(); initSCM(); - scm = HddsTestUtils.getScm(conf); + scm = TestUtils.getScm(conf); scm.start(); scmRatisServer = scm.getScmRatisServer(); scmStateMachine = scm.getScmRatisServer().getScmStateMachine(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 5baa65b43c37..649c4a93e47d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB; @@ -302,7 +303,7 @@ public void restartStorageContainerManager(boolean waitForDatanode) AuthenticationException { scm.stop(); scm.join(); - scm = StorageContainerManager.createSCM(conf); + scm = TestUtils.getScmSimple(conf); scm.start(); if (waitForDatanode) { waitForClusterToBeReady(); @@ -628,7 +629,7 @@ protected StorageContainerManager createSCM() configureSCM(); SCMStorageConfig scmStore = new SCMStorageConfig(conf); initializeScmStorage(scmStore); - StorageContainerManager scm = StorageContainerManager.createSCM(conf); + StorageContainerManager scm = TestUtils.getScmSimple(conf); HealthyPipelineSafeModeRule rule = scm.getScmSafeModeManager().getHealthyPipelineSafeModeRule(); if (rule != null) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index 291f19f88e49..a9fa1e52bdc9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -36,9 +36,9 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; -import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.ScmConfig; import org.apache.hadoop.hdds.scm.ScmInfo; +import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; @@ -267,7 +267,7 @@ private void setSecureConfig() throws IOException { public void testSecureScmStartupSuccess() throws Exception { initSCM(); - scm = StorageContainerManager.createSCM(conf); + scm = TestUtils.getScmSimple(conf); //Reads the SCM Info from SCM instance ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo(); assertEquals(clusterId, scmInfo.getClusterId()); @@ -278,7 +278,7 @@ public void testSecureScmStartupSuccess() throws Exception { public void testSCMSecurityProtocol() throws Exception { initSCM(); - scm = HddsTestUtils.getScm(conf); + scm = TestUtils.getScmSimple(conf); //Reads the SCM Info from SCM instance try { scm.start(); @@ -338,7 +338,7 @@ public void testSecureScmStartupFailure() throws Exception { LambdaTestUtils.intercept(IOException.class, "Running in secure mode, but config doesn't have a keytab", - () -> StorageContainerManager.createSCM(conf)); + () -> TestUtils.getScmSimple(conf)); conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/_HOST@EXAMPLE.com"); @@ -346,7 +346,7 @@ public void testSecureScmStartupFailure() throws Exception { "/etc/security/keytabs/scm.keytab"); testCommonKerberosFailures( - () -> StorageContainerManager.createSCM(conf)); + () -> TestUtils.getScmSimple(conf)); } @@ -375,7 +375,7 @@ private void testCommonKerberosFailures(Callable test) throws Exception { public void testSecureOMInitializationFailure() throws Exception { initSCM(); // Create a secure SCM instance as om client will connect to it - scm = StorageContainerManager.createSCM(conf); + scm = TestUtils.getScmSimple(conf); setupOm(conf); conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, "non-existent-user@EXAMPLE.com"); @@ -389,7 +389,7 @@ public void testSecureOMInitializationFailure() throws Exception { public void testSecureOmInitializationSuccess() throws Exception { initSCM(); // Create a secure SCM instance as om client will connect to it - scm = StorageContainerManager.createSCM(conf); + scm = TestUtils.getScmSimple(conf); LogCapturer logs = LogCapturer.captureLogs(OzoneManager.getLogger()); GenericTestUtils.setLogLevel(OzoneManager.getLogger(), INFO); @@ -407,7 +407,7 @@ public void testSecureOmInitializationSuccess() throws Exception { public void testAccessControlExceptionOnClient() throws Exception { initSCM(); // Create a secure SCM instance as om client will connect to it - scm = StorageContainerManager.createSCM(conf); + scm = TestUtils.getScmSimple(conf); LogCapturer logs = LogCapturer.captureLogs(OzoneManager.getLogger()); GenericTestUtils.setLogLevel(OzoneManager.getLogger(), INFO); setupOm(conf); @@ -632,7 +632,7 @@ public void testSecureOmReInit() throws Exception { initSCM(); try { - scm = HddsTestUtils.getScm(conf); + scm = TestUtils.getScmSimple(conf); scm.start(); conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, false); OMStorage omStore = new OMStorage(conf); @@ -678,7 +678,7 @@ public void testSecureOmInitSuccess() throws Exception { omLogs.clearOutput(); initSCM(); try { - scm = HddsTestUtils.getScm(conf); + scm = TestUtils.getScmSimple(conf); scm.start(); OMStorage omStore = new OMStorage(conf); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java index 1320b5b9cd89..597a317433cd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java @@ -26,6 +26,8 @@ import static org.apache.hadoop.hdds.HddsConfigKeys .HDDS_SCM_SAFEMODE_PIPELINE_CREATION; import static org.junit.Assert.fail; + +import org.apache.hadoop.hdds.scm.TestUtils; import org.junit.Ignore; import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; @@ -490,7 +492,7 @@ public void testSCMInitializationFailure() exception.expect(SCMException.class); exception.expectMessage( "SCM not initialized due to storage config failure"); - StorageContainerManager.createSCM(conf); + TestUtils.getScmSimple(conf); } @Test @@ -508,7 +510,7 @@ public void testScmInfo() throws Exception { scmStore.setScmId(scmId); // writes the version file properties scmStore.initialize(); - StorageContainerManager scm = StorageContainerManager.createSCM(conf); + StorageContainerManager scm = TestUtils.getScmSimple(conf); //Reads the SCM Info from SCM instance ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo(); Assert.assertEquals(clusterId, scmInfo.getClusterId()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index 7a1cb5b278dd..b122d7d546b5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes; +import org.apache.hadoop.hdds.scm.ha.MockSCMHAManager; import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl; import org.apache.hadoop.hdds.scm.net.NodeSchema; @@ -170,6 +171,7 @@ public static void setUp() throws Exception { SCMConfigurator configurator = new SCMConfigurator(); configurator.setScmNodeManager(nodeManager); configurator.setNetworkTopology(clusterMap); + configurator.setSCMHAManager(MockSCMHAManager.getInstance()); scm = TestUtils.getScm(conf, configurator); scm.start(); scm.exitSafeMode(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java index 646b91571260..2da50b7771be 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithData.java @@ -75,7 +75,7 @@ public void testAllOMNodesRunning() throws Exception { @Test public void testOneOMNodeDown() throws Exception { getCluster().stopOzoneManager(1); - Thread.sleep(NODE_FAILURE_TIMEOUT * 2); + Thread.sleep(NODE_FAILURE_TIMEOUT * 4); createVolumeTest(true); @@ -90,7 +90,7 @@ public void testOneOMNodeDown() throws Exception { public void testTwoOMNodesDown() throws Exception { getCluster().stopOzoneManager(1); getCluster().stopOzoneManager(2); - Thread.sleep(NODE_FAILURE_TIMEOUT * 2); + Thread.sleep(NODE_FAILURE_TIMEOUT * 4); createVolumeTest(false); @@ -308,7 +308,7 @@ public void testMultipartUploadWithOneOmNodeDown() throws Exception { // Stop one of the ozone manager, to see when the OM leader changes // multipart upload is happening successfully or not. getCluster().stopOzoneManager(leaderOMNodeId); - Thread.sleep(NODE_FAILURE_TIMEOUT * 2); + Thread.sleep(NODE_FAILURE_TIMEOUT * 4); createMultipartKeyAndReadKey(ozoneBucket, keyName, uploadID); @@ -340,7 +340,7 @@ public void testIncrementalWaitTimeWithSameNodeFailover() throws Exception { String leaderOMNodeId = omFailoverProxyProvider.getCurrentProxyOMNodeId(); getCluster().stopOzoneManager(leaderOMNodeId); - Thread.sleep(NODE_FAILURE_TIMEOUT * 2); + Thread.sleep(NODE_FAILURE_TIMEOUT * 4); createKeyTest(true); // failover should happen to new node long numTimesTriedToSameNode = omFailoverProxyProvider.getWaitTime() @@ -641,7 +641,7 @@ public void testListParts() throws Exception { // Stop leader OM, and then validate list parts. stopLeaderOM(); - Thread.sleep(NODE_FAILURE_TIMEOUT * 2); + Thread.sleep(NODE_FAILURE_TIMEOUT * 4); validateListParts(ozoneBucket, keyName, uploadID, partsMap);