From 7114407215d6304c1fac168a875739b1361ffcd9 Mon Sep 17 00:00:00 2001 From: cespejor Date: Sun, 26 Apr 2020 14:35:47 +0200 Subject: [PATCH 01/10] HDDS-3479. Use SCMMetadataStore high level abstraction instead of DBStoreBuilder and SCMDBDefinition directly HDDS-3479. Rename SCMMetadataStoreRDBImpl to SCMMetadataStoreImpl as it is not longer RocksDB specific. --- ...RDBImpl.java => SCMMetadataStoreImpl.java} | 6 +- .../scm/server/StorageContainerManager.java | 4 +- .../hdds/scm/block/TestBlockManager.java | 4 +- .../TestCloseContainerEventHandler.java | 19 ++- .../container/TestSCMContainerManager.java | 13 +- .../hdds/scm/node/TestContainerPlacement.java | 22 ++-- .../scm/pipeline/TestSCMPipelineManager.java | 36 +++--- .../TestHealthyPipelineSafeModeRule.java | 122 +++++++++--------- .../TestOneReplicaPipelineSafeModeRule.java | 10 +- .../scm/safemode/TestSCMSafeModeManager.java | 25 ++-- .../hadoop/ozone/genesis/GenesisUtil.java | 12 +- 11 files changed, 131 insertions(+), 142 deletions(-) rename hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/{SCMMetadataStoreRDBImpl.java => SCMMetadataStoreImpl.java} (96%) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java similarity index 96% rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java rename to hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java index 3823fd838071..4ab545776080 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java @@ -48,7 +48,7 @@ * A RocksDB based implementation of SCM Metadata Store. * */ -public class SCMMetadataStoreRDBImpl implements SCMMetadataStore { +public class SCMMetadataStoreImpl implements SCMMetadataStore { private Table deletedBlocksTable; @@ -61,7 +61,7 @@ public class SCMMetadataStoreRDBImpl implements SCMMetadataStore { private Table pipelineTable; private static final Logger LOG = - LoggerFactory.getLogger(SCMMetadataStoreRDBImpl.class); + LoggerFactory.getLogger(SCMMetadataStoreImpl.class); private DBStore store; private final OzoneConfiguration configuration; private final AtomicLong txID; @@ -72,7 +72,7 @@ public class SCMMetadataStoreRDBImpl implements SCMMetadataStore { * @param config - Ozone Configuration. * @throws IOException - on Failure. */ - public SCMMetadataStoreRDBImpl(OzoneConfiguration config) + public SCMMetadataStoreImpl(OzoneConfiguration config) throws IOException { this.configuration = config; start(this.configuration); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 8498a25a4b0b..98fb5d26537d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -67,7 +67,7 @@ import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; -import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreRDBImpl; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl; import org.apache.hadoop.hdds.scm.node.DeadNodeHandler; @@ -502,7 +502,7 @@ private void initalizeMetadataStore(OzoneConfiguration conf, if(configurator.getMetadataStore() != null) { scmMetadataStore = configurator.getMetadataStore(); } else { - scmMetadataStore = new SCMMetadataStoreRDBImpl(conf); + scmMetadataStore = new SCMMetadataStoreImpl(conf); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java index c9b5fde158b2..547838a554db 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; -import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreRDBImpl; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; @@ -107,7 +107,7 @@ public void setUp() throws Exception { nodeManager = new MockNodeManager(true, 10); eventQueue = new EventQueue(); - scmMetadataStore = new SCMMetadataStoreRDBImpl(conf); + scmMetadataStore = new SCMMetadataStoreImpl(conf); scmMetadataStore.start(conf); pipelineManager = new SCMPipelineManager(conf, nodeManager, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java index 09b41a5a47fb..4fde38ec5f16 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java @@ -28,13 +28,12 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.test.GenericTestUtils; @@ -61,7 +60,7 @@ public class TestCloseContainerEventHandler { private static long size; private static File testDir; private static EventQueue eventQueue; - private static DBStore dbStore; + private static SCMMetadataStore scmMetadataStore; @BeforeClass public static void setUp() throws Exception { @@ -75,11 +74,11 @@ public static void setUp() throws Exception { configuration.setInt(ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT, 16); nodeManager = new MockNodeManager(true, 10); eventQueue = new EventQueue(); - dbStore = - DBStoreBuilder.createDBStore(configuration, new SCMDBDefinition()); + scmMetadataStore = new SCMMetadataStoreImpl(configuration); + pipelineManager = new SCMPipelineManager(configuration, nodeManager, - SCMDBDefinition.PIPELINES.getTable(dbStore), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -87,7 +86,7 @@ public static void setUp() throws Exception { pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider); containerManager = new SCMContainerManager(configuration, - SCMDBDefinition.CONTAINERS.getTable(dbStore), dbStore, pipelineManager); + scmMetadataStore.getContainerTable(), scmMetadataStore.getStore(), pipelineManager); pipelineManager.triggerPipelineCreation(); eventQueue.addHandler(CLOSE_CONTAINER, new CloseContainerEventHandler(pipelineManager, containerManager)); @@ -105,8 +104,8 @@ public static void tearDown() throws Exception { if (pipelineManager != null) { pipelineManager.close(); } - if (dbStore != null) { - dbStore.close(); + if (scmMetadataStore.getStore() != null) { + scmMetadataStore.getStore().close(); } FileUtil.fullyDelete(testDir); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java index 1821e92bb691..fc0ec8f59d79 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java @@ -44,12 +44,11 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.test.GenericTestUtils; @@ -98,14 +97,14 @@ public static void setUp() throws Exception { throw new IOException("Unable to create test directory path"); } nodeManager = new MockNodeManager(true, 10); - DBStore dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition()); + SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(conf); pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(dbStore), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); containerManager = new SCMContainerManager(conf, - SCMDBDefinition.CONTAINERS.getTable(dbStore), - dbStore, + scmMetadataStore.getContainerTable(), + scmMetadataStore.getStore(), pipelineManager); xceiverClientManager = new XceiverClientManager(conf); replicationFactor = SCMTestUtils.getReplicationFactor(conf); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java index 6ce66a298108..a2180a82ea7c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java @@ -36,21 +36,19 @@ import org.apache.hadoop.hdds.scm.container.SCMContainerManager; import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity; import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +//import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.test.PathUtils; import org.apache.commons.io.IOUtils; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; -import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.CONTAINERS; -import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.PIPELINES; import org.junit.After; import static org.junit.Assert.assertEquals; import org.junit.Before; @@ -67,17 +65,17 @@ public class TestContainerPlacement { @Rule public ExpectedException thrown = ExpectedException.none(); - private DBStore dbStore; + //private DBStore dbStore; + private SCMMetadataStore scmMetadataStore; @Before public void createDbStore() throws IOException { - dbStore = - DBStoreBuilder.createDBStore(getConf(), new SCMDBDefinition()); + scmMetadataStore = new SCMMetadataStoreImpl(getConf()); } @After public void destroyDBStore() throws Exception { - dbStore.close(); + scmMetadataStore.getStore().close(); } /** * Returns a new copy of Configuration. @@ -120,9 +118,9 @@ SCMContainerManager createContainerManager(ConfigurationSource config, PipelineManager pipelineManager = new SCMPipelineManager(config, scmNodeManager, - PIPELINES.getTable(dbStore), eventQueue); - return new SCMContainerManager(config, CONTAINERS.getTable(dbStore), - dbStore, + scmMetadataStore.getPipelineTable(), eventQueue); + return new SCMContainerManager(config, scmMetadataStore.getContainerTable(), + scmMetadataStore.getStore(), pipelineManager); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java index 007f071c510b..1c843fe5c5ca 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java @@ -37,12 +37,11 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.exceptions.SCMException; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.test.GenericTestUtils; @@ -66,7 +65,7 @@ public class TestSCMPipelineManager { private static MockNodeManager nodeManager; private static File testDir; private static OzoneConfiguration conf; - private DBStore store; + private static SCMMetadataStore scmMetadataStore; @Before public void setUp() throws Exception { @@ -82,13 +81,12 @@ public void setUp() throws Exception { } nodeManager = new MockNodeManager(true, 20); - store = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition()); - + scmMetadataStore = new SCMMetadataStoreImpl(conf); } @After public void cleanup() throws Exception { - store.close(); + scmMetadataStore.getStore().close(); FileUtil.fullyDelete(testDir); } @@ -97,7 +95,7 @@ public void testPipelineReload() throws IOException { SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = @@ -119,7 +117,7 @@ public void testPipelineReload() throws IOException { // new pipeline manager should be able to load the pipelines from the db pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -151,7 +149,7 @@ public void testPipelineReload() throws IOException { public void testRemovePipeline() throws IOException { SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -171,7 +169,7 @@ public void testRemovePipeline() throws IOException { // new pipeline manager should not be able to load removed pipelines pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); try { pipelineManager.getPipeline(pipeline.getId()); fail("Pipeline should not have been retrieved"); @@ -188,7 +186,7 @@ public void testPipelineReport() throws IOException { EventQueue eventQueue = new EventQueue(); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -255,7 +253,7 @@ public void testPipelineCreationFailedMetric() throws Exception { 20); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManagerMock, - SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManagerMock, @@ -315,7 +313,7 @@ public void testPipelineCreationFailedMetric() throws Exception { public void testActivateDeactivatePipeline() throws IOException { final SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); final PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -364,7 +362,7 @@ public void testPipelineOpenOnlyWhenLeaderReported() throws Exception { EventQueue eventQueue = new EventQueue(); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -381,7 +379,7 @@ public void testPipelineOpenOnlyWhenLeaderReported() throws Exception { // new pipeline manager loads the pipelines from the db in ALLOCATED state pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), conf); @@ -427,7 +425,7 @@ public void testScrubPipeline() throws IOException { EventQueue eventQueue = new EventQueue(); final SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); final PipelineProvider ratisProvider = new MockRatisPipelineProvider( nodeManager, pipelineManager.getStateManager(), conf, eventQueue, @@ -471,7 +469,7 @@ public void testPipelineNotCreatedUntilSafeModePrecheck() EventQueue eventQueue = new EventQueue(); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); final PipelineProvider ratisProvider = new MockRatisPipelineProvider( nodeManager, pipelineManager.getStateManager(), conf, eventQueue, false); @@ -517,7 +515,7 @@ public void testSafeModeUpdatedOnSafemodeExit() EventQueue eventQueue = new EventQueue(); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); final PipelineProvider ratisProvider = new MockRatisPipelineProvider( nodeManager, pipelineManager.getStateManager(), conf, eventQueue, false); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java index 700479d40645..d6afe32e5e13 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java @@ -31,14 +31,13 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; @@ -53,25 +52,27 @@ public class TestHealthyPipelineSafeModeRule { @Test public void testHealthyPipelineSafeModeRuleWithNoPipelines() throws Exception { - DBStore store = null; + //DBStore store = null; + EventQueue eventQueue = new EventQueue(); + List containers = + new ArrayList<>(HddsTestUtils.getContainerInfo(1)); + String storageDir = GenericTestUtils.getTempPath( - TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID()); + TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID()); + OzoneConfiguration config = new OzoneConfiguration(); + MockNodeManager nodeManager = new MockNodeManager(true, 0); + config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); + // enable pipeline check + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); + //store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); + SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config); + try { - EventQueue eventQueue = new EventQueue(); - List containers = - new ArrayList<>(HddsTestUtils.getContainerInfo(1)); - - OzoneConfiguration config = new OzoneConfiguration(); - MockNodeManager nodeManager = new MockNodeManager(true, 0); - config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); - // enable pipeline check - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); - store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + nodeManager, scmMetadataStore.getPipelineTable(), eventQueue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), config); @@ -86,7 +87,7 @@ public void testHealthyPipelineSafeModeRuleWithNoPipelines() // This should be immediately satisfied, as no pipelines are there yet. Assert.assertTrue(healthyPipelineSafeModeRule.validate()); } finally { - store.close(); + scmMetadataStore.getStore().close(); FileUtil.fullyDelete(new File(storageDir)); } } @@ -95,27 +96,27 @@ public void testHealthyPipelineSafeModeRuleWithNoPipelines() public void testHealthyPipelineSafeModeRuleWithPipelines() throws Exception { String storageDir = GenericTestUtils.getTempPath( TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID()); - DBStore store = null; + + EventQueue eventQueue = new EventQueue(); + List containers = + new ArrayList<>(HddsTestUtils.getContainerInfo(1)); + + OzoneConfiguration config = new OzoneConfiguration(); + // In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are + // stale and last one is dead, and this repeats. So for a 12 node, 9 + // healthy, 2 stale and one dead. + MockNodeManager nodeManager = new MockNodeManager(true, 12); + config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); + // enable pipeline check + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); + + SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config); try { - EventQueue eventQueue = new EventQueue(); - List containers = - new ArrayList<>(HddsTestUtils.getContainerInfo(1)); - - OzoneConfiguration config = new OzoneConfiguration(); - // In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are - // stale and last one is dead, and this repeats. So for a 12 node, 9 - // healthy, 2 stale and one dead. - MockNodeManager nodeManager = new MockNodeManager(true, 12); - config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); - // enable pipeline check - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); - - store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + nodeManager, scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = @@ -158,7 +159,7 @@ public void testHealthyPipelineSafeModeRuleWithPipelines() throws Exception { GenericTestUtils.waitFor(() -> healthyPipelineSafeModeRule.validate(), 1000, 5000); } finally { - store.close(); + scmMetadataStore.getStore().close(); FileUtil.fullyDelete(new File(storageDir)); } } @@ -170,29 +171,28 @@ public void testHealthyPipelineSafeModeRuleWithMixedPipelines() String storageDir = GenericTestUtils.getTempPath( TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID()); - DBStore store = null; + EventQueue eventQueue = new EventQueue(); + List containers = + new ArrayList<>(HddsTestUtils.getContainerInfo(1)); + + OzoneConfiguration config = new OzoneConfiguration(); + + // In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are + // stale and last one is dead, and this repeats. So for a 12 node, 9 + // healthy, 2 stale and one dead. + MockNodeManager nodeManager = new MockNodeManager(true, 12); + config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); + // enable pipeline check + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); + + SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config); try { - EventQueue eventQueue = new EventQueue(); - List containers = - new ArrayList<>(HddsTestUtils.getContainerInfo(1)); - - OzoneConfiguration config = new OzoneConfiguration(); - - // In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are - // stale and last one is dead, and this repeats. So for a 12 node, 9 - // healthy, 2 stale and one dead. - MockNodeManager nodeManager = new MockNodeManager(true, 12); - config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); - // enable pipeline check - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); - - store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + nodeManager, scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = @@ -244,7 +244,7 @@ public void testHealthyPipelineSafeModeRuleWithMixedPipelines() 1000, 5000); } finally { - store.close(); + scmMetadataStore.getStore().close(); FileUtil.fullyDelete(new File(storageDir)); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java index c1f09faa3cb0..84ecda90aa6b 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java @@ -27,14 +27,13 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; @@ -70,12 +69,11 @@ private void setup(int nodes, int pipelineFactorThreeCount, eventQueue = new EventQueue(); - DBStore dbStore = - DBStoreBuilder.createDBStore(ozoneConfiguration, new SCMDBDefinition()); + SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(ozoneConfiguration); pipelineManager = new SCMPipelineManager(ozoneConfiguration, mockNodeManager, - SCMDBDefinition.PIPELINES.getTable(dbStore), + scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java index 9d2230492290..989c02f8d7b8 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java @@ -34,7 +34,8 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; @@ -43,8 +44,6 @@ import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; @@ -74,7 +73,7 @@ public class TestSCMSafeModeManager { @Rule public final TemporaryFolder tempDir = new TemporaryFolder(); - private DBStore dbStore; + private SCMMetadataStore scmMetadataStore; @Before public void setUp() { @@ -88,13 +87,13 @@ public void setUp() { public void initDbStore() throws IOException { config.set(HddsConfigKeys.OZONE_METADATA_DIRS, tempDir.newFolder().getAbsolutePath()); - dbStore = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); + scmMetadataStore = new SCMMetadataStoreImpl(config); } @After public void destroyDbStore() throws Exception { - if (dbStore != null) { - dbStore.close(); + if (scmMetadataStore.getStore() != null) { + scmMetadataStore.getStore().close(); } } @@ -242,7 +241,7 @@ public void testFailWithIncorrectValueForHealthyPipelinePercent() 0.9); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + mockNodeManager, scmMetadataStore.getPipelineTable(), queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForHealthyPipelinePercent"); @@ -260,7 +259,7 @@ public void testFailWithIncorrectValueForOneReplicaPipelinePercent() 200); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + mockNodeManager, scmMetadataStore.getPipelineTable(), queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForOneReplicaPipelinePercent"); @@ -277,7 +276,7 @@ public void testFailWithIncorrectValueForSafeModePercent() throws Exception { conf.setDouble(HddsConfigKeys.HDDS_SCM_SAFEMODE_THRESHOLD_PCT, -1.0); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + mockNodeManager, scmMetadataStore.getPipelineTable(), queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForSafeModePercent"); @@ -301,7 +300,7 @@ public void testSafeModeExitRuleWithPipelineAvailabilityCheck( MockNodeManager mockNodeManager = new MockNodeManager(true, nodeCount); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + mockNodeManager, scmMetadataStore.getPipelineTable(), queue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(mockNodeManager, pipelineManager.getStateManager(), config, true); @@ -518,7 +517,7 @@ public void testSafeModePipelineExitRule() throws Exception { HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + nodeManager, scmMetadataStore.getPipelineTable(), queue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -572,7 +571,7 @@ public void testPipelinesNotCreatedUntilPreCheckPasses() HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + nodeManager, scmMetadataStore.getPipelineTable(), queue); PipelineProvider mockRatisProvider = diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java index 797c8051bc78..464d3fdc86f6 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java @@ -31,7 +31,8 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.server.SCMConfigurator; @@ -39,8 +40,6 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.MetadataStore; import org.apache.hadoop.hdds.utils.MetadataStoreBuilder; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.common.Storage; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -150,10 +149,10 @@ static void configureSCM(OzoneConfiguration conf, int numHandlers) { static void addPipelines(HddsProtos.ReplicationFactor factor, int numPipelines, ConfigurationSource conf) throws Exception { - DBStore dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition()); + SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl((OzoneConfiguration)conf); Table pipelineTable = - SCMDBDefinition.PIPELINES.getTable(dbStore); + scmMetadataStore.getPipelineTable(); List nodes = new ArrayList<>(); for (int i = 0; i < factor.getNumber(); i++) { nodes @@ -171,8 +170,7 @@ static void addPipelines(HddsProtos.ReplicationFactor factor, pipelineTable.put(pipeline.getId(), pipeline); } - - dbStore.close(); + scmMetadataStore.getStore().close(); } static OzoneManager getOm(OzoneConfiguration conf) From fdb7babccdcccf1ac6800d662a2297d5dd84cf49 Mon Sep 17 00:00:00 2001 From: cespejor Date: Sun, 31 May 2020 15:20:31 +0200 Subject: [PATCH 02/10] HDDS-3479. Fixed style issues. --- .../hdds/scm/container/TestCloseContainerEventHandler.java | 4 +++- .../hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java | 3 ++- .../hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java | 3 ++- .../java/org/apache/hadoop/ozone/genesis/GenesisUtil.java | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java index 4fde38ec5f16..c53d29323bc7 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java @@ -89,7 +89,9 @@ public static void setUp() throws Exception { scmMetadataStore.getContainerTable(), scmMetadataStore.getStore(), pipelineManager); pipelineManager.triggerPipelineCreation(); eventQueue.addHandler(CLOSE_CONTAINER, - new CloseContainerEventHandler(pipelineManager, containerManager)); + new CloseContainerEventHandler( + pipelineManager, + containerManager)); eventQueue.addHandler(DATANODE_COMMAND, nodeManager); // Move all pipelines created by background from ALLOCATED to OPEN state Thread.sleep(2000); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java index d6afe32e5e13..5739b8e354a1 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java @@ -58,7 +58,8 @@ public void testHealthyPipelineSafeModeRuleWithNoPipelines() new ArrayList<>(HddsTestUtils.getContainerInfo(1)); String storageDir = GenericTestUtils.getTempPath( - TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID()); + TestHealthyPipelineSafeModeRule.class.getName() + + UUID.randomUUID()); OzoneConfiguration config = new OzoneConfiguration(); MockNodeManager nodeManager = new MockNodeManager(true, 0); config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java index 84ecda90aa6b..4e1cf6fcb2d3 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java @@ -69,7 +69,8 @@ private void setup(int nodes, int pipelineFactorThreeCount, eventQueue = new EventQueue(); - SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(ozoneConfiguration); + SCMMetadataStore scmMetadataStore = + new SCMMetadataStoreImpl(ozoneConfiguration); pipelineManager = new SCMPipelineManager(ozoneConfiguration, mockNodeManager, diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java index 464d3fdc86f6..156b6a26941f 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java @@ -149,7 +149,8 @@ static void configureSCM(OzoneConfiguration conf, int numHandlers) { static void addPipelines(HddsProtos.ReplicationFactor factor, int numPipelines, ConfigurationSource conf) throws Exception { - SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl((OzoneConfiguration)conf); + SCMMetadataStore scmMetadataStore = + new SCMMetadataStoreImpl((OzoneConfiguration)conf); Table pipelineTable = scmMetadataStore.getPipelineTable(); From 795a1efc826e3fd0f1761085f0d7830945db8815 Mon Sep 17 00:00:00 2001 From: cespejor Date: Sun, 31 May 2020 15:26:12 +0200 Subject: [PATCH 03/10] HDDS-3479. Fixed style issues. --- .../hdds/scm/container/TestCloseContainerEventHandler.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java index c53d29323bc7..b080ea1a8204 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java @@ -85,8 +85,11 @@ public static void setUp() throws Exception { pipelineManager.getStateManager(), configuration, eventQueue); pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider); - containerManager = new SCMContainerManager(configuration, - scmMetadataStore.getContainerTable(), scmMetadataStore.getStore(), pipelineManager); + containerManager = new SCMContainerManager( + configuration, + scmMetadataStore.getContainerTable(), + scmMetadataStore.getStore(), + pipelineManager); pipelineManager.triggerPipelineCreation(); eventQueue.addHandler(CLOSE_CONTAINER, new CloseContainerEventHandler( From 0d3090945a17a334f1427c82fbbfca98f6075411 Mon Sep 17 00:00:00 2001 From: cespejor Date: Tue, 2 Jun 2020 15:47:44 +0200 Subject: [PATCH 04/10] retrigger ci From ac8a7f1bf3404ae57b2966cd38106ade0a3ddf78 Mon Sep 17 00:00:00 2001 From: cespejor Date: Thu, 4 Jun 2020 17:28:06 +0200 Subject: [PATCH 05/10] retrigger ci From 8c81fb708f4613f7066eb1914d91e8ed1f18ebf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Wed, 24 Jun 2020 12:55:04 +0200 Subject: [PATCH 06/10] remove the commented lines --- .../org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java | 1 - .../hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java | 1 - 2 files changed, 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java index a2180a82ea7c..9f16f4cec4aa 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hdds.scm.container.SCMContainerManager; import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity; import org.apache.hadoop.hdds.scm.events.SCMEvents; -//import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java index 5739b8e354a1..916eebc0ffa2 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java @@ -52,7 +52,6 @@ public class TestHealthyPipelineSafeModeRule { @Test public void testHealthyPipelineSafeModeRuleWithNoPipelines() throws Exception { - //DBStore store = null; EventQueue eventQueue = new EventQueue(); List containers = new ArrayList<>(HddsTestUtils.getContainerInfo(1)); From 0e3ef4b9b23d20ca11e1755425250e72e8e143fe Mon Sep 17 00:00:00 2001 From: cespejor Date: Wed, 24 Jun 2020 13:22:09 +0200 Subject: [PATCH 07/10] HDDS-3479. modified according to review comments. --- .../org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java | 1 - .../hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java | 1 - 2 files changed, 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java index 9f16f4cec4aa..797709ecc5f2 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java @@ -64,7 +64,6 @@ public class TestContainerPlacement { @Rule public ExpectedException thrown = ExpectedException.none(); - //private DBStore dbStore; private SCMMetadataStore scmMetadataStore; @Before diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java index 916eebc0ffa2..7a40e3e47c9e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java @@ -67,7 +67,6 @@ public void testHealthyPipelineSafeModeRuleWithNoPipelines() HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); config.setBoolean( HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); - //store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config); try { From 5c22fb6e3d90a0f584db04f91772106e33c59b4c Mon Sep 17 00:00:00 2001 From: cespejor Date: Wed, 24 Jun 2020 13:30:33 +0200 Subject: [PATCH 08/10] HDDS-3479. rebased master in fork from upstream --- .../dist/src/main/compose/ozone-csi/result/docker-ozone-csi.log | 1 + .../main/compose/ozone-mr/hadoop31/result/docker-hadoop31.log | 0 .../main/compose/ozone-mr/hadoop32/result/docker-hadoop32.log | 1 + .../main/compose/ozone-topology/result/docker-ozone-topology.log | 1 + hadoop-ozone/dist/src/main/compose/ozone/result/docker-ozone.log | 1 + .../compose/ozones3-haproxy/result/docker-ozones3-haproxy.log | 1 + .../main/compose/ozonesecure-mr/result/docker-ozonesecure-mr.log | 1 + .../ozonesecure-om-ha/result/docker-ozonesecure-om-ha.log | 1 + .../src/main/compose/ozonesecure/result/docker-ozonesecure.log | 1 + hadoop-ozone/dist/src/main/compose/result/docker-hadoop31.log | 1 + hadoop-ozone/dist/src/main/compose/result/docker-hadoop32.log | 1 + hadoop-ozone/dist/src/main/compose/result/docker-ozone-csi.log | 1 + .../dist/src/main/compose/result/docker-ozone-topology.log | 1 + hadoop-ozone/dist/src/main/compose/result/docker-ozone.log | 1 + .../dist/src/main/compose/result/docker-ozones3-haproxy.log | 1 + .../dist/src/main/compose/result/docker-ozonesecure-mr.log | 1 + .../dist/src/main/compose/result/docker-ozonesecure-om-ha.log | 1 + hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure.log | 1 + 18 files changed, 17 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-csi/result/docker-ozone-csi.log create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/result/docker-hadoop31.log create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/result/docker-hadoop32.log create mode 100644 hadoop-ozone/dist/src/main/compose/ozone-topology/result/docker-ozone-topology.log create mode 100644 hadoop-ozone/dist/src/main/compose/ozone/result/docker-ozone.log create mode 100644 hadoop-ozone/dist/src/main/compose/ozones3-haproxy/result/docker-ozones3-haproxy.log create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-mr/result/docker-ozonesecure-mr.log create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/result/docker-ozonesecure-om-ha.log create mode 100644 hadoop-ozone/dist/src/main/compose/ozonesecure/result/docker-ozonesecure.log create mode 100644 hadoop-ozone/dist/src/main/compose/result/docker-hadoop31.log create mode 100644 hadoop-ozone/dist/src/main/compose/result/docker-hadoop32.log create mode 100644 hadoop-ozone/dist/src/main/compose/result/docker-ozone-csi.log create mode 100644 hadoop-ozone/dist/src/main/compose/result/docker-ozone-topology.log create mode 100644 hadoop-ozone/dist/src/main/compose/result/docker-ozone.log create mode 100644 hadoop-ozone/dist/src/main/compose/result/docker-ozones3-haproxy.log create mode 100644 hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure-mr.log create mode 100644 hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure-om-ha.log create mode 100644 hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure.log diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/result/docker-ozone-csi.log b/hadoop-ozone/dist/src/main/compose/ozone-csi/result/docker-ozone-csi.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-csi/result/docker-ozone-csi.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/result/docker-hadoop31.log b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/result/docker-hadoop31.log new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/result/docker-hadoop32.log b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/result/docker-hadoop32.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/result/docker-hadoop32.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/result/docker-ozone-topology.log b/hadoop-ozone/dist/src/main/compose/ozone-topology/result/docker-ozone-topology.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/result/docker-ozone-topology.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/ozone/result/docker-ozone.log b/hadoop-ozone/dist/src/main/compose/ozone/result/docker-ozone.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/result/docker-ozone.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/result/docker-ozones3-haproxy.log b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/result/docker-ozones3-haproxy.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/result/docker-ozones3-haproxy.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/result/docker-ozonesecure-mr.log b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/result/docker-ozonesecure-mr.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/result/docker-ozonesecure-mr.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/result/docker-ozonesecure-om-ha.log b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/result/docker-ozonesecure-om-ha.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/result/docker-ozonesecure-om-ha.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/result/docker-ozonesecure.log b/hadoop-ozone/dist/src/main/compose/ozonesecure/result/docker-ozonesecure.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/result/docker-ozonesecure.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/result/docker-hadoop31.log b/hadoop-ozone/dist/src/main/compose/result/docker-hadoop31.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/result/docker-hadoop31.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/result/docker-hadoop32.log b/hadoop-ozone/dist/src/main/compose/result/docker-hadoop32.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/result/docker-hadoop32.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/result/docker-ozone-csi.log b/hadoop-ozone/dist/src/main/compose/result/docker-ozone-csi.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/result/docker-ozone-csi.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/result/docker-ozone-topology.log b/hadoop-ozone/dist/src/main/compose/result/docker-ozone-topology.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/result/docker-ozone-topology.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/result/docker-ozone.log b/hadoop-ozone/dist/src/main/compose/result/docker-ozone.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/result/docker-ozone.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/result/docker-ozones3-haproxy.log b/hadoop-ozone/dist/src/main/compose/result/docker-ozones3-haproxy.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/result/docker-ozones3-haproxy.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure-mr.log b/hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure-mr.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure-mr.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure-om-ha.log b/hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure-om-ha.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure-om-ha.log @@ -0,0 +1 @@ +Attaching to diff --git a/hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure.log b/hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure.log new file mode 100644 index 000000000000..3141e6e82154 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/result/docker-ozonesecure.log @@ -0,0 +1 @@ +Attaching to From 127a0837c3dd20a877b0c3b3ed8363b07186da87 Mon Sep 17 00:00:00 2001 From: cespejor Date: Wed, 24 Jun 2020 16:59:42 +0200 Subject: [PATCH 09/10] retrigger ci From 41a8ecbe24baffccab2cb0d58c4f00a5990f6bb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Fri, 26 Jun 2020 00:34:09 +0200 Subject: [PATCH 10/10] retrigger build with empty commit