diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java similarity index 96% rename from hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java rename to hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java index 3823fd83807..4ab54577608 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreRDBImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMMetadataStoreImpl.java @@ -48,7 +48,7 @@ * A RocksDB based implementation of SCM Metadata Store. * */ -public class SCMMetadataStoreRDBImpl implements SCMMetadataStore { +public class SCMMetadataStoreImpl implements SCMMetadataStore { private Table deletedBlocksTable; @@ -61,7 +61,7 @@ public class SCMMetadataStoreRDBImpl implements SCMMetadataStore { private Table pipelineTable; private static final Logger LOG = - LoggerFactory.getLogger(SCMMetadataStoreRDBImpl.class); + LoggerFactory.getLogger(SCMMetadataStoreImpl.class); private DBStore store; private final OzoneConfiguration configuration; private final AtomicLong txID; @@ -72,7 +72,7 @@ public class SCMMetadataStoreRDBImpl implements SCMMetadataStore { * @param config - Ozone Configuration. * @throws IOException - on Failure. */ - public SCMMetadataStoreRDBImpl(OzoneConfiguration config) + public SCMMetadataStoreImpl(OzoneConfiguration config) throws IOException { this.configuration = config; start(this.configuration); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index c73b19fd3bf..73113d1f696 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -68,7 +68,7 @@ import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; -import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreRDBImpl; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl; import org.apache.hadoop.hdds.scm.node.DeadNodeHandler; @@ -503,7 +503,7 @@ private void initalizeMetadataStore(OzoneConfiguration conf, if(configurator.getMetadataStore() != null) { scmMetadataStore = configurator.getMetadataStore(); } else { - scmMetadataStore = new SCMMetadataStoreRDBImpl(conf); + scmMetadataStore = new SCMMetadataStoreImpl(conf); } } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java index c2ad2649ffb..c89065c15a1 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; -import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreRDBImpl; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; @@ -109,7 +109,7 @@ public void setUp() throws Exception { nodeManager = new MockNodeManager(true, 10); eventQueue = new EventQueue(); - scmMetadataStore = new SCMMetadataStoreRDBImpl(conf); + scmMetadataStore = new SCMMetadataStoreImpl(conf); scmMetadataStore.start(conf); pipelineManager = new SCMPipelineManager(conf, nodeManager, diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java index 09b41a5a47f..b080ea1a820 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java @@ -28,13 +28,12 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.test.GenericTestUtils; @@ -61,7 +60,7 @@ public class TestCloseContainerEventHandler { private static long size; private static File testDir; private static EventQueue eventQueue; - private static DBStore dbStore; + private static SCMMetadataStore scmMetadataStore; @BeforeClass public static void setUp() throws Exception { @@ -75,22 +74,27 @@ public static void setUp() throws Exception { configuration.setInt(ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT, 16); nodeManager = new MockNodeManager(true, 10); eventQueue = new EventQueue(); - dbStore = - DBStoreBuilder.createDBStore(configuration, new SCMDBDefinition()); + scmMetadataStore = new SCMMetadataStoreImpl(configuration); + pipelineManager = new SCMPipelineManager(configuration, nodeManager, - SCMDBDefinition.PIPELINES.getTable(dbStore), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), configuration, eventQueue); pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS, mockRatisProvider); - containerManager = new SCMContainerManager(configuration, - SCMDBDefinition.CONTAINERS.getTable(dbStore), dbStore, pipelineManager); + containerManager = new SCMContainerManager( + configuration, + scmMetadataStore.getContainerTable(), + scmMetadataStore.getStore(), + pipelineManager); pipelineManager.triggerPipelineCreation(); eventQueue.addHandler(CLOSE_CONTAINER, - new CloseContainerEventHandler(pipelineManager, containerManager)); + new CloseContainerEventHandler( + pipelineManager, + containerManager)); eventQueue.addHandler(DATANODE_COMMAND, nodeManager); // Move all pipelines created by background from ALLOCATED to OPEN state Thread.sleep(2000); @@ -105,8 +109,8 @@ public static void tearDown() throws Exception { if (pipelineManager != null) { pipelineManager.close(); } - if (dbStore != null) { - dbStore.close(); + if (scmMetadataStore.getStore() != null) { + scmMetadataStore.getStore().close(); } FileUtil.fullyDelete(testDir); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java index 82b51c77a87..12c62a95699 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java @@ -42,12 +42,11 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.test.GenericTestUtils; @@ -92,14 +91,14 @@ public static void setUp() throws Exception { throw new IOException("Unable to create test directory path"); } nodeManager = new MockNodeManager(true, 10); - DBStore dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition()); + SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(conf); pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(dbStore), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); containerManager = new SCMContainerManager(conf, - SCMDBDefinition.CONTAINERS.getTable(dbStore), - dbStore, + scmMetadataStore.getContainerTable(), + scmMetadataStore.getStore(), pipelineManager); xceiverClientManager = new XceiverClientManager(conf); replicationFactor = SCMTestUtils.getReplicationFactor(conf); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java index 6ce66a29810..797709ecc5f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java @@ -36,21 +36,18 @@ import org.apache.hadoop.hdds.scm.container.SCMContainerManager; import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity; import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.test.PathUtils; import org.apache.commons.io.IOUtils; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; -import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.CONTAINERS; -import static org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.PIPELINES; import org.junit.After; import static org.junit.Assert.assertEquals; import org.junit.Before; @@ -67,17 +64,16 @@ public class TestContainerPlacement { @Rule public ExpectedException thrown = ExpectedException.none(); - private DBStore dbStore; + private SCMMetadataStore scmMetadataStore; @Before public void createDbStore() throws IOException { - dbStore = - DBStoreBuilder.createDBStore(getConf(), new SCMDBDefinition()); + scmMetadataStore = new SCMMetadataStoreImpl(getConf()); } @After public void destroyDBStore() throws Exception { - dbStore.close(); + scmMetadataStore.getStore().close(); } /** * Returns a new copy of Configuration. @@ -120,9 +116,9 @@ SCMContainerManager createContainerManager(ConfigurationSource config, PipelineManager pipelineManager = new SCMPipelineManager(config, scmNodeManager, - PIPELINES.getTable(dbStore), eventQueue); - return new SCMContainerManager(config, CONTAINERS.getTable(dbStore), - dbStore, + scmMetadataStore.getPipelineTable(), eventQueue); + return new SCMContainerManager(config, scmMetadataStore.getContainerTable(), + scmMetadataStore.getStore(), pipelineManager); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java index 1b273feaf85..7c2f17e8584 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java @@ -37,12 +37,11 @@ import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.exceptions.SCMException; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.test.GenericTestUtils; @@ -66,7 +65,7 @@ public class TestSCMPipelineManager { private static MockNodeManager nodeManager; private static File testDir; private static OzoneConfiguration conf; - private DBStore store; + private static SCMMetadataStore scmMetadataStore; @Before public void setUp() throws Exception { @@ -82,13 +81,12 @@ public void setUp() throws Exception { } nodeManager = new MockNodeManager(true, 20); - store = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition()); - + scmMetadataStore = new SCMMetadataStoreImpl(conf); } @After public void cleanup() throws Exception { - store.close(); + scmMetadataStore.getStore().close(); FileUtil.fullyDelete(testDir); } @@ -97,7 +95,7 @@ public void testPipelineReload() throws IOException { SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = @@ -119,7 +117,7 @@ public void testPipelineReload() throws IOException { // new pipeline manager should be able to load the pipelines from the db pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -151,7 +149,7 @@ public void testPipelineReload() throws IOException { public void testRemovePipeline() throws IOException { SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -171,7 +169,7 @@ public void testRemovePipeline() throws IOException { // new pipeline manager should not be able to load removed pipelines pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); try { pipelineManager.getPipeline(pipeline.getId()); fail("Pipeline should not have been retrieved"); @@ -188,7 +186,7 @@ public void testPipelineReport() throws IOException { EventQueue eventQueue = new EventQueue(); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -255,7 +253,7 @@ public void testPipelineCreationFailedMetric() throws Exception { 20); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManagerMock, - SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManagerMock, @@ -315,7 +313,7 @@ public void testPipelineCreationFailedMetric() throws Exception { public void testActivateDeactivatePipeline() throws IOException { final SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), new EventQueue()); + scmMetadataStore.getPipelineTable(), new EventQueue()); pipelineManager.allowPipelineCreation(); final PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -364,7 +362,7 @@ public void testPipelineOpenOnlyWhenLeaderReported() throws Exception { EventQueue eventQueue = new EventQueue(); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -381,7 +379,7 @@ public void testPipelineOpenOnlyWhenLeaderReported() throws Exception { // new pipeline manager loads the pipelines from the db in ALLOCATED state pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), conf); @@ -427,7 +425,7 @@ public void testScrubPipeline() throws IOException { EventQueue eventQueue = new EventQueue(); final SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); final PipelineProvider ratisProvider = new MockRatisPipelineProvider( nodeManager, pipelineManager.getStateManager(), conf, eventQueue, @@ -471,7 +469,7 @@ public void testPipelineNotCreatedUntilSafeModePrecheck() EventQueue eventQueue = new EventQueue(); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); final PipelineProvider ratisProvider = new MockRatisPipelineProvider( nodeManager, pipelineManager.getStateManager(), conf, eventQueue, false); @@ -517,7 +515,7 @@ public void testSafeModeUpdatedOnSafemodeExit() EventQueue eventQueue = new EventQueue(); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, nodeManager, - SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + scmMetadataStore.getPipelineTable(), eventQueue); final PipelineProvider ratisProvider = new MockRatisPipelineProvider( nodeManager, pipelineManager.getStateManager(), conf, eventQueue, false); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java index 700479d4064..7a40e3e47c9 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java @@ -31,14 +31,13 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; @@ -53,25 +52,26 @@ public class TestHealthyPipelineSafeModeRule { @Test public void testHealthyPipelineSafeModeRuleWithNoPipelines() throws Exception { - DBStore store = null; + EventQueue eventQueue = new EventQueue(); + List containers = + new ArrayList<>(HddsTestUtils.getContainerInfo(1)); + String storageDir = GenericTestUtils.getTempPath( - TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID()); + TestHealthyPipelineSafeModeRule.class.getName() + + UUID.randomUUID()); + OzoneConfiguration config = new OzoneConfiguration(); + MockNodeManager nodeManager = new MockNodeManager(true, 0); + config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); + // enable pipeline check + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); + SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config); + try { - EventQueue eventQueue = new EventQueue(); - List containers = - new ArrayList<>(HddsTestUtils.getContainerInfo(1)); - - OzoneConfiguration config = new OzoneConfiguration(); - MockNodeManager nodeManager = new MockNodeManager(true, 0); - config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); - // enable pipeline check - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); - store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + nodeManager, scmMetadataStore.getPipelineTable(), eventQueue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, pipelineManager.getStateManager(), config); @@ -86,7 +86,7 @@ public void testHealthyPipelineSafeModeRuleWithNoPipelines() // This should be immediately satisfied, as no pipelines are there yet. Assert.assertTrue(healthyPipelineSafeModeRule.validate()); } finally { - store.close(); + scmMetadataStore.getStore().close(); FileUtil.fullyDelete(new File(storageDir)); } } @@ -95,27 +95,27 @@ public void testHealthyPipelineSafeModeRuleWithNoPipelines() public void testHealthyPipelineSafeModeRuleWithPipelines() throws Exception { String storageDir = GenericTestUtils.getTempPath( TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID()); - DBStore store = null; + + EventQueue eventQueue = new EventQueue(); + List containers = + new ArrayList<>(HddsTestUtils.getContainerInfo(1)); + + OzoneConfiguration config = new OzoneConfiguration(); + // In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are + // stale and last one is dead, and this repeats. So for a 12 node, 9 + // healthy, 2 stale and one dead. + MockNodeManager nodeManager = new MockNodeManager(true, 12); + config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); + // enable pipeline check + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); + + SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config); try { - EventQueue eventQueue = new EventQueue(); - List containers = - new ArrayList<>(HddsTestUtils.getContainerInfo(1)); - - OzoneConfiguration config = new OzoneConfiguration(); - // In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are - // stale and last one is dead, and this repeats. So for a 12 node, 9 - // healthy, 2 stale and one dead. - MockNodeManager nodeManager = new MockNodeManager(true, 12); - config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); - // enable pipeline check - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); - - store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + nodeManager, scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = @@ -158,7 +158,7 @@ public void testHealthyPipelineSafeModeRuleWithPipelines() throws Exception { GenericTestUtils.waitFor(() -> healthyPipelineSafeModeRule.validate(), 1000, 5000); } finally { - store.close(); + scmMetadataStore.getStore().close(); FileUtil.fullyDelete(new File(storageDir)); } } @@ -170,29 +170,28 @@ public void testHealthyPipelineSafeModeRuleWithMixedPipelines() String storageDir = GenericTestUtils.getTempPath( TestHealthyPipelineSafeModeRule.class.getName() + UUID.randomUUID()); - DBStore store = null; + EventQueue eventQueue = new EventQueue(); + List containers = + new ArrayList<>(HddsTestUtils.getContainerInfo(1)); + + OzoneConfiguration config = new OzoneConfiguration(); + + // In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are + // stale and last one is dead, and this repeats. So for a 12 node, 9 + // healthy, 2 stale and one dead. + MockNodeManager nodeManager = new MockNodeManager(true, 12); + config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); + // enable pipeline check + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); + config.setBoolean( + HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); + + SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config); try { - EventQueue eventQueue = new EventQueue(); - List containers = - new ArrayList<>(HddsTestUtils.getContainerInfo(1)); - - OzoneConfiguration config = new OzoneConfiguration(); - - // In Mock Node Manager, first 8 nodes are healthy, next 2 nodes are - // stale and last one is dead, and this repeats. So for a 12 node, 9 - // healthy, 2 stale and one dead. - MockNodeManager nodeManager = new MockNodeManager(true, 12); - config.set(HddsConfigKeys.OZONE_METADATA_DIRS, storageDir); - // enable pipeline check - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); - config.setBoolean( - HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false); - - store = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, SCMDBDefinition.PIPELINES.getTable(store), eventQueue); + nodeManager, scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); PipelineProvider mockRatisProvider = @@ -244,7 +243,7 @@ public void testHealthyPipelineSafeModeRuleWithMixedPipelines() 1000, 5000); } finally { - store.close(); + scmMetadataStore.getStore().close(); FileUtil.fullyDelete(new File(storageDir)); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java index c1f09faa3cb..4e1cf6fcb2d 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java @@ -27,14 +27,13 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; @@ -70,12 +69,12 @@ private void setup(int nodes, int pipelineFactorThreeCount, eventQueue = new EventQueue(); - DBStore dbStore = - DBStoreBuilder.createDBStore(ozoneConfiguration, new SCMDBDefinition()); + SCMMetadataStore scmMetadataStore = + new SCMMetadataStoreImpl(ozoneConfiguration); pipelineManager = new SCMPipelineManager(ozoneConfiguration, mockNodeManager, - SCMDBDefinition.PIPELINES.getTable(dbStore), + scmMetadataStore.getPipelineTable(), eventQueue); pipelineManager.allowPipelineCreation(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java index b86d35473ab..418d945f638 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java @@ -35,7 +35,8 @@ import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.container.MockNodeManager; import org.apache.hadoop.hdds.scm.events.SCMEvents; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.MockRatisPipelineProvider; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; @@ -45,8 +46,6 @@ import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; @@ -76,7 +75,7 @@ public class TestSCMSafeModeManager { @Rule public final TemporaryFolder tempDir = new TemporaryFolder(); - private DBStore dbStore; + private SCMMetadataStore scmMetadataStore; @Before public void setUp() { @@ -90,13 +89,13 @@ public void setUp() { public void initDbStore() throws IOException { config.set(HddsConfigKeys.OZONE_METADATA_DIRS, tempDir.newFolder().getAbsolutePath()); - dbStore = DBStoreBuilder.createDBStore(config, new SCMDBDefinition()); + scmMetadataStore = new SCMMetadataStoreImpl(config); } @After public void destroyDbStore() throws Exception { - if (dbStore != null) { - dbStore.close(); + if (scmMetadataStore.getStore() != null) { + scmMetadataStore.getStore().close(); } } @@ -297,7 +296,7 @@ public void testFailWithIncorrectValueForHealthyPipelinePercent() 0.9); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + mockNodeManager, scmMetadataStore.getPipelineTable(), queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForHealthyPipelinePercent"); @@ -315,7 +314,7 @@ public void testFailWithIncorrectValueForOneReplicaPipelinePercent() 200); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + mockNodeManager, scmMetadataStore.getPipelineTable(), queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForOneReplicaPipelinePercent"); @@ -332,7 +331,7 @@ public void testFailWithIncorrectValueForSafeModePercent() throws Exception { conf.setDouble(HddsConfigKeys.HDDS_SCM_SAFEMODE_THRESHOLD_PCT, -1.0); MockNodeManager mockNodeManager = new MockNodeManager(true, 10); PipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + mockNodeManager, scmMetadataStore.getPipelineTable(), queue); scmSafeModeManager = new SCMSafeModeManager( conf, containers, pipelineManager, queue); fail("testFailWithIncorrectValueForSafeModePercent"); @@ -356,7 +355,7 @@ public void testSafeModeExitRuleWithPipelineAvailabilityCheck( MockNodeManager mockNodeManager = new MockNodeManager(true, nodeCount); SCMPipelineManager pipelineManager = new SCMPipelineManager(conf, - mockNodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + mockNodeManager, scmMetadataStore.getPipelineTable(), queue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(mockNodeManager, pipelineManager.getStateManager(), config, true); @@ -573,7 +572,7 @@ public void testSafeModePipelineExitRule() throws Exception { HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + nodeManager, scmMetadataStore.getPipelineTable(), queue); PipelineProvider mockRatisProvider = new MockRatisPipelineProvider(nodeManager, @@ -627,7 +626,7 @@ public void testPipelinesNotCreatedUntilPreCheckPasses() HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true); SCMPipelineManager pipelineManager = new SCMPipelineManager(config, - nodeManager, SCMDBDefinition.PIPELINES.getTable(dbStore), queue); + nodeManager, scmMetadataStore.getPipelineTable(), queue); PipelineProvider mockRatisProvider = diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java index 797c8051bc7..156b6a26941 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java @@ -31,7 +31,8 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore; +import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.server.SCMConfigurator; @@ -39,8 +40,6 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.MetadataStore; import org.apache.hadoop.hdds.utils.MetadataStoreBuilder; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.common.Storage; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -150,10 +149,11 @@ static void configureSCM(OzoneConfiguration conf, int numHandlers) { static void addPipelines(HddsProtos.ReplicationFactor factor, int numPipelines, ConfigurationSource conf) throws Exception { - DBStore dbStore = DBStoreBuilder.createDBStore(conf, new SCMDBDefinition()); + SCMMetadataStore scmMetadataStore = + new SCMMetadataStoreImpl((OzoneConfiguration)conf); Table pipelineTable = - SCMDBDefinition.PIPELINES.getTable(dbStore); + scmMetadataStore.getPipelineTable(); List nodes = new ArrayList<>(); for (int i = 0; i < factor.getNumber(); i++) { nodes @@ -171,8 +171,7 @@ static void addPipelines(HddsProtos.ReplicationFactor factor, pipelineTable.put(pipeline.getId(), pipeline); } - - dbStore.close(); + scmMetadataStore.getStore().close(); } static OzoneManager getOm(OzoneConfiguration conf)