diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java index 555055a630d0..f8ef8e01c15d 100644 --- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java +++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java @@ -72,7 +72,7 @@ public MiniOzoneChaosCluster(OzoneConfiguration conf, List hddsDatanodes, String clusterPath, Set> clazzes) { super(conf, new SCMConfigurator(), omService, scmService, hddsDatanodes, - clusterPath, null); + clusterPath, Collections.emptyList()); this.numDatanodes = getHddsDatanodes().size(); this.numOzoneManagers = omService.getServices().size(); this.numStorageContainerManagers = scmService.getServices().size(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java new file mode 100644 index 000000000000..99f0faa35fde --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon; + +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR; +import static org.apache.ozone.test.GenericTestUtils.PortAllocator.localhostWithFreePort; + +import java.io.File; +import java.util.Objects; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.ozone.recon.schema.ReconSqlDbConfig; +import org.apache.ratis.util.Preconditions; + +/** Recon for {@link MiniOzoneCluster}. */ +public class ReconService implements MiniOzoneCluster.Service { + + private static final String[] NO_ARGS = new String[0]; + + private final String httpAddress; + private final String datanodeAddress; + + private ReconServer reconServer; + + public ReconService(OzoneConfiguration conf) { + httpAddress = localhostWithFreePort(); + datanodeAddress = localhostWithFreePort(); + setReconAddress(conf); + } + + @Override + public void start(OzoneConfiguration conf) { + Preconditions.assertNull(reconServer, "Recon already started"); + + ConfigurationProvider.resetConfiguration(); + ConfigurationProvider.setConfiguration(conf); + configureRecon(conf); + + reconServer = new ReconServer(); + reconServer.execute(NO_ARGS); + } + + @Override + public void stop() { + final ReconServer instance = reconServer; + Preconditions.assertNotNull(instance, "Recon not running"); + instance.stop(); + instance.join(); + reconServer = null; + } + + @Override + public String toString() { + final ReconServer instance = reconServer; + return instance != null + ? "Recon(http=" + instance.getHttpServer().getHttpAddress() + + ", https=" + instance.getHttpServer().getHttpsAddress() + ")" + : "Recon"; + } + + ReconServer getReconServer() { + return reconServer; + } + + private void configureRecon(OzoneConfiguration conf) { + String metadataDir = Objects.requireNonNull(conf.get(OZONE_METADATA_DIRS), OZONE_METADATA_DIRS + " must be set"); + File dir = new File(metadataDir, "recon"); + conf.set(OZONE_RECON_DB_DIR, dir.getAbsolutePath()); + conf.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR, dir.getAbsolutePath()); + conf.set(OZONE_RECON_SCM_DB_DIR, dir.getAbsolutePath()); + + ReconSqlDbConfig dbConfig = conf.getObject(ReconSqlDbConfig.class); + dbConfig.setJdbcUrl("jdbc:derby:" + dir.getAbsolutePath() + + "/ozone_recon_derby.db"); + conf.setFromObject(dbConfig); + + conf.set(OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD, "10s"); + + setReconAddress(conf); + } + + private void setReconAddress(OzoneConfiguration conf) { + conf.set(OZONE_RECON_ADDRESS_KEY, datanodeAddress); + conf.set(OZONE_RECON_DATANODE_ADDRESS_KEY, datanodeAddress); + conf.set(OZONE_RECON_HTTP_ADDRESS_KEY, httpAddress); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java index 1913b1738487..70be91b78624 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java @@ -112,6 +112,7 @@ class TestReconAndAdminContainerCLI { private static OzoneBucket ozoneBucket; private static ContainerManager scmContainerManager; private static ContainerManager reconContainerManager; + private static ReconService recon; private static Stream outOfServiceNodeStateArgs() { return Stream.of( @@ -125,10 +126,11 @@ private static Stream outOfServiceNodeStateArgs() { @BeforeAll static void init() throws Exception { setupConfigKeys(); + recon = new ReconService(CONF); cluster = MiniOzoneCluster.newBuilder(CONF) - .setNumDatanodes(5) - .includeRecon(true) - .build(); + .setNumDatanodes(5) + .addService(recon) + .build(); cluster.waitForClusterToBeReady(); GenericTestUtils.setLogLevel(ReconNodeManager.class, Level.DEBUG); @@ -140,7 +142,7 @@ static void init() throws Exception { ReconStorageContainerManagerFacade reconScm = (ReconStorageContainerManagerFacade) - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); PipelineManager reconPipelineManager = reconScm.getPipelineManager(); reconContainerManager = reconScm.getContainerManager(); @@ -168,7 +170,7 @@ static void init() throws Exception { client, volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED); String keyNameR3 = "key1"; - containerIdR3 = setupRatisKey(keyNameR3, + containerIdR3 = setupRatisKey(recon, keyNameR3, HddsProtos.ReplicationFactor.THREE); } @@ -186,7 +188,7 @@ static void shutdown() { @Test void testMissingContainer() throws Exception { String keyNameR1 = "key2"; - long containerID = setupRatisKey(keyNameR1, + long containerID = setupRatisKey(recon, keyNameR1, HddsProtos.ReplicationFactor.ONE); Pipeline pipeline = @@ -377,7 +379,7 @@ private static boolean assertReportsMatch(UnHealthyContainerStates state) { return true; } - private static long setupRatisKey(String keyName, + private static long setupRatisKey(ReconService reconService, String keyName, HddsProtos.ReplicationFactor replicationFactor) throws Exception { OmKeyInfo omKeyInfo = createTestKey(keyName, RatisReplicationConfig.getInstance(replicationFactor)); @@ -395,7 +397,7 @@ private static long setupRatisKey(String keyName, reconContainerManager.getContainers()); ReconContainerMetadataManager reconContainerMetadataManager = - cluster.getReconServer().getReconContainerMetadataManager(); + reconService.getReconServer().getReconContainerMetadataManager(); // Verify Recon picked up the new keys and // updated its container key mappings. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java index 93c6e17dbaa8..1b47f9723822 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java @@ -59,14 +59,18 @@ public class TestReconAsPassiveScm { private MiniOzoneCluster cluster; private OzoneConfiguration conf; + private ReconService recon; @BeforeEach public void init() throws Exception { conf = new OzoneConfiguration(); conf.set(HDDS_CONTAINER_REPORT_INTERVAL, "5s"); conf.set(HDDS_PIPELINE_REPORT_INTERVAL, "5s"); - cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3) - .includeRecon(true).build(); + recon = new ReconService(conf); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(3) + .addService(recon) + .build(); cluster.waitForClusterToBeReady(); GenericTestUtils.setLogLevel(ReconNodeManager.class, Level.DEBUG); } @@ -82,7 +86,7 @@ public void shutdown() { public void testDatanodeRegistrationAndReports() throws Exception { ReconStorageContainerManagerFacade reconScm = (ReconStorageContainerManagerFacade) - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); StorageContainerManager scm = cluster.getStorageContainerManager(); PipelineManager reconPipelineManager = reconScm.getPipelineManager(); PipelineManager scmPipelineManager = scm.getPipelineManager(); @@ -143,7 +147,7 @@ public void testDatanodeRegistrationAndReports() throws Exception { @Test public void testReconRestart() throws Exception { final OzoneStorageContainerManager reconScm = - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); StorageContainerManager scm = cluster.getStorageContainerManager(); // Stop Recon @@ -155,7 +159,7 @@ public void testReconRestart() throws Exception { LambdaTestUtils.await(60000, 5000, () -> (reconScm.getScmNodeManager().getAllNodes().size() == 3)); - cluster.stopRecon(); + recon.stop(); // Create container in SCM. ContainerInfo containerInfo = @@ -180,7 +184,7 @@ public void testReconRestart() throws Exception { scmPipelineManager.deletePipeline(pipelineToClose.get().getId()); // Start Recon - cluster.startRecon(); + recon.start(cluster.getConf()); // Verify if Recon has all the nodes on restart (even if heartbeats are // not yet received). @@ -191,7 +195,7 @@ public void testReconRestart() throws Exception { // Verify Recon picks up new container, close pipeline SCM actions. OzoneStorageContainerManager newReconScm = - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); PipelineManager reconPipelineManager = newReconScm.getPipelineManager(); assertFalse( reconPipelineManager.containsPipeline(pipelineToClose.get().getId())); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerEndpoint.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerEndpoint.java index 14e72e36563b..45325042d1b6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerEndpoint.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerEndpoint.java @@ -55,15 +55,17 @@ public class TestReconContainerEndpoint { private MiniOzoneCluster cluster; private OzoneClient client; private ObjectStore store; + private ReconService recon; @BeforeEach public void init() throws Exception { conf = new OzoneConfiguration(); conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED); + recon = new ReconService(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) - .includeRecon(true) + .addService(recon) .build(); cluster.waitForClusterToBeReady(); client = cluster.newClient(); @@ -102,7 +104,7 @@ public void testContainerEndpointForFSOLayout() throws Exception { // Synchronize data from OM to Recon OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) - cluster.getReconServer().getOzoneManagerServiceProvider(); + recon.getReconServer().getOzoneManagerServiceProvider(); impl.syncDataFromOM(); //Search for the bucket from the bucket table and verify its FSO @@ -162,7 +164,7 @@ public void testContainerEndpointForOBSBucket() throws Exception { writeTestData(volumeName, obsBucketName, obsSingleFileKey, "Hello OBS!"); OzoneManagerServiceProviderImpl impl = - (OzoneManagerServiceProviderImpl) cluster.getReconServer() + (OzoneManagerServiceProviderImpl) recon.getReconServer() .getOzoneManagerServiceProvider(); impl.syncDataFromOM(); @@ -191,19 +193,19 @@ public void testContainerEndpointForOBSBucket() throws Exception { private Response getContainerEndpointResponse(long containerId) { OzoneStorageContainerManager reconSCM = - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); ReconContainerManager reconContainerManager = (ReconContainerManager) reconSCM.getContainerManager(); ContainerHealthSchemaManager containerHealthSchemaManager = reconContainerManager.getContainerSchemaManager(); ReconOMMetadataManager omMetadataManagerInstance = (ReconOMMetadataManager) - cluster.getReconServer().getOzoneManagerServiceProvider() + recon.getReconServer().getOzoneManagerServiceProvider() .getOMMetadataManagerInstance(); ContainerEndpoint containerEndpoint = new ContainerEndpoint(reconSCM, containerHealthSchemaManager, - cluster.getReconServer().getReconNamespaceSummaryManager(), - cluster.getReconServer().getReconContainerMetadataManager(), + recon.getReconServer().getReconNamespaceSummaryManager(), + recon.getReconServer().getReconContainerMetadataManager(), omMetadataManagerInstance); return containerEndpoint.getKeysForContainer(containerId, 10, ""); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java index 14b11c74fd2e..904a1a919327 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java @@ -82,6 +82,7 @@ public class TestReconInsightsForDeletedDirectories { private static String volumeName; private static String bucketName; private static OzoneClient client; + private static ReconService recon; @BeforeAll public static void init() throws Exception { @@ -90,9 +91,10 @@ public static void init() throws Exception { conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 10000000, TimeUnit.MILLISECONDS); conf.setBoolean(OZONE_ACL_ENABLED, true); + recon = new ReconService(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) - .includeRecon(true) + .addService(recon) .build(); cluster.waitForClusterToBeReady(); client = cluster.newClient(); @@ -174,7 +176,7 @@ public void testGetDeletedDirectoryInfo() // Retrieve tables from Recon's OM-DB. ReconOMMetadataManager reconOmMetadataManagerInstance = - (ReconOMMetadataManager) cluster.getReconServer() + (ReconOMMetadataManager) recon.getReconServer() .getOzoneManagerServiceProvider().getOMMetadataManagerInstance(); Table reconFileTable = reconOmMetadataManagerInstance.getKeyTable(getFSOBucketLayout()); @@ -203,7 +205,7 @@ public void testGetDeletedDirectoryInfo() } else { // Retrieve Namespace Summary for dir1 from Recon. ReconNamespaceSummaryManagerImpl namespaceSummaryManager = - (ReconNamespaceSummaryManagerImpl) cluster.getReconServer() + (ReconNamespaceSummaryManagerImpl) recon.getReconServer() .getReconNamespaceSummaryManager(); NSSummary summary = namespaceSummaryManager.getNSSummary(directoryObjectId); @@ -222,9 +224,9 @@ public void testGetDeletedDirectoryInfo() // Create an Instance of OMDBInsightEndpoint. OzoneStorageContainerManager reconSCM = - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager = - (ReconNamespaceSummaryManagerImpl) cluster.getReconServer() + (ReconNamespaceSummaryManagerImpl) recon.getReconServer() .getReconNamespaceSummaryManager(); OMDBInsightEndpoint omdbInsightEndpoint = @@ -286,7 +288,7 @@ public void testGetDeletedDirectoryInfoForNestedDirectories() // Retrieve tables from Recon's OM-DB. ReconOMMetadataManager reconOmMetadataManagerInstance = - (ReconOMMetadataManager) cluster.getReconServer() + (ReconOMMetadataManager) recon.getReconServer() .getOzoneManagerServiceProvider().getOMMetadataManagerInstance(); Table reconFileTable = reconOmMetadataManagerInstance.getKeyTable(getFSOBucketLayout()); @@ -302,9 +304,9 @@ public void testGetDeletedDirectoryInfoForNestedDirectories() // Create an Instance of OMDBInsightEndpoint. OzoneStorageContainerManager reconSCM = - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); ReconNamespaceSummaryManagerImpl namespaceSummaryManager = - (ReconNamespaceSummaryManagerImpl) cluster.getReconServer() + (ReconNamespaceSummaryManagerImpl) recon.getReconServer() .getReconNamespaceSummaryManager(); OMDBInsightEndpoint omdbInsightEndpoint = @@ -373,12 +375,12 @@ public void testGetDeletedDirectoryInfoWithMultipleSubdirectories() // Fetch the deleted directory info from Recon OmDbInsightEndpoint. OzoneStorageContainerManager reconSCM = - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); ReconNamespaceSummaryManagerImpl namespaceSummaryManager = - (ReconNamespaceSummaryManagerImpl) cluster.getReconServer() + (ReconNamespaceSummaryManagerImpl) recon.getReconServer() .getReconNamespaceSummaryManager(); ReconOMMetadataManager reconOmMetadataManagerInstance = - (ReconOMMetadataManager) cluster.getReconServer() + (ReconOMMetadataManager) recon.getReconServer() .getOzoneManagerServiceProvider().getOMMetadataManagerInstance(); OMDBInsightEndpoint omdbInsightEndpoint = new OMDBInsightEndpoint(reconSCM, reconOmMetadataManagerInstance, @@ -467,7 +469,7 @@ private boolean assertTableRowCount(int expectedCount, AtomicLong count = new AtomicLong(0L); assertDoesNotThrow(() -> { if (isRecon) { - count.set(cluster.getReconServer().getOzoneManagerServiceProvider() + count.set(recon.getReconServer().getOzoneManagerServiceProvider() .getOMMetadataManagerInstance().countRowsInTable(table)); } else { count.set(cluster.getOzoneManager().getMetadataManager() @@ -482,7 +484,7 @@ private boolean assertTableRowCount(int expectedCount, private void syncDataFromOM() throws IOException { // Sync data from Ozone Manager to Recon. OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) - cluster.getReconServer().getOzoneManagerServiceProvider(); + recon.getReconServer().getOzoneManagerServiceProvider(); impl.syncDataFromOM(); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconScmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconScmSnapshot.java index 9f75a3f08a46..2084af458d2f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconScmSnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconScmSnapshot.java @@ -46,7 +46,8 @@ */ public class TestReconScmSnapshot { private OzoneConfiguration conf; - private MiniOzoneCluster ozoneCluster = null; + private MiniOzoneCluster cluster; + private ReconService recon; @BeforeEach public void setup() throws Exception { @@ -56,33 +57,30 @@ public void setup() throws Exception { conf.setBoolean( ReconServerConfigKeys.OZONE_RECON_SCM_SNAPSHOT_ENABLED, true); conf.setInt(ReconServerConfigKeys.OZONE_RECON_SCM_CONTAINER_THRESHOLD, 0); - ozoneCluster = MiniOzoneCluster.newBuilder(conf) + recon = new ReconService(conf); + cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(4) - .includeRecon(true) + .addService(recon) .build(); - ozoneCluster.waitForClusterToBeReady(); + cluster.waitForClusterToBeReady(); } @Test public void testScmSnapshot() throws Exception { - testSnapshot(ozoneCluster); - } - - public static void testSnapshot(MiniOzoneCluster cluster) throws Exception { LogCapturer logCapturer = LogCapturer.captureLogs(ReconStorageContainerManagerFacade.class); - List reconContainers = cluster.getReconServer() + List reconContainers = recon.getReconServer() .getReconStorageContainerManager().getContainerManager() .getContainers(); assertEquals(0, reconContainers.size()); ReconNodeManager nodeManager; - nodeManager = (ReconNodeManager) cluster.getReconServer() + nodeManager = (ReconNodeManager) recon.getReconServer() .getReconStorageContainerManager().getScmNodeManager(); long keyCountBefore = nodeManager.getNodeDBKeyCount(); //Stopping Recon to add Containers in SCM - cluster.stopRecon(); + recon.stop(); ContainerManager containerManager; containerManager = cluster.getStorageContainerManager() @@ -93,12 +91,12 @@ public static void testSnapshot(MiniOzoneCluster cluster) throws Exception { HddsProtos.ReplicationFactor.ONE), "testOwner"); } - cluster.startRecon(); + recon.start(conf); //ContainerCount after Recon DB is updated with SCM DB containerManager = cluster.getStorageContainerManager() .getContainerManager(); - ContainerManager reconContainerManager = cluster.getReconServer() + ContainerManager reconContainerManager = recon.getReconServer() .getReconStorageContainerManager().getContainerManager(); assertTrue(logCapturer.getOutput() .contains("Recon Container Count: " + reconContainers.size() + @@ -109,13 +107,13 @@ public static void testSnapshot(MiniOzoneCluster cluster) throws Exception { //PipelineCount after Recon DB is updated with SCM DB PipelineManager scmPipelineManager = cluster.getStorageContainerManager() .getPipelineManager(); - PipelineManager reconPipelineManager = cluster.getReconServer() + PipelineManager reconPipelineManager = recon.getReconServer() .getReconStorageContainerManager().getPipelineManager(); assertEquals(scmPipelineManager.getPipelines().size(), reconPipelineManager.getPipelines().size()); //NodeCount after Recon DB updated with SCM DB - nodeManager = (ReconNodeManager) cluster.getReconServer() + nodeManager = (ReconNodeManager) recon.getReconServer() .getReconStorageContainerManager().getScmNodeManager(); long keyCountAfter = nodeManager.getNodeDBKeyCount(); assertEquals(keyCountAfter, keyCountBefore); @@ -124,14 +122,14 @@ public static void testSnapshot(MiniOzoneCluster cluster) throws Exception { @Test @Flaky("HDDS-11645") public void testExplicitRemovalOfNode() throws Exception { - ReconNodeManager nodeManager = (ReconNodeManager) ozoneCluster.getReconServer() + ReconNodeManager nodeManager = (ReconNodeManager) recon.getReconServer() .getReconStorageContainerManager().getScmNodeManager(); long nodeDBCountBefore = nodeManager.getNodeDBKeyCount(); List allNodes = nodeManager.getAllNodes(); assertEquals(nodeDBCountBefore, allNodes.size()); DatanodeDetails datanodeDetails = allNodes.get(3); - ozoneCluster.shutdownHddsDatanode(datanodeDetails); + cluster.shutdownHddsDatanode(datanodeDetails); GenericTestUtils.waitFor(() -> { try { @@ -168,8 +166,8 @@ public void testExplicitRemovalOfNode() throws Exception { @AfterEach public void shutdown() throws Exception { - if (ozoneCluster != null) { - ozoneCluster.shutdown(); + if (cluster != null) { + cluster.shutdown(); } } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java index f5ee2b4b053d..c4a7dbb0fbe8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconTasks.java @@ -60,6 +60,7 @@ public class TestReconTasks { private MiniOzoneCluster cluster = null; private OzoneConfiguration conf; + private ReconService recon; @BeforeEach public void init() throws Exception { @@ -73,8 +74,10 @@ public void init() throws Exception { conf.set("ozone.scm.stale.node.interval", "6s"); conf.set("ozone.scm.dead.node.interval", "8s"); + recon = new ReconService(conf); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1) - .includeRecon(true).build(); + .addService(recon) + .build(); cluster.waitForClusterToBeReady(); cluster.waitForPipelineTobeReady(ONE, 30000); GenericTestUtils.setLogLevel(SCMDatanodeHeartbeatDispatcher.class, @@ -92,7 +95,7 @@ public void shutdown() { public void testSyncSCMContainerInfo() throws Exception { ReconStorageContainerManagerFacade reconScm = (ReconStorageContainerManagerFacade) - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); StorageContainerManager scm = cluster.getStorageContainerManager(); ContainerManager scmContainerManager = scm.getContainerManager(); ContainerManager reconContainerManager = reconScm.getContainerManager(); @@ -124,9 +127,9 @@ public void testSyncSCMContainerInfo() throws Exception { public void testMissingContainerDownNode() throws Exception { ReconStorageContainerManagerFacade reconScm = (ReconStorageContainerManagerFacade) - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); ReconContainerMetadataManager reconContainerMetadataManager = - cluster.getReconServer().getReconContainerMetadataManager(); + recon.getReconServer().getReconContainerMetadataManager(); StorageContainerManager scm = cluster.getStorageContainerManager(); PipelineManager reconPipelineManager = reconScm.getPipelineManager(); @@ -205,9 +208,9 @@ public void testMissingContainerDownNode() throws Exception { public void testEmptyMissingContainerDownNode() throws Exception { ReconStorageContainerManagerFacade reconScm = (ReconStorageContainerManagerFacade) - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); ReconContainerMetadataManager reconContainerMetadataManager = - cluster.getReconServer().getReconContainerMetadataManager(); + recon.getReconServer().getReconContainerMetadataManager(); StorageContainerManager scm = cluster.getStorageContainerManager(); PipelineManager reconPipelineManager = reconScm.getPipelineManager(); PipelineManager scmPipelineManager = scm.getPipelineManager(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java index adbbf7971c32..4f3a354edfaf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManager.java @@ -19,6 +19,7 @@ import static java.net.HttpURLConnection.HTTP_CREATED; import static java.net.HttpURLConnection.HTTP_OK; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_CONNECTION_REQUEST_TIMEOUT; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_CONNECTION_REQUEST_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_CONNECTION_TIMEOUT; @@ -34,7 +35,6 @@ import static org.slf4j.event.Level.INFO; import java.io.IOException; -import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -79,6 +79,7 @@ public class TestReconWithOzoneManager { private static OMMetadataManager metadataManager; private static CloseableHttpClient httpClient; private static String taskStatusURL; + private static ReconService recon; @BeforeAll public static void init() throws Exception { @@ -109,19 +110,17 @@ public static void init() throws Exception { .setConnectionRequestTimeout(connectionTimeout) .setSocketTimeout(connectionRequestTimeout).build(); - cluster = - MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(1) - .includeRecon(true) - .build(); + recon = new ReconService(conf); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(1) + .addService(recon) + .build(); cluster.waitForClusterToBeReady(); metadataManager = cluster.getOzoneManager().getMetadataManager(); cluster.getStorageContainerManager().exitSafeMode(); - InetSocketAddress address = - cluster.getReconServer().getHttpServer().getHttpAddress(); - String reconHTTPAddress = address.getHostName() + ":" + address.getPort(); + String reconHTTPAddress = conf.get(OZONE_RECON_HTTP_ADDRESS_KEY); taskStatusURL = "http://" + reconHTTPAddress + "/api/v1/task/status"; // initialize HTTPClient @@ -181,7 +180,7 @@ public void testOmDBSyncing() throws Exception { assertEquals("bucket0", keyInfo1.getBucketName()); OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) - cluster.getReconServer().getOzoneManagerServiceProvider(); + recon.getReconServer().getOzoneManagerServiceProvider(); impl.syncDataFromOM(); OzoneManagerSyncMetrics metrics = impl.getMetrics(); @@ -223,10 +222,11 @@ public void testOmDBSyncing() throws Exception { "lastUpdatedTimestamp"); //restart Recon - cluster.restartReconServer(); + recon.stop(); + recon.start(cluster.getConf()); impl = (OzoneManagerServiceProviderImpl) - cluster.getReconServer().getOzoneManagerServiceProvider(); + recon.getReconServer().getOzoneManagerServiceProvider(); //add 5 more keys to OM addKeys(5, 10); @@ -287,7 +287,7 @@ public void testOmDBSyncWithSeqNumberMismatch() throws Exception { assertEquals("bucket10", keyInfo1.getBucketName()); OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) - cluster.getReconServer().getOzoneManagerServiceProvider(); + recon.getReconServer().getOzoneManagerServiceProvider(); impl.syncDataFromOM(); OzoneManagerSyncMetrics metrics = impl.getMetrics(); @@ -296,7 +296,7 @@ public void testOmDBSyncWithSeqNumberMismatch() throws Exception { .getDb().getLatestSequenceNumber(); OMMetadataManager reconMetadataManagerInstance = - cluster.getReconServer().getOzoneManagerServiceProvider() + recon.getReconServer().getOzoneManagerServiceProvider() .getOMMetadataManagerInstance(); long reconLatestSeqNumber = ((RDBStore) reconMetadataManagerInstance.getStore()).getDb() diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java index e2e8485466a9..1fc0547ab333 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerFSO.java @@ -55,17 +55,18 @@ public class TestReconWithOzoneManagerFSO { private static MiniOzoneCluster cluster = null; private static OzoneConfiguration conf; private static ObjectStore store; + private static ReconService recon; @BeforeAll public static void init() throws Exception { conf = new OzoneConfiguration(); conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED); - cluster = - MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) - .includeRecon(true) - .build(); + recon = new ReconService(conf); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(3) + .addService(recon) + .build(); cluster.waitForClusterToBeReady(); cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.ONE, 30000); @@ -98,16 +99,16 @@ public void testNamespaceSummaryAPI() throws Exception { // add a vol, bucket and key addKeys(0, 10, "dir"); OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) - cluster.getReconServer().getOzoneManagerServiceProvider(); + recon.getReconServer().getOzoneManagerServiceProvider(); impl.syncDataFromOM(); ReconNamespaceSummaryManager namespaceSummaryManager = - cluster.getReconServer().getReconNamespaceSummaryManager(); + recon.getReconServer().getReconNamespaceSummaryManager(); ReconOMMetadataManager omMetadataManagerInstance = (ReconOMMetadataManager) - cluster.getReconServer().getOzoneManagerServiceProvider() + recon.getReconServer().getOzoneManagerServiceProvider() .getOMMetadataManagerInstance(); OzoneStorageContainerManager reconSCM = - cluster.getReconServer().getReconStorageContainerManager(); + recon.getReconServer().getReconStorageContainerManager(); NSSummaryEndpoint endpoint = new NSSummaryEndpoint(namespaceSummaryManager, omMetadataManagerInstance, reconSCM); Response basicInfo = endpoint.getBasicInfo("/vol1/bucket1/dir1"); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java index 7a6673b01e76..da6930833710 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java @@ -59,6 +59,7 @@ public class TestReconWithOzoneManagerHA { private static final String OM_SERVICE_ID = "omService1"; private static final String VOL_NAME = "testrecon"; private OzoneClient client; + private ReconService recon; @BeforeEach public void setup() throws Exception { @@ -70,10 +71,11 @@ public void setup() throws Exception { conf.setFromObject(dbConf); MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf); + recon = new ReconService(conf); builder.setOMServiceId(OM_SERVICE_ID) .setNumOfOzoneManagers(3) .setNumDatanodes(1) - .includeRecon(true); + .addService(recon); cluster = builder.build(); cluster.waitForClusterToBeReady(); client = OzoneClientFactory.getRpcClient(OM_SERVICE_ID, conf); @@ -108,7 +110,7 @@ public void testReconGetsSnapshotFromLeader() throws Exception { assertTrue(ozoneManager.get().isLeaderReady(), "Should have gotten the leader!"); OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) - cluster.getReconServer().getOzoneManagerServiceProvider(); + recon.getReconServer().getOzoneManagerServiceProvider(); String hostname = ozoneManager.get().getHttpServer().getHttpAddress().getHostName(); @@ -131,7 +133,7 @@ public void testReconGetsSnapshotFromLeader() throws Exception { impl.syncDataFromOM(); ReconContainerMetadataManager reconContainerMetadataManager = - cluster.getReconServer().getReconContainerMetadataManager(); + recon.getReconServer().getReconContainerMetadataManager(); try (TableIterator iterator = reconContainerMetadataManager.getContainerTableIterator()) { String reconKeyPrefix = null; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java index a4292eeab499..89f98d6fcf88 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java @@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.ozone.recon.ReconService; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -59,7 +60,9 @@ public static void init() throws Exception { OMRequestTestUtils.configureFSOptimizedPaths(conf, true); conf.set(OZONE_RECON_ADDRESS_KEY, "localhost:9888"); cluster = MiniOzoneCluster.newBuilder(conf) - .withoutDatanodes().includeRecon(true).build(); + .withoutDatanodes() + .addService(new ReconService(conf)) + .build(); cluster.waitForClusterToBeReady(); client = cluster.newClient(); store = client.getObjectStore(); diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index 3b376ceefa28..a1bcadd79366 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.recon.ReconServer; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.function.CheckedFunction; @@ -137,13 +136,6 @@ void waitForPipelineTobeReady(HddsProtos.ReplicationFactor factor, HddsDatanodeService getHddsDatanode(DatanodeDetails dn) throws IOException; - /** - * Returns a {@link ReconServer} instance. - * - * @return {@link ReconServer} instance if it is initialized, otherwise null. - */ - ReconServer getReconServer(); - /** * Returns an {@link OzoneClient} to access the {@link MiniOzoneCluster}. * The caller is responsible for closing the client after use. @@ -171,11 +163,6 @@ void restartStorageContainerManager(boolean waitForDatanode) */ void restartOzoneManager() throws IOException; - /** - * Restarts Recon instance. - */ - void restartReconServer() throws Exception; - /** * Restart a particular HddsDatanode. * @@ -208,16 +195,6 @@ void restartHddsDatanode(DatanodeDetails dn, boolean waitForDatanode) */ void shutdownHddsDatanode(DatanodeDetails dn) throws IOException; - /** - * Start Recon. - */ - void startRecon(); - - /** - * Stop Recon. - */ - void stopRecon(); - /** * Shutdown the MiniOzoneCluster and delete the storage dirs. */ @@ -276,8 +253,6 @@ abstract class Builder { protected String scmId = UUID.randomUUID().toString(); protected String omId = UUID.randomUUID().toString(); - protected boolean includeRecon = false; - protected int numOfDatanodes = 3; protected boolean startDataNodes = true; protected CertificateClient certClient; @@ -377,11 +352,6 @@ public Builder setDatanodeFactory(DatanodeFactory factory) { return this; } - public Builder includeRecon(boolean include) { - this.includeRecon = include; - return this; - } - public Builder addService(Service service) { services.add(service); return this; diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index dd0d476170a8..cf18edbb5ba1 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -19,14 +19,6 @@ import static java.util.Collections.singletonList; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; -import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY; -import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY; -import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY; -import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD; -import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR; -import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR; -import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR; -import static org.apache.ozone.test.GenericTestUtils.PortAllocator.anyHostWithFreePort; import static org.apache.ozone.test.GenericTestUtils.PortAllocator.getFreePort; import static org.apache.ozone.test.GenericTestUtils.PortAllocator.localhostWithFreePort; @@ -63,7 +55,6 @@ import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider; import org.apache.hadoop.hdds.scm.safemode.HealthyPipelineSafeModeRule; import org.apache.hadoop.hdds.scm.safemode.SafeModeRuleFactory; -import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; @@ -82,10 +73,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMStorage; import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.recon.ConfigurationProvider; -import org.apache.hadoop.ozone.recon.ReconServer; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.ozone.recon.schema.ReconSqlDbConfig; import org.apache.ozone.test.GenericTestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -112,7 +100,6 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster { private StorageContainerManager scm; private OzoneManager ozoneManager; private final List hddsDatanodes; - private ReconServer reconServer; private final List services; // Timeout for the cluster to be ready @@ -121,20 +108,16 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster { private final Set clients = ConcurrentHashMap.newKeySet(); private SecretKeyClient secretKeyClient; - /** - * Creates a new MiniOzoneCluster with Recon. - */ private MiniOzoneClusterImpl(OzoneConfiguration conf, SCMConfigurator scmConfigurator, OzoneManager ozoneManager, StorageContainerManager scm, List hddsDatanodes, - ReconServer reconServer, List services) { + List services) { this.conf = conf; this.ozoneManager = ozoneManager; this.scm = scm; this.hddsDatanodes = hddsDatanodes; - this.reconServer = reconServer; this.scmConfigurator = scmConfigurator; this.services = services; } @@ -146,11 +129,10 @@ private MiniOzoneClusterImpl(OzoneConfiguration conf, * OzoneManagers and StorageContainerManagers. */ MiniOzoneClusterImpl(OzoneConfiguration conf, SCMConfigurator scmConfigurator, - List hddsDatanodes, ReconServer reconServer, List services) { + List hddsDatanodes, List services) { this.scmConfigurator = scmConfigurator; this.conf = conf; this.hddsDatanodes = hddsDatanodes; - this.reconServer = reconServer; this.services = services; } @@ -256,11 +238,6 @@ public HddsDatanodeService getHddsDatanode(DatanodeDetails dn) "Not able to find datanode with datanode Id " + dn.getUuid()); } - @Override - public ReconServer getReconServer() { - return this.reconServer; - } - @Override public int getHddsDatanodeIndex(DatanodeDetails dn) throws IOException { for (HddsDatanodeService service : hddsDatanodes) { @@ -321,12 +298,6 @@ public void restartOzoneManager() throws IOException { ozoneManager.restart(); } - @Override - public void restartReconServer() { - stopRecon(reconServer); - startRecon(); - } - private void waitForHddsDatanodeToStop(DatanodeDetails dn) throws TimeoutException, InterruptedException { GenericTestUtils.waitFor(() -> { @@ -411,7 +382,6 @@ public void stop() { stopOM(ozoneManager); stopDatanodes(hddsDatanodes); stopSCM(scm); - stopRecon(reconServer); stopServices(services); } @@ -441,17 +411,6 @@ public void shutdownHddsDatanodes() { }); } - @Override - public void startRecon() { - reconServer = new ReconServer(); - reconServer.execute(NO_ARGS); - } - - @Override - public void stopRecon() { - stopRecon(reconServer); - } - public void startServices() throws Exception { for (Service service : services) { service.start(getConf()); @@ -500,18 +459,6 @@ protected static void stopOM(OzoneManager om) { } } - private static void stopRecon(ReconServer reconServer) { - try { - if (reconServer != null) { - LOG.info("Stopping Recon"); - reconServer.stop(); - reconServer.join(); - } - } catch (Exception e) { - LOG.error("Exception while shutting down Recon.", e); - } - } - private static void stopServices(List services) { // stop in reverse order List reverse = new ArrayList<>(services); @@ -548,32 +495,27 @@ public MiniOzoneCluster build() throws IOException { initializeConfiguration(); StorageContainerManager scm = null; OzoneManager om = null; - ReconServer reconServer = null; List hddsDatanodes = Collections.emptyList(); try { scm = createAndStartSingleSCM(); om = createAndStartSingleOM(); - reconServer = createRecon(); hddsDatanodes = createHddsDatanodes(); MiniOzoneClusterImpl cluster = new MiniOzoneClusterImpl(conf, scmConfigurator, om, scm, - hddsDatanodes, reconServer, getServices()); + hddsDatanodes, getServices()); + cluster.startServices(); cluster.setCAClient(certClient); cluster.setSecretKeyClient(secretKeyClient); if (startDataNodes) { cluster.startHddsDatanodes(); } - cluster.startServices(); prepareForNextBuild(); return cluster; } catch (Exception ex) { stopOM(om); - if (includeRecon) { - stopRecon(reconServer); - } stopServices(getServices()); if (startDataNodes) { stopDatanodes(hddsDatanodes); @@ -727,22 +669,6 @@ protected void configureScmDatanodeAddress(List scms) { conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, getSCMAddresses(scms)); } - protected ReconServer createRecon() { - ReconServer reconServer = null; - if (includeRecon) { - configureRecon(); - reconServer = new ReconServer(); - reconServer.execute(NO_ARGS); - - OzoneStorageContainerManager reconScm = - reconServer.getReconStorageContainerManager(); - conf.set(OZONE_RECON_ADDRESS_KEY, - reconScm.getDatanodeRpcAddress().getHostString() + ":" + - reconScm.getDatanodeRpcAddress().getPort()); - } - return reconServer; - } - /** * Creates HddsDatanodeService(s) instance. * @@ -786,27 +712,5 @@ private void configureOM() { conf.setInt(OMConfigKeys.OZONE_OM_RATIS_PORT_KEY, getFreePort()); } - protected void configureRecon() { - ConfigurationProvider.resetConfiguration(); - - File tempNewFolder = new File(path, "recon"); - conf.set(OZONE_RECON_DB_DIR, - tempNewFolder.getAbsolutePath()); - conf.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR, tempNewFolder - .getAbsolutePath()); - conf.set(OZONE_RECON_SCM_DB_DIR, - tempNewFolder.getAbsolutePath()); - - ReconSqlDbConfig dbConfig = conf.getObject(ReconSqlDbConfig.class); - dbConfig.setJdbcUrl("jdbc:derby:" + tempNewFolder.getAbsolutePath() - + "/ozone_recon_derby.db"); - conf.setFromObject(dbConfig); - - conf.set(OZONE_RECON_HTTP_ADDRESS_KEY, anyHostWithFreePort()); - conf.set(OZONE_RECON_DATANODE_ADDRESS_KEY, anyHostWithFreePort()); - conf.set(OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD, "10s"); - - ConfigurationProvider.setConfiguration(conf); - } } } diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java index fdea7b0f4fe8..8612eaa17484 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java @@ -17,7 +17,6 @@ package org.apache.hadoop.ozone; -import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.ozone.test.GenericTestUtils.PortAllocator.getFreePort; @@ -52,7 +51,6 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; -import org.apache.hadoop.ozone.recon.ReconServer; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.util.function.CheckedConsumer; @@ -87,8 +85,8 @@ public MiniOzoneHAClusterImpl( SCMHAService scmhaService, List hddsDatanodes, String clusterPath, - ReconServer reconServer) { - super(conf, scmConfigurator, hddsDatanodes, reconServer, emptyList()); + List services) { + super(conf, scmConfigurator, hddsDatanodes, services); this.omhaService = omhaService; this.scmhaService = scmhaService; this.clusterMetaPath = clusterPath; @@ -421,11 +419,9 @@ public MiniOzoneHAClusterImpl build() throws IOException { initOMRatisConf(); SCMHAService scmService; OMHAService omService; - ReconServer reconServer; try { scmService = createSCMService(); omService = createOMService(); - reconServer = createRecon(); } catch (AuthenticationException ex) { throw new IOException("Unable to build MiniOzoneCluster. ", ex); } @@ -433,8 +429,12 @@ public MiniOzoneHAClusterImpl build() throws IOException { final List hddsDatanodes = createHddsDatanodes(); MiniOzoneHAClusterImpl cluster = new MiniOzoneHAClusterImpl(conf, - scmConfigurator, omService, scmService, hddsDatanodes, path, - reconServer); + scmConfigurator, omService, scmService, hddsDatanodes, path, getServices()); + try { + cluster.startServices(); + } catch (Exception e) { + throw new IOException("Unable to start services", e); + } if (startDataNodes) { cluster.startHddsDatanodes();