HDDS-12568. Implement MiniOzoneCluster.Service for Recon

hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java

@@ -72,7 +72,7 @@ public MiniOzoneChaosCluster(OzoneConfiguration conf,
       List<HddsDatanodeService> hddsDatanodes, String clusterPath,
       Set<Class<? extends Failures>> clazzes) {
     super(conf, new SCMConfigurator(), omService, scmService, hddsDatanodes,
-        clusterPath, null);
+        clusterPath, Collections.emptyList());
     this.numDatanodes = getHddsDatanodes().size();
     this.numOzoneManagers = omService.getServices().size();
     this.numStorageContainerManagers = scmService.getServices().size();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java

@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon;
+
+import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY;
+import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR;
+import static org.apache.ozone.test.GenericTestUtils.PortAllocator.localhostWithFreePort;
+
+import java.io.File;
+import java.util.Objects;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.ozone.recon.schema.ReconSqlDbConfig;
+import org.apache.ratis.util.Preconditions;
+
+/** Recon for {@link MiniOzoneCluster}. */
+public class ReconService implements MiniOzoneCluster.Service {
+
+  private static final String[] NO_ARGS = new String[0];
+
+  private final String httpAddress;
+  private final String datanodeAddress;
+
+  private ReconServer reconServer;
+
+  public ReconService(OzoneConfiguration conf) {
+    httpAddress = localhostWithFreePort();
+    datanodeAddress = localhostWithFreePort();
+    setReconAddress(conf);
+  }
+
+  @Override
+  public void start(OzoneConfiguration conf) {
+    Preconditions.assertNull(reconServer, "Recon already started");
+
+    ConfigurationProvider.resetConfiguration();
+    ConfigurationProvider.setConfiguration(conf);
+    configureRecon(conf);
+
+    reconServer = new ReconServer();
+    reconServer.execute(NO_ARGS);
+  }
+
+  @Override
+  public void stop() {
+    final ReconServer instance = reconServer;
+    Preconditions.assertNotNull(instance, "Recon not running");
+    instance.stop();
+    instance.join();
+    reconServer = null;
+  }
+
+  @Override
+  public String toString() {
+    final ReconServer instance = reconServer;
+    return instance != null
+        ? "Recon(http=" + instance.getHttpServer().getHttpAddress()
+            + ", https=" + instance.getHttpServer().getHttpsAddress() + ")"
+        : "Recon";
+  }
+
+  ReconServer getReconServer() {
+    return reconServer;
+  }
+
+  private void configureRecon(OzoneConfiguration conf) {
+    String metadataDir = Objects.requireNonNull(conf.get(OZONE_METADATA_DIRS), OZONE_METADATA_DIRS + " must be set");
+    File dir = new File(metadataDir, "recon");
+    conf.set(OZONE_RECON_DB_DIR, dir.getAbsolutePath());
+    conf.set(OZONE_RECON_OM_SNAPSHOT_DB_DIR, dir.getAbsolutePath());
+    conf.set(OZONE_RECON_SCM_DB_DIR, dir.getAbsolutePath());
+
+    ReconSqlDbConfig dbConfig = conf.getObject(ReconSqlDbConfig.class);
+    dbConfig.setJdbcUrl("jdbc:derby:" + dir.getAbsolutePath()
+        + "/ozone_recon_derby.db");
+    conf.setFromObject(dbConfig);
+
+    conf.set(OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD, "10s");
+
+    setReconAddress(conf);
+  }
+
+  private void setReconAddress(OzoneConfiguration conf) {
+    conf.set(OZONE_RECON_ADDRESS_KEY, datanodeAddress);
+    conf.set(OZONE_RECON_DATANODE_ADDRESS_KEY, datanodeAddress);
+    conf.set(OZONE_RECON_HTTP_ADDRESS_KEY, httpAddress);
+  }
+}

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAndAdminContainerCLI.java

@@ -112,6 +112,7 @@ class TestReconAndAdminContainerCLI {
   private static OzoneBucket ozoneBucket;
   private static ContainerManager scmContainerManager;
   private static ContainerManager reconContainerManager;
+  private static ReconService recon;
 
   private static Stream<Arguments> outOfServiceNodeStateArgs() {
     return Stream.of(
@@ -125,10 +126,11 @@ private static Stream<Arguments> outOfServiceNodeStateArgs() {
   @BeforeAll
   static void init() throws Exception {
     setupConfigKeys();
+    recon = new ReconService(CONF);
     cluster = MiniOzoneCluster.newBuilder(CONF)
-                  .setNumDatanodes(5)
-                  .includeRecon(true)
-                  .build();
+        .setNumDatanodes(5)
+        .addService(recon)
+        .build();
     cluster.waitForClusterToBeReady();
     GenericTestUtils.setLogLevel(ReconNodeManager.class, Level.DEBUG);
 
@@ -140,7 +142,7 @@ static void init() throws Exception {
 
     ReconStorageContainerManagerFacade reconScm =
         (ReconStorageContainerManagerFacade)
-            cluster.getReconServer().getReconStorageContainerManager();
+            recon.getReconServer().getReconStorageContainerManager();
     PipelineManager reconPipelineManager = reconScm.getPipelineManager();
     reconContainerManager = reconScm.getContainerManager();
 
@@ -168,7 +170,7 @@ static void init() throws Exception {
         client, volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED);
 
     String keyNameR3 = "key1";
-    containerIdR3 = setupRatisKey(keyNameR3,
+    containerIdR3 = setupRatisKey(recon, keyNameR3,
         HddsProtos.ReplicationFactor.THREE);
   }
 
@@ -186,7 +188,7 @@ static void shutdown() {
   @Test
   void testMissingContainer() throws Exception {
     String keyNameR1 = "key2";
-    long containerID = setupRatisKey(keyNameR1,
+    long containerID = setupRatisKey(recon, keyNameR1,
         HddsProtos.ReplicationFactor.ONE);
 
     Pipeline pipeline =
@@ -377,7 +379,7 @@ private static boolean assertReportsMatch(UnHealthyContainerStates state) {
     return true;
   }
 
-  private static long setupRatisKey(String keyName,
+  private static long setupRatisKey(ReconService reconService, String keyName,
       HddsProtos.ReplicationFactor replicationFactor) throws Exception {
     OmKeyInfo omKeyInfo = createTestKey(keyName,
         RatisReplicationConfig.getInstance(replicationFactor));
@@ -395,7 +397,7 @@ private static long setupRatisKey(String keyName,
         reconContainerManager.getContainers());
 
     ReconContainerMetadataManager reconContainerMetadataManager =
-        cluster.getReconServer().getReconContainerMetadataManager();
+        reconService.getReconServer().getReconContainerMetadataManager();
 
     // Verify Recon picked up the new keys and
     // updated its container key mappings.

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconAsPassiveScm.java

@@ -59,14 +59,18 @@
 public class TestReconAsPassiveScm {
   private MiniOzoneCluster cluster;
   private OzoneConfiguration conf;
+  private ReconService recon;
 
   @BeforeEach
   public void init() throws Exception {
     conf = new OzoneConfiguration();
     conf.set(HDDS_CONTAINER_REPORT_INTERVAL, "5s");
     conf.set(HDDS_PIPELINE_REPORT_INTERVAL, "5s");
-    cluster =  MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3)
-        .includeRecon(true).build();
+    recon = new ReconService(conf);
+    cluster =  MiniOzoneCluster.newBuilder(conf)
+        .setNumDatanodes(3)
+        .addService(recon)
+        .build();
     cluster.waitForClusterToBeReady();
     GenericTestUtils.setLogLevel(ReconNodeManager.class, Level.DEBUG);
   }
@@ -82,7 +86,7 @@ public void shutdown() {
   public void testDatanodeRegistrationAndReports() throws Exception {
     ReconStorageContainerManagerFacade reconScm =
         (ReconStorageContainerManagerFacade)
-        cluster.getReconServer().getReconStorageContainerManager();
+        recon.getReconServer().getReconStorageContainerManager();
     StorageContainerManager scm = cluster.getStorageContainerManager();
     PipelineManager reconPipelineManager = reconScm.getPipelineManager();
     PipelineManager scmPipelineManager = scm.getPipelineManager();
@@ -143,7 +147,7 @@ public void testDatanodeRegistrationAndReports() throws Exception {
   @Test
   public void testReconRestart() throws Exception {
     final OzoneStorageContainerManager reconScm =
-            cluster.getReconServer().getReconStorageContainerManager();
+            recon.getReconServer().getReconStorageContainerManager();
     StorageContainerManager scm = cluster.getStorageContainerManager();
 
     // Stop Recon
@@ -155,7 +159,7 @@ public void testReconRestart() throws Exception {
     LambdaTestUtils.await(60000, 5000,
         () -> (reconScm.getScmNodeManager().getAllNodes().size() == 3));
 
-    cluster.stopRecon();
+    recon.stop();
 
     // Create container in SCM.
     ContainerInfo containerInfo =
@@ -180,7 +184,7 @@ public void testReconRestart() throws Exception {
     scmPipelineManager.deletePipeline(pipelineToClose.get().getId());
 
     // Start Recon
-    cluster.startRecon();
+    recon.start(cluster.getConf());
 
     // Verify if Recon has all the nodes on restart (even if heartbeats are
     // not yet received).
@@ -191,7 +195,7 @@ public void testReconRestart() throws Exception {
 
     // Verify Recon picks up new container, close pipeline SCM actions.
     OzoneStorageContainerManager newReconScm =
-        cluster.getReconServer().getReconStorageContainerManager();
+        recon.getReconServer().getReconStorageContainerManager();
     PipelineManager reconPipelineManager = newReconScm.getPipelineManager();
     assertFalse(
         reconPipelineManager.containsPipeline(pipelineToClose.get().getId()));

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerEndpoint.java

@@ -55,15 +55,17 @@ public class TestReconContainerEndpoint {
   private MiniOzoneCluster cluster;
   private OzoneClient client;
   private ObjectStore store;
+  private ReconService recon;
 
   @BeforeEach
   public void init() throws Exception {
     conf = new OzoneConfiguration();
     conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
         OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED);
+    recon = new ReconService(conf);
     cluster = MiniOzoneCluster.newBuilder(conf)
         .setNumDatanodes(3)
-        .includeRecon(true)
+        .addService(recon)
         .build();
     cluster.waitForClusterToBeReady();
     client = cluster.newClient();
@@ -102,7 +104,7 @@ public void testContainerEndpointForFSOLayout() throws Exception {
 
     // Synchronize data from OM to Recon
     OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl)
-        cluster.getReconServer().getOzoneManagerServiceProvider();
+        recon.getReconServer().getOzoneManagerServiceProvider();
     impl.syncDataFromOM();
 
     //Search for the bucket from the bucket table and verify its FSO
@@ -162,7 +164,7 @@ public void testContainerEndpointForOBSBucket() throws Exception {
     writeTestData(volumeName, obsBucketName, obsSingleFileKey, "Hello OBS!");
 
     OzoneManagerServiceProviderImpl impl =
-        (OzoneManagerServiceProviderImpl) cluster.getReconServer()
+        (OzoneManagerServiceProviderImpl) recon.getReconServer()
             .getOzoneManagerServiceProvider();
     impl.syncDataFromOM();
 
@@ -191,19 +193,19 @@ public void testContainerEndpointForOBSBucket() throws Exception {
 
   private Response getContainerEndpointResponse(long containerId) {
     OzoneStorageContainerManager reconSCM =
-        cluster.getReconServer().getReconStorageContainerManager();
+        recon.getReconServer().getReconStorageContainerManager();
     ReconContainerManager reconContainerManager =
         (ReconContainerManager) reconSCM.getContainerManager();
     ContainerHealthSchemaManager containerHealthSchemaManager =
         reconContainerManager.getContainerSchemaManager();
     ReconOMMetadataManager omMetadataManagerInstance =
         (ReconOMMetadataManager)
-            cluster.getReconServer().getOzoneManagerServiceProvider()
+            recon.getReconServer().getOzoneManagerServiceProvider()
                 .getOMMetadataManagerInstance();
     ContainerEndpoint containerEndpoint =
         new ContainerEndpoint(reconSCM, containerHealthSchemaManager,
-            cluster.getReconServer().getReconNamespaceSummaryManager(),
-            cluster.getReconServer().getReconContainerMetadataManager(),
+            recon.getReconServer().getReconNamespaceSummaryManager(),
+            recon.getReconServer().getReconContainerMetadataManager(),
             omMetadataManagerInstance);
     return containerEndpoint.getKeysForContainer(containerId, 10, "");
   }