HDDS-3689. Add various profiles to MiniOzoneChaosCluster to run different modes.

hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/bin/start-chaos.sh

@@ -46,7 +46,7 @@ echo "logging chaos logs and heapdump to ${logfiledirectory}"
 echo "Starting MiniOzoneChaosCluster with ${MVN_OPTS}"
 mvn clean install -DskipTests > "${compilefilename}" 2>&1
 mvn exec:java \
-  -Dexec.mainClass="org.apache.hadoop.ozone.TestMiniChaosOzoneCluster" \
+  -Dexec.mainClass="org.apache.hadoop.ozone.OzoneChaosCluster" \
   -Dexec.classpathScope=test \
   -Dchaoslogfilename=${chaosfilename} \
   -Dproblemlogfilename=${problemfilename} \

hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java

@@ -20,7 +20,6 @@
 
 import java.io.IOException;
 import java.time.Duration;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
@@ -94,7 +93,7 @@ public static FailureService of(String serviceName) {
   public MiniOzoneChaosCluster(OzoneConfiguration conf,
       List<OzoneManager> ozoneManagers, StorageContainerManager scm,
       List<HddsDatanodeService> hddsDatanodes, String omServiceID,
-      List<Class<? extends Failures>> clazzes) {
+      Set<Class<? extends Failures>> clazzes) {
     super(conf, ozoneManagers, scm, hddsDatanodes, omServiceID);
     this.numDatanodes = getHddsDatanodes().size();
     this.numOzoneManagers = ozoneManagers.size();
@@ -150,7 +149,7 @@ public void waitForClusterToBeReady()
    */
   public static class Builder extends MiniOzoneHAClusterImpl.Builder {
 
-    private final List<Class<? extends Failures>> clazzes = new ArrayList<>();
+    private final Set<Class<? extends Failures>> clazzes = new HashSet<>();
 
     /**
      * Creates a new Builder.

hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java

@@ -23,12 +23,14 @@
 import org.apache.hadoop.ozone.loadgenerators.DataBuffer;
 import org.apache.hadoop.ozone.loadgenerators.LoadExecutors;
 import org.apache.hadoop.ozone.loadgenerators.LoadGenerator;
-import org.apache.hadoop.ozone.utils.LoadBucket;
+import org.apache.hadoop.ozone.loadgenerators.LoadBucket;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -48,7 +50,7 @@ public class MiniOzoneLoadGenerator {
 
   MiniOzoneLoadGenerator(OzoneVolume volume, int numThreads,
       int numBuffers, OzoneConfiguration conf, String omServiceId,
-      List<Class<? extends LoadGenerator>> loadGenratorClazzes)
+      Set<Class<? extends LoadGenerator>> loadGenratorClazzes)
       throws Exception {
     DataBuffer buffer = new DataBuffer(numBuffers);
     loadGenerators = new ArrayList<>();
@@ -92,7 +94,7 @@ void shutdownLoadGenerator() {
    * Builder to create Ozone load generator.
    */
   public static class Builder {
-    private List<Class<? extends LoadGenerator>> clazzes = new ArrayList<>();
+    private Set<Class<? extends LoadGenerator>> clazzes = new HashSet<>();
     private String omServiceId;
     private OzoneConfiguration conf;
     private int numBuffers;

hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/OzoneChaosCluster.java

@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import org.apache.hadoop.hdds.cli.GenericCli;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import picocli.CommandLine;
+
+/**
+ * Main driver class for Ozone Chaos Cluster
+ * This has multiple sub implementations of chaos cluster as options.
+ */
+@CommandLine.Command(
+    name = "chaos",
+    description = "Starts IO with MiniOzoneChaosCluster",
+    subcommands = {
+        TestAllMiniChaosOzoneCluster.class,
+        TestDatanodeMiniChaosOzoneCluster.class,
+        TestOzoneManagerMiniChaosOzoneCluster.class
+    },
+    versionProvider = HddsVersionProvider.class,
+    mixinStandardHelpOptions = true)
+public class OzoneChaosCluster extends GenericCli {
+  @Override
+  public void execute(String[] argv) {
+    super.execute(argv);
+  }
+
+  public static void main(String[] args) {
+    new OzoneChaosCluster().run(args);
+  }
+}

hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestAllMiniChaosOzoneCluster.java

@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.ozone.failure.Failures;
+import org.apache.hadoop.ozone.loadgenerators.*;
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Command line utility to parse and dump a datanode ratis segment file.
+ */
+@CommandLine.Command(
+    name = "all",
+    description = "run chaos cluster across all daemons",
+    mixinStandardHelpOptions = true,
+    versionProvider = HddsVersionProvider.class)
+public class TestAllMiniChaosOzoneCluster extends TestMiniChaosOzoneCluster
+    implements Callable<Void> {
+
+  @CommandLine.ParentCommand
+  private OzoneChaosCluster chaosCluster;
+
+  @Override
+  public Void call() throws Exception {
+    setNumOzoneManagers(3, true);
+
+    LoadGenerator.getClassList().forEach(
+        TestMiniChaosOzoneCluster::addLoadClasses);
+    Failures.getClassList().forEach(
+        TestMiniChaosOzoneCluster::addFailureClasses);
+
+    startChaosCluster();
+
+    return null;
+  }
+
+}

hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestDatanodeMiniChaosOzoneCluster.java

@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.ozone.failure.Failures;
+import org.apache.hadoop.ozone.loadgenerators.RandomLoadGenerator;
+import org.apache.hadoop.ozone.loadgenerators.AgedLoadGenerator;
+
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Test Datanode with Chaos.
+ */
+@CommandLine.Command(
+    name = "datanode",
+    description = "run chaos cluster across Ozone Datanodes",
+    mixinStandardHelpOptions = true,
+    versionProvider = HddsVersionProvider.class)
+public class TestDatanodeMiniChaosOzoneCluster extends
+    TestMiniChaosOzoneCluster implements Callable<Void> {
+
+  @Override
+  public Void call() throws Exception {
+    addLoadClasses(RandomLoadGenerator.class);
+    addLoadClasses(AgedLoadGenerator.class);
+
+    addFailureClasses(Failures.DatanodeStartStopFailure.class);
+    addFailureClasses(Failures.DatanodeRestartFailure.class);
+
+    startChaosCluster();
+
+    return null;
+  }
+
+}

hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java

@@ -22,24 +22,17 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.client.ObjectStore;
 import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.MiniOzoneChaosCluster.FailureService;
 import org.apache.hadoop.ozone.failure.Failures;
-import org.apache.hadoop.ozone.loadgenerators.RandomLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.ReadOnlyLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.FilesystemLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.AgedLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.AgedDirLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.RandomDirLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.NestedDirLoadGenerator;
+import org.apache.hadoop.ozone.loadgenerators.LoadGenerator;
 import org.junit.BeforeClass;
 import org.junit.AfterClass;
 import org.junit.Ignore;
 import org.junit.Test;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -49,8 +42,12 @@
 @Command(description = "Starts IO with MiniOzoneChaosCluster",
     name = "chaos", mixinStandardHelpOptions = true)
 public class TestMiniChaosOzoneCluster extends GenericCli {
-  static final Logger LOG =
-      LoggerFactory.getLogger(TestMiniChaosOzoneCluster.class);
+
+  private static List<Class<? extends Failures>> failureClasses
+      = new ArrayList<>();
+
+  private static List<Class<? extends LoadGenerator>> loadClasses
+      = new ArrayList<>();
 
   @Option(names = {"-d", "--num-datanodes", "--numDatanodes"},
       description = "num of datanodes. Full name --numDatanodes will be" +
@@ -62,12 +59,6 @@ public class TestMiniChaosOzoneCluster extends GenericCli {
           " be removed in later versions.")
   private static int numOzoneManagers = 1;
 
-  @Option(names = {"-s", "--failure-service", "--failureService"},
-      description = "service (datanode or ozoneManager) to test chaos on. " +
-          "Full --failureService name will be removed in later versions.",
-      defaultValue = "datanode")
-  private static String failureService = "datanode";
-
   @Option(names = {"-t", "--num-threads", "--numThreads"},
       description = "num of IO threads. Full name --numThreads will be" +
           " removed in later versions.")
@@ -96,62 +87,61 @@ public class TestMiniChaosOzoneCluster extends GenericCli {
   private static MiniOzoneChaosCluster cluster;
   private static MiniOzoneLoadGenerator loadGenerator;
 
+  private static String omServiceId = null;
+
   private static final String OM_SERVICE_ID = "ozoneChaosTest";
 
   @BeforeClass
   public static void init() throws Exception {
     OzoneConfiguration configuration = new OzoneConfiguration();
-    FailureService service = FailureService.of(failureService);
-    String omServiceID;
 
-    MiniOzoneChaosCluster.Builder builder =
+    MiniOzoneChaosCluster.Builder chaosBuilder =
         new MiniOzoneChaosCluster.Builder(configuration);
 
-    switch (service) {
-    case DATANODE:
-      omServiceID = null;
-      builder
-          .addFailures(Failures.DatanodeRestartFailure.class)
-          .addFailures(Failures.DatanodeStartStopFailure.class);
-      break;
-    case OZONE_MANAGER:
-      omServiceID = OM_SERVICE_ID;
-      builder
-          .addFailures(Failures.OzoneManagerStartStopFailure.class)
-          .addFailures(Failures.OzoneManagerRestartFailure.class);
-      break;
-    default:
-      throw new IllegalArgumentException();
-    }
-
-    builder
+    chaosBuilder
         .setNumDatanodes(numDatanodes)
         .setNumOzoneManagers(numOzoneManagers)
-        .setOMServiceID(omServiceID)
+        .setOMServiceID(omServiceId)
         .setNumDataVolumes(numDataVolumes);
+    failureClasses.forEach(chaosBuilder::addFailures);
 
-    cluster = builder.build();
+    cluster = chaosBuilder.build();
     cluster.waitForClusterToBeReady();
 
     String volumeName = RandomStringUtils.randomAlphabetic(10).toLowerCase();
     ObjectStore store = cluster.getRpcClient().getObjectStore();
     store.createVolume(volumeName);
     OzoneVolume volume = store.getVolume(volumeName);
 
-    loadGenerator = new MiniOzoneLoadGenerator.Builder()
+    MiniOzoneLoadGenerator.Builder loadBuilder =
+        new MiniOzoneLoadGenerator.Builder()
         .setVolume(volume)
         .setConf(configuration)
         .setNumBuffers(numBuffers)
         .setNumThreads(numThreads)
-        .setOMServiceId(omServiceID)
-        .addLoadGenerator(RandomLoadGenerator.class)
-        .addLoadGenerator(AgedLoadGenerator.class)
-        .addLoadGenerator(FilesystemLoadGenerator.class)
-        .addLoadGenerator(ReadOnlyLoadGenerator.class)
-        .addLoadGenerator(RandomDirLoadGenerator.class)
-        .addLoadGenerator(AgedDirLoadGenerator.class)
-        .addLoadGenerator(NestedDirLoadGenerator.class)
-        .build();
+        .setOMServiceId(omServiceId);
+    loadClasses.forEach(loadBuilder::addLoadGenerator);
+    loadGenerator = loadBuilder.build();
+  }
+
+  static void addFailureClasses(Class<? extends Failures> clz) {
+    failureClasses.add(clz);
+  }
+
+  static void addLoadClasses(Class<? extends LoadGenerator> clz) {
+    loadClasses.add(clz);
+  }
+
+  static void setNumDatanodes(int nDns) {
+    numDatanodes = nDns;
+  }
+
+  static void setNumOzoneManagers(int nOms, boolean enableHA) {
+
+    if (nOms > 1 || enableHA) {
+      omServiceId = OM_SERVICE_ID;
+    }
+    numOzoneManagers = nOms;
   }
 
   /**
@@ -168,20 +158,14 @@ public static void shutdown() {
     }
   }
 
-  @Override
-  public Void call() throws Exception {
+  public void startChaosCluster() throws Exception {
     try {
       init();
       cluster.startChaos(failureInterval, failureInterval, TimeUnit.SECONDS);
       loadGenerator.startIO(numMinutes, TimeUnit.MINUTES);
     } finally {
       shutdown();
     }
-    return null;
-  }
-
-  public static void main(String... args) {
-    new TestMiniChaosOzoneCluster().run(args);
   }
 
   @Test