HDDS-4046. Extensible subcommands for CLI applications

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java

@@ -19,13 +19,15 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.ServiceLoader;
 import java.util.concurrent.Callable;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 
 import com.google.common.annotations.VisibleForTesting;
 import picocli.CommandLine;
+import picocli.CommandLine.Command;
 import picocli.CommandLine.ExecutionException;
 import picocli.CommandLine.Model.CommandSpec;
 import picocli.CommandLine.Option;
@@ -52,6 +54,25 @@ public GenericCli() {
     cmd = new CommandLine(this);
   }
 
+  public GenericCli(Class<?> type) {
+    this();
+    addSubcommands(getCmd(), type);
+  }
+
+  private void addSubcommands(CommandLine cli, Class<?> type) {
+    ServiceLoader<SubcommandWithParent> registeredSubcommands =
+        ServiceLoader.load(SubcommandWithParent.class);
+    for (SubcommandWithParent subcommand : registeredSubcommands) {
+      if (subcommand.getParentType().equals(type)) {
+        final Command commandAnnotation =
+            subcommand.getClass().getAnnotation(Command.class);
+        CommandLine subcommandCommandLine = new CommandLine(subcommand);
+        addSubcommands(subcommandCommandLine, subcommand.getClass());
+        cli.addSubcommand(commandAnnotation.name(), subcommandCommandLine);
+      }
+    }
+  }
+
   /**
    * Handle the error when subcommand is required but not set.
    */

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/SubcommandWithParent.java

@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.cli;
+
+/**
+ * Defineds parent command for SPI based subcommand registration.
+ */
+public interface SubcommandWithParent {
+
+  /**
+   * Java type of the parent command.
+   */
+  Class<?> getParentType();
+
+}

hadoop-ozone/insight/src/main/java/org/apache/hadoop/ozone/insight/Insight.java

@@ -34,6 +34,10 @@
     mixinStandardHelpOptions = true)
 public class Insight extends GenericCli {
 
+  public Insight() {
+    super(Insight.class);
+  }
+
   public static void main(String[] args) throws Exception {
     new Insight().run(args);
   }

hadoop-ozone/tools/pom.xml

@@ -105,6 +105,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-s3</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.kohsuke.metainf-services</groupId>
+      <artifactId>metainf-services</artifactId>
+      <version>1.1</version>
+      <optional>true</optional>
+    </dependency>
     <dependency>
       <groupId>com.github.spotbugs</groupId>
       <artifactId>spotbugs</artifactId>

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java

@@ -34,7 +34,6 @@
 import org.apache.hadoop.hdds.scm.cli.datanode.DatanodeCommands;
 import org.apache.hadoop.hdds.scm.cli.pipeline.PipelineCommands;
 import org.apache.hadoop.hdds.scm.client.ScmClient;
-import org.apache.hadoop.ozone.admin.om.OMAdmin;
 import org.apache.hadoop.util.NativeCodeLoader;
 
 import org.apache.commons.lang3.StringUtils;
@@ -55,7 +54,6 @@
     description = "Developer tools for Ozone Admin operations",
     versionProvider = HddsVersionProvider.class,
     subcommands = {
-        OMAdmin.class,
         SafeModeCommands.class,
         ContainerCommands.class,
         PipelineCommands.class,
@@ -71,6 +69,10 @@ public class OzoneAdmin extends GenericCli implements WithScmClient {
   @Option(names = {"--scm"}, description = "The destination scm (host:port)")
   private String scm = "";
 
+  public OzoneAdmin() {
+    super(OzoneAdmin.class);
+  }
+
   public OzoneConfiguration getOzoneConf() {
     if (ozoneConf == null) {
       ozoneConf = createOzoneConfiguration();

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java

@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.hdds.cli.GenericCli;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.cli.SubcommandWithParent;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.RPC;
@@ -35,6 +36,7 @@
 
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
 import org.apache.ratis.protocol.ClientId;
+import org.kohsuke.MetaInfServices;
 import picocli.CommandLine;
 import picocli.CommandLine.Model.CommandSpec;
 import picocli.CommandLine.Spec;
@@ -50,7 +52,8 @@
     subcommands = {
         GetServiceRolesSubcommand.class
     })
-public class OMAdmin extends GenericCli {
+@MetaInfServices(SubcommandWithParent.class)
+public class OMAdmin extends GenericCli implements SubcommandWithParent {
 
   @CommandLine.ParentCommand
   private OzoneAdmin parent;
@@ -102,4 +105,9 @@ public OzoneManagerProtocolClientSideTranslatorPB createOmClient(
           conf.getTrimmedStringCollection(OZONE_OM_SERVICE_IDS_KEY));
     }
   }
+
+  @Override
+  public Class<?> getParentType() {
+    return OzoneAdmin.class;
+  }
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkKeyHandler.java

@@ -23,6 +23,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.hadoop.hdds.cli.SubcommandWithParent;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.scm.XceiverClientManager;
@@ -49,6 +50,7 @@
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import org.apache.ratis.protocol.ClientId;
+import org.kohsuke.MetaInfServices;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Parameters;
 
@@ -58,7 +60,9 @@
 @Command(name = "chunkinfo",
         description = "returns chunk location"
                 + " information about an existing key")
-public class ChunkKeyHandler  extends KeyHandler {
+@MetaInfServices(SubcommandWithParent.class)
+public class ChunkKeyHandler extends KeyHandler implements
+    SubcommandWithParent {
 
   @Parameters(arity = "1..1", description = "key to be located")
     private String uri;
@@ -170,5 +174,8 @@ protected void execute(OzoneClient client, OzoneAddress address)
     System.out.println(prettyJson);
   }
 
-
+  @Override
+  public Class<?> getParentType() {
+    return OzoneDebug.class;
+  }
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java

@@ -18,27 +18,39 @@
 
 package org.apache.hadoop.ozone.debug;
 
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
-import org.apache.hadoop.hdds.utils.db.DBDefinition;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.rocksdb.*;
-import picocli.CommandLine;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
 import java.util.concurrent.Callable;
 
+import org.apache.hadoop.hdds.cli.SubcommandWithParent;
+import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
+import org.apache.hadoop.hdds.utils.db.DBDefinition;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.kohsuke.MetaInfServices;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.Options;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksIterator;
+import picocli.CommandLine;
+
 /**
  * Parser for scm.db file.
  */
 @CommandLine.Command(
         name = "scan",
         description = "Parse specified metadataTable"
 )
-public class DBScanner implements Callable<Void> {
+@MetaInfServices(SubcommandWithParent.class)
+public class DBScanner implements Callable<Void>, SubcommandWithParent {
 
   @CommandLine.Option(names = {"--column_family"},
             description = "Table name")
@@ -145,4 +157,9 @@ private String removeTrailingSlashIfNeeded(String dbPath) {
     }
     return dbPath;
   }
+
+  @Override
+  public Class<?> getParentType() {
+    return RDBParser.class;
+  }
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ListTables.java

@@ -18,14 +18,17 @@
 
 package org.apache.hadoop.ozone.debug;
 
-import org.rocksdb.Options;
-import org.rocksdb.RocksDB;
-import picocli.CommandLine;
-
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.concurrent.Callable;
 
+import org.apache.hadoop.hdds.cli.SubcommandWithParent;
+
+import org.kohsuke.MetaInfServices;
+import org.rocksdb.Options;
+import org.rocksdb.RocksDB;
+import picocli.CommandLine;
+
 /**
  * List all column Families/Tables in db.
  */
@@ -34,7 +37,8 @@
         aliases = "ls",
         description = "list all column families in db."
 )
-public class ListTables implements Callable<Void> {
+@MetaInfServices(SubcommandWithParent.class)
+public class ListTables implements Callable<Void>, SubcommandWithParent {
 
   @CommandLine.ParentCommand
   private RDBParser parent;
@@ -48,4 +52,9 @@ public Void call() throws Exception {
     }
     return null;
   }
+
+  @Override
+  public Class<?> getParentType() {
+    return RDBParser.class;
+  }
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/OzoneDebug.java

@@ -20,7 +20,7 @@
 
 import org.apache.hadoop.hdds.cli.GenericCli;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.ozone.segmentparser.RatisLogParser;
+
 import picocli.CommandLine;
 
 /**
@@ -29,14 +29,13 @@
 @CommandLine.Command(name = "ozone debug",
         description = "Developer tools for Ozone Debug operations",
         versionProvider = HddsVersionProvider.class,
-        subcommands = {
-                ChunkKeyHandler.class,
-                RatisLogParser.class,
-                RDBParser.class
-        },
         mixinStandardHelpOptions = true)
 public class OzoneDebug extends GenericCli {
 
+  public OzoneDebug() {
+    super(OzoneDebug.class);
+  }
+
   /**
      * Main for the Ozone Debug shell Command handling.
      *

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/RDBParser.java

@@ -18,20 +18,27 @@
 
 package org.apache.hadoop.ozone.debug;
 
+import java.util.concurrent.Callable;
+
 import org.apache.hadoop.hdds.cli.GenericCli;
+import org.apache.hadoop.hdds.cli.SubcommandWithParent;
+
+import org.kohsuke.MetaInfServices;
 import picocli.CommandLine;
+import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.Spec;
 
 /**
  * Tool that parses rocksdb file.
  */
 @CommandLine.Command(
         name = "ldb",
-        description = "Parse rocksdb file content",
-        subcommands = {
-                DBScanner.class,
-                ListTables.class
-        })
-public class RDBParser extends GenericCli {
+        description = "Parse rocksdb file content")
+@MetaInfServices(SubcommandWithParent.class)
+public class RDBParser implements Callable<Void>, SubcommandWithParent {
+
+  @Spec
+  private CommandSpec spec;
 
   @CommandLine.Option(names = {"--db"},
             description = "Database File Path")
@@ -42,7 +49,13 @@ public String getDbPath() {
   }
 
   @Override
-  public void execute(String[] argv) {
-    new RDBParser().run(argv);
+  public Class<?> getParentType() {
+    return OzoneDebug.class;
+  }
+
+  @Override
+  public Void call() throws Exception {
+    GenericCli.missingSubcommand(spec);
+    return null;
   }
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java

@@ -58,6 +58,10 @@ public class Freon extends GenericCli {
 
   public static final Logger LOG = LoggerFactory.getLogger(Freon.class);
 
+  public Freon() {
+    super(Freon.class);
+  }
+
   @Option(names = "--server",
       description = "Enable internal http server to provide metric "
           + "and profile endpoint")