cli: migrate from Bash to Python

* Convert the main Rose script from Bash to Python.
* Register Python commands as entry points, hardcode Bash commands
  in the main Rose script.
* Update centralised option help text to the more up to date
  help text in the sub commands.

Author: oliver-sanders

ACKNOWLEDGEMENT.md

@@ -36,3 +36,7 @@ metomi/rosie/lib/html/static/js/livestamp.min.js
 metomi/rosie/lib/html/static/js/moment.min.js
 * Unmodified external software library released under the MIT license.
   See <http://momentjs.com/>
+
+metomi/rose/opt_parse.py
+* Contains modification of the Python 3.7 optparse source released under
+  the PSF license.

bin/rose-mpi-launch

@@ -125,6 +125,10 @@ ROSE_COMMAND=
 ROSE_COMMAND_FILE=
 while (($# > 0)); do
     case $1 in
+    --help)
+        rose_help
+        exit
+        :;;
     --command-file=*)
         ROSE_COMMAND_FILE=${1#*=}
         shift 1

bin/rose-tutorial

@@ -28,9 +28,16 @@
 #
 # DESCRIPTION
 #     Make a copy of one of the tutorial SUITE in the cylc-run directory.
-#
 #-------------------------------------------------------------------------------
 set -eu
+
+if [[ $1 == '--help' ]]; then
+    # shellcheck source=metomi/rose/etc/lib/bash/rose_usage
+    . "$(rose resource lib/bash/rose_usage)"
+    rose_help
+    exit 0
+fi
+
 mkdir -p "$HOME/cylc-run"
 
 usage () {

metomi/rose/app_run.py

@@ -511,10 +511,44 @@ def _command(self, conf_tree, opts, args):
 
 def main():
     """Launcher for the CLI."""
-    opt_parser = RoseOptionParser()
+    opt_parser = RoseOptionParser(
+        usage='%prog [OPTIONS] [--] [COMMAND ...]',
+        description='''
+Run an application according to its configuration.
+
+May run a builtin application (if the `mode` setting in the configuration
+specifies the name of a builtin application) or a command.
+
+Determine the command to run in this order:
+
+1. If `COMMAND` is specified, invoke the command.
+2. If the `--command-key=KEY` option is defined, invoke the command
+   specified in `[command]KEY`.
+3. If the `ROSE_APP_COMMAND_KEY` environment variable is set, the command
+   specified in the `[command]KEY` setting in the application
+   configuration whose `KEY` matches it is used.
+4. If the environment variable `ROSE_TASK_NAME` is defined and a setting
+   in the `[command]` section has a key matching the value of the
+   environment variable, then the value of the setting is used as the
+   command.
+5. Invoke the command specified in `[command]default`.
+        ''',
+        epilog='''
+ENVIRONMENT VARIABLES
+    optional ROSE_APP_COMMAND_KEY
+        Switch to a particular command specified in `[command]KEY`.
+    optional ROSE_APP_MODE
+        Specifies a builtin application to run.
+    optional ROSE_APP_OPT_CONF_KEYS
+        Each `KEY` in this space delimited list switches on an optional
+        configuration. The configurations are applied first-to-last.
+    optional ROSE_FILE_INSTALL_ROOT
+        If specified, change to the specified directory to install files.
+        '''
+    )
     option_keys = AppRunner.OPTIONS
     opt_parser.add_my_options(*option_keys)
-    opts, args = opt_parser.parse_args(sys.argv[1:])
+    opts, args = opt_parser.parse_args()
     event_handler = Reporter(opts.verbosity - opts.quietness)
     runner = AppRunner(event_handler)
     try:

metomi/rose/config_cli.py

@@ -61,7 +61,56 @@ def get_meta_path(root_node, rel_path=None, meta_key=False):
 
 def main():
     """Implement the "rose config" command."""
-    opt_parser = RoseOptionParser()
+    opt_parser = RoseOptionParser(
+        description='''
+Parse and print rose configuration files.
+
+With no option and no argument, print the rose site + user configuration.
+
+EXAMPLES
+    # Print the value of OPTION in SECTION.
+    rose config SECTION OPTION
+
+    # Print the value of OPTION in SECTION in FILE.
+    rose config --file=FILE SECTION OPTION
+
+    # Print the value of OPTION in SECTION if exists, or VALUE otherwise.
+    rose config --default=VALUE SECTION OPTION
+
+    # Print the OPTION=VALUE pairs in SECTION.
+    rose config SECTION
+
+    # Print the value of a top level OPTION.
+    rose config OPTION
+
+    # Print the OPTION keys in SECTION.
+    rose config --keys SECTION
+
+    # Print the SECTION keys.
+    rose config --keys
+
+    # Exit with 0 if OPTION exists in SECTION, or 1 otherwise.
+    rose config -q SECTION OPTION
+
+    # Exit with 0 if SECTION exists, or 1 otherwise.
+    rose config -q SECTION
+
+    # Combine the configurations in FILE1 and FILE2, and dump the result.
+    rose config --file=FILE1 --file=FILE2
+
+    # Print the value of OPTION in SECTION of the metadata associated with
+    # the specified config FILE
+    rose config --file=FILE --meta SECTION OPTION
+
+    # Print the value of a specified metadata KEY
+    rose config --meta-key=KEY
+        ''',
+        epilog='''
+ENVIRONMENT VARIABLES
+    optional ROSE_META_PATH
+        Prepend `$ROSE_META_PATH` to the metadata search path.
+        '''
+    )
     opt_parser.add_my_options(
         "default",
         "env_var_process_mode",
@@ -73,9 +122,17 @@ def main():
         "no_opts",
         "print_conf_mode",
     )
+    # the quietness argument is non-standard for this command
+    opt_parser.modify_option(
+        'quietness',
+        help=(
+            "Exit with 0 if the specified `SECTION` and/or `OPTION` exist in"
+            " the configuration, or 1 otherwise."
+        ),
+    )
+
     opts, args = opt_parser.parse_args()
     report = Reporter(opts.verbosity - opts.quietness)
-
     metomi.rose.macro.add_meta_paths()
 
     if opts.meta_key:

metomi/rose/config_diff.py

@@ -135,7 +135,73 @@ def format_metadata_as_text(metadata, only_these_options=None):
 
 def main():
     """Implement the "rose config-diff" command."""
-    opt_parser = metomi.rose.opt_parse.RoseOptionParser()
+    opt_parser = metomi.rose.opt_parse.RoseOptionParser(
+        description='''
+Display the metadata-annotated difference between two Rose config files.
+
+EXAMPLES
+    # Display the metadata-annotated diff between two Rose config files.
+    rose config-diff FILE1 FILE2
+
+    # Display the metadata-annotated diff between two Rose config dirs.
+    rose config-diff DIR1 DIR2
+
+    # Display the diff, ignoring particular setting patterns
+    rose config-diff --ignore=namelist:foo FILE1 FILE2
+
+    # Display the diff with a particular diff tool
+    rose config-diff --diff-tool=kdiff3 FILE1 FILE2
+
+    # Display the diff with some diff tool specific options/arguments
+    rose config-diff FILE1 FILE2 -- [DIFF_OPTIONS] [DIFF_ARGUMENTS]
+        ''',
+        epilog='''
+ENVIRONMENT VARIABLES
+    optional ROSE_META_PATH
+        Prepend `$ROSE_META_PATH` to the metadata search path.
+
+ARGUMENTS
+    PATH1, PATH2
+        Two Rose configuration files or directories to compare.
+        If the path is a directory, look underneath for a Rose configuration
+        file. '-' for `PATH1` or `PATH2` denotes read in from standard input.
+    --
+        Options and arguments after a `--` token are passed directly to the
+        diff tool.
+
+CONFIGURATION
+    [external]diff-tool, [external]gdiff-tool
+       You can override the default non-graphical and graphical diff tools
+       by setting e.g::
+
+          [external]
+          diff-tool=diff3
+          gdiff-tool=kompare
+
+       in your site or user Rose configuration (`rose.conf`).
+
+    [rose-config-diff]properties, [rose-config-diff]ignore{...}
+       You can override the default metadata properties to display by
+       setting e.g::
+
+          [rose-config-diff]
+          properties=title,ns,description,help
+
+       in your site or user Rose configuration (`rose.conf`).
+       You can also set shorthand ignore patterns by setting e.g.::
+
+         [rose-config-diff]
+         ignore{foo}=namelist:bar,namelist:baz
+
+       in the same location. This will allow you to run::
+
+          rose config-diff --ignore=foo ...
+
+       instead of::
+
+          rose config-diff --ignore=namelist:bar --ignore=namelist:baz ...
+        '''
+    )
     opt_parser.add_my_options(
         "diff_tool",
         "graphical",

metomi/rose/config_dump.py

@@ -39,7 +39,22 @@ def __str__(self):
 
 def main():
     """Implement the "rose config-dump" command."""
-    opt_parser = RoseOptionParser()
+    opt_parser = RoseOptionParser(
+        description='''
+Re-dump Rose configuration files in the common format.
+
+Load and dump `"rose-*.conf"` files in place. Apply format-specific
+pretty-printing.
+
+By default, it recursively loads and dumps all `rose-*.conf` files in the
+current working directory.
+
+EXAMPLES
+    rose config-dump
+    rose config-dump -C /path/to/conf/dir
+    rose config-dump -f /path/to/file1 -f /path/to/file2
+        '''
+    )
     opt_parser.add_my_options("conf_dir", "files", "no_pretty_mode")
     opts = opt_parser.parse_args()[0]
     verbosity = opts.verbosity - opts.quietness

bin/rose-date metomi/rose/date_cli.py

@@ -1,7 +1,4 @@
-#!/usr/bin/env python3
-# -----------------------------------------------------------------------------
 # Copyright (C) British Crown (Met Office) & Contributors.
-#
 # This file is part of Rose, a framework for meteorological suites.
 #
 # Rose is free software: you can redistribute it and/or modify
@@ -187,6 +184,13 @@ def main():
         file=sys.stderr
     )
 
+    if '--help' in sys.argv:
+        print('\n' + __doc__)
+        sys.exit()
+
+    # strip the rose end of the CLI args
+    sys.argv = sys.argv[2:]
+
     # Handle Legacy Rose-date -c functionality
     if '-c' in sys.argv or '--use-task-cycle-time' in sys.argv:
         if os.getenv('ROSE_TASK_CYCLE_TIME'):
@@ -206,10 +210,3 @@ def main():
         os.environ['ISODATETIMECALENDAR'] = os.getenv('ROSE_CYCLING_MODE')
 
     sys.exit(iso_main())
-
-
-if __name__ == '__main__':
-    if '--help' in sys.argv:
-        print(__doc__)
-    else:
-        main()

metomi/rose/env_cat.py

@@ -25,8 +25,34 @@
 
 def main():
     """Implement "rose env-cat"."""
-    opt_parser = RoseOptionParser()
+    opt_parser = RoseOptionParser(
+        usage='rose env-cat [OPTIONS] [FILE ...]',
+        description=r'''
+Substitute environment variables in input files and print.
+
+If no argument is specified, read from STDIN. One `FILE` argument may be
+`-`, which means read from STDIN.
+
+In `match-mode=default`, the command will look for `$NAME` or `${NAME}`
+syntax and substitute them with the value of the environment variable
+`NAME`. A backslash in front of the syntax, e.g. `\$NAME` or `\${NAME}`
+will escape the substitution.
+
+In `match-mode=brace`, the command will look for `${NAME}` syntax only.
+
+EXAMPLES
+    rose env-cat [OPTIONS] [FILE ...]
+        '''
+    )
     opt_parser.add_my_options("match_mode", "output_file", "unbound")
+    opt_parser.modify_option(
+        'output_file',
+        help=(
+            "Specify an output file."
+            "\nIf no output file is specified or if `FILE`"
+            "is `-`, write output to STDOUT."
+        ),
+    )
     opts, args = opt_parser.parse_args()
     if not args:
         args = ["-"]

metomi/rose/etc/lib/bash/rose_usage

@@ -29,6 +29,16 @@
 #     formatted like this one. If CODE is specified, exits the current shell
 #     with it. If CODE is 0, prints the usage to STDOUT. If code is not 0,
 #     prints the usage to STDERR.
+#
+# NAME
+#     rose_help
+#
+# SYNOPSIS
+#     rose_help
+#
+# DESCRIPTION
+#     Prints help information extracted from the header of $0.
+#     Suitable for use with the --help option.
 #-------------------------------------------------------------------------------
 function rose_usage() {
     local CODE=${1:-}
@@ -50,3 +60,7 @@ function rose_usage() {
         exit "$CODE"
     fi
 }
+
+function rose_help() {
+    sed -n 's/^# //; s/^#//; /NAME/,/------/p' "$0" | head -n -1
+}