gmtb/develop: metadata2html batch processing

doc/CCPPtechnical/source/ScientificDocRules.inc

@@ -539,7 +539,7 @@ script from the directory above where the CCPP is installed is:
 
 .. code-block:: fortran
 
-   ccpp/framework/scripts/metadata2html.py -m ccpp/physics/physics/file.meta -o ccpp/physics/physics/docs
+   ./ccpp/framework/scripts/metadata2html.py -m ccpp/physics/physics/file.meta -o ccpp/physics/physics/docs
 
 where ``-m`` is used to specify a file with metadata information and ``-o`` is used to specify
 the directory for output. Note that a single input file (``.meta``) may have more than one CCPP entrypoint
@@ -549,8 +549,20 @@ Note that the ``.meta`` files are supplied with the CCPP Physics, and that there
 each Fortran file that contains one or more CCPP entrypoint schemes. The ``.meta`` files are located in the same
 directory as the scheme Fortran files (``ccpp/physics/physics``).
 
-To generate a complete Scientific Documentation, documentation, script ``/ccpp/framework/scripts/metadata2html.py`` 
-must be run separately for each ``.meta`` file available in ccpp/physics/physics.
+To generate a complete Scientific Documentation, documentation, script ``./ccpp/framework/scripts/metadata2html.py``
+must be run separately for each ``.meta`` file available in ``ccpp/physics/physics``. Alternatively, a batch mode exists
+that converts all metadata files associated with schemes and variable definitions in the CCPP prebuild config:
+
+.. code-block:: fortran
+
+   ./ccpp/framework/scripts/metadata2html.py -c ccpp/config/ccpp_prebuild_config.py
+
+Note that the options ``-c`` and ``-m`` are mutually exclusive, but that one of them is required. Option ``-m`` also requires
+to specify ``-o``, while option ``-c`` will ignore ``-o``. For more information, use
+
+.. code-block:: fortran
+
+   ./ccpp/framework/scripts/metadata2html.py --help
 
 Using Doxygen
 -------------------------------

scripts/ccpp_prebuild.py

@@ -9,9 +9,9 @@
 import re
 import sys
 
-# Local modules
+# CCPP framework imports
 from common import encode_container, decode_container, decode_container_as_dict, execute
-from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE
+from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE
 from common import split_var_name_and_array_reference
 from metadata_parser import merge_dictionaries, parse_scheme_tables, parse_variable_tables
 from mkcap import Cap, CapsMakefile, CapsCMakefile, SchemesMakefile, SchemesCMakefile
@@ -32,15 +32,6 @@
 # BASEDIR is the current directory where this script is executed
 BASEDIR = os.getcwd()
 
-# SCRIPTDIR is the directory where the ccpp_prebuild.py and its Python modules are located
-SCRIPTDIR = os.path.abspath(os.path.split(__file__)[0])
-
-# SRCDIR is the directory where the CCPP framework source code (C, Fortran) is located
-SRCDIR = os.path.abspath(os.path.join(SCRIPTDIR, '..', 'src'))
-
-# Definition of variables (metadata tables) that are provided by CCPP
-CCPP_INTERNAL_VARIABLE_DEFINITON_FILE = os.path.join(SRCDIR, 'ccpp_types.F90')
-
 ###############################################################################
 # Functions and subroutines                                                   #
 ###############################################################################
@@ -99,7 +90,7 @@ def import_config(configfile):
     # Template code in host-model dependent CCPP prebuild config script
     config['ccpp_data_structure']            = ccpp_prebuild_config.CCPP_DATA_STRUCTURE
 
-    # Add model-intependent, CCPP-internal variable definition files
+    # Add model-independent, CCPP-internal variable definition files
     config['variable_definition_files'].append(CCPP_INTERNAL_VARIABLE_DEFINITON_FILE)
 
     # To handle new metadata: import DDT references (if exist)

scripts/common.py

@@ -2,6 +2,7 @@
 
 import keyword
 import logging
+import os
 import re
 import subprocess
 import sys
@@ -14,6 +15,16 @@
 
 CCPP_TYPE = 'ccpp_t'
 
+# SCRIPTDIR is the directory where ccpp_prebuild.py and its Python modules are located
+SCRIPTDIR = os.path.abspath(os.path.split(__file__)[0])
+
+# SRCDIR is the directory where the CCPP framework source code (C, Fortran) is located
+SRCDIR = os.path.abspath(os.path.join(SCRIPTDIR, '..', 'src'))
+
+# Definition of variables (metadata tables) that are provided by CCPP
+CCPP_INTERNAL_VARIABLE_DEFINITON_FILE = os.path.join(SRCDIR, 'ccpp_types.F90')
+
+# List of internal variables provided by the CCPP
 CCPP_INTERNAL_VARIABLES = {
     CCPP_ERROR_FLAG_VARIABLE : 'cdata%errflg',
     CCPP_ERROR_MSG_VARIABLE  : 'cdata%errmsg',

scripts/metadata2html.py

@@ -6,26 +6,84 @@
 import sys
 
 # CCPP framework imports
+from common import CCPP_INTERNAL_VARIABLE_DEFINITON_FILE
 from parse_tools import init_log, set_log_level
 from metadata_table import MetadataHeader
 
+###############################################################################
+# Set up the command line argument parser and other global variables          #
+###############################################################################
+
 parser = argparse.ArgumentParser()
-parser.add_argument('--metafile', '-m', action='store',
-                    help='name of metadata file to convert',
-                    required=True)
+method = parser.add_mutually_exclusive_group(required=True)
+method.add_argument('--config', '-c', action='store',
+                    help='path to CCPP prebuild configuration file')
+method.add_argument('--metafile', '-m', action='store',
+                    help='name of metadata file to convert (requires -o)')
 parser.add_argument('--outputdir', '-o', action='store',
                     help='directory where to write the html files',
-                    required=True)
+                    required='--metafile' in sys.argv or '-m' in sys.argv)
 
-attributes = [ 'local_name', 'standard_name', 'long_name', 'units',
+# List and order of variable attributes to output to HTML
+ATTRIBUTES = [ 'local_name', 'standard_name', 'long_name', 'units',
                'type', 'dimensions', 'kind', 'intent', 'optional' ]
 
+###############################################################################
+# Functions and subroutines                                                   #
+###############################################################################
+
 def parse_arguments():
     """Parse command line arguments."""
     args = parser.parse_args()
+    config = args.config
     filename = args.metafile
     outdir = args.outputdir
-    return (filename, outdir)
+    return (config, filename, outdir)
+
+def import_config(configfile, logger):
+    """Import the configuration from a given configuration file"""
+
+    if not os.path.isfile(configfile):
+        raise Exception("Configuration file {0} not found".format(configfile))
+
+    # Import the host-model specific CCPP prebuild config;
+    # split into path and module name for import
+    configpath = os.path.abspath(os.path.split(configfile)[0])
+    configmodule = os.path.split(configfile)[1].rstrip('.py')
+    sys.path.append(configpath)
+    ccpp_prebuild_config = __import__(configmodule)
+
+    config = {}
+    # Definitions in host-model dependent CCPP prebuild config script
+    config['variable_definition_files'] = ccpp_prebuild_config.VARIABLE_DEFINITION_FILES
+    config['scheme_files']              = ccpp_prebuild_config.SCHEME_FILES
+    # Add model-independent, CCPP-internal variable definition files
+    config['variable_definition_files'].append(CCPP_INTERNAL_VARIABLE_DEFINITON_FILE)
+    # Output directory for converted metadata tables
+    config['metadata_html_output_dir'] = ccpp_prebuild_config.METADATA_HTML_OUTPUT_DIR
+
+    return config
+
+def get_metadata_files_from_config(config, logger):
+    """Create a list of metadata filenames for a CCPP prebuild configuration"""
+    filenames = []
+    for sourcefile in config['variable_definition_files'] + config['scheme_files'].keys():
+        metafile = os.path.splitext(sourcefile)[0]+'.meta'
+        if os.path.isfile(metafile):
+            filenames.append(metafile)
+        else:
+            # DH* Warn for now, raise exception later when
+            # old metadata format is no longer supported
+            logger.warn("Metadata file {} for source file {} not found, assuming old metadata format".format(
+                                                                                       metafile, sourcefile))
+    return filenames
+
+def get_output_directory_from_config(config, logger):
+    """Return the html output directory for a CCPP prebuild configuration"""
+    outdir = config['metadata_html_output_dir']
+    if not os.path.isdir(outdir):
+        raise Exception("Output directory {} for converted metadata tables does not exist".format(outdir))
+    return outdir
 
 def convert_to_html(filename_in, outdir, logger):
     """Convert a metadata file into html (one html file for each table)"""
@@ -34,7 +92,7 @@ def convert_to_html(filename_in, outdir, logger):
     logger.info("Converting file {} to HTML".format(filename_in))
     metadata_headers = MetadataHeader.parse_metadata_file(filename_in)
     for metadata_header in metadata_headers:
-        filename_out = metadata_header.to_html(outdir, attributes)
+        filename_out = metadata_header.to_html(outdir, ATTRIBUTES)
         if filename_out:
             logger.info("  ... wrote {}".format(filename_out))
 
@@ -43,8 +101,15 @@ def main():
     logger = init_log('metadata2html')
     set_log_level(logger, logging.INFO)
     # Convert metadata file
-    (filename, outdir) = parse_arguments()
-    convert_to_html(filename, outdir, logger)
+    (configfile, filename, outdir) = parse_arguments()
+    if configfile:
+        config = import_config(configfile, logger)
+        filenames = get_metadata_files_from_config(config, logger)
+        outdir = get_output_directory_from_config(config, logger)
+        for filename in filenames:
+            convert_to_html(filename, outdir, logger)
+    else:
+        convert_to_html(filename, outdir, logger)
 
 if __name__ == '__main__':
     main()