From d9f538e2f19aae3c0f44c7e80cea10d31ff47c5d Mon Sep 17 00:00:00 2001 From: climbfuji Date: Thu, 12 Sep 2019 08:33:08 -0600 Subject: [PATCH 1/3] scripts/common.py, scripts/ccpp_prebuild.py: move definition of CCPP_INTERNAL_VARIABLE_DEFINITON_FILE to common.py, correct typo --- scripts/ccpp_prebuild.py | 15 +++------------ scripts/common.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py index 75787bcf..e3e3517a 100755 --- a/scripts/ccpp_prebuild.py +++ b/scripts/ccpp_prebuild.py @@ -9,9 +9,9 @@ import re import sys -# Local modules +# CCPP framework imports from common import encode_container, decode_container, decode_container_as_dict, execute -from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE +from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE from common import split_var_name_and_array_reference from metadata_parser import merge_dictionaries, parse_scheme_tables, parse_variable_tables from mkcap import Cap, CapsMakefile, CapsCMakefile, SchemesMakefile, SchemesCMakefile @@ -32,15 +32,6 @@ # BASEDIR is the current directory where this script is executed BASEDIR = os.getcwd() -# SCRIPTDIR is the directory where the ccpp_prebuild.py and its Python modules are located -SCRIPTDIR = os.path.abspath(os.path.split(__file__)[0]) - -# SRCDIR is the directory where the CCPP framework source code (C, Fortran) is located -SRCDIR = os.path.abspath(os.path.join(SCRIPTDIR, '..', 'src')) - -# Definition of variables (metadata tables) that are provided by CCPP -CCPP_INTERNAL_VARIABLE_DEFINITON_FILE = os.path.join(SRCDIR, 'ccpp_types.F90') - ############################################################################### # Functions and subroutines # ############################################################################### @@ -99,7 +90,7 @@ def import_config(configfile): # Template code in host-model dependent CCPP prebuild config script config['ccpp_data_structure'] = ccpp_prebuild_config.CCPP_DATA_STRUCTURE - # Add model-intependent, CCPP-internal variable definition files + # Add model-independent, CCPP-internal variable definition files config['variable_definition_files'].append(CCPP_INTERNAL_VARIABLE_DEFINITON_FILE) # To handle new metadata: import DDT references (if exist) diff --git a/scripts/common.py b/scripts/common.py index 033a66ca..37e2081b 100755 --- a/scripts/common.py +++ b/scripts/common.py @@ -2,6 +2,7 @@ import keyword import logging +import os import re import subprocess import sys @@ -14,6 +15,16 @@ CCPP_TYPE = 'ccpp_t' +# SCRIPTDIR is the directory where ccpp_prebuild.py and its Python modules are located +SCRIPTDIR = os.path.abspath(os.path.split(__file__)[0]) + +# SRCDIR is the directory where the CCPP framework source code (C, Fortran) is located +SRCDIR = os.path.abspath(os.path.join(SCRIPTDIR, '..', 'src')) + +# Definition of variables (metadata tables) that are provided by CCPP +CCPP_INTERNAL_VARIABLE_DEFINITON_FILE = os.path.join(SRCDIR, 'ccpp_types.F90') + +# List of internal variables provided by the CCPP CCPP_INTERNAL_VARIABLES = { CCPP_ERROR_FLAG_VARIABLE : 'cdata%errflg', CCPP_ERROR_MSG_VARIABLE : 'cdata%errmsg', From 7f3c188ba263c52a14834960f6288aa6c3d69246 Mon Sep 17 00:00:00 2001 From: climbfuji Date: Thu, 12 Sep 2019 08:43:13 -0600 Subject: [PATCH 2/3] scripts/metadata2html.py: add batch processing option using CCPP prebuild config --- scripts/metadata2html.py | 83 +++++++++++++++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 9 deletions(-) diff --git a/scripts/metadata2html.py b/scripts/metadata2html.py index 76cbbc4b..a37a6a76 100755 --- a/scripts/metadata2html.py +++ b/scripts/metadata2html.py @@ -6,26 +6,84 @@ import sys # CCPP framework imports +from common import CCPP_INTERNAL_VARIABLE_DEFINITON_FILE from parse_tools import init_log, set_log_level from metadata_table import MetadataHeader +############################################################################### +# Set up the command line argument parser and other global variables # +############################################################################### + parser = argparse.ArgumentParser() -parser.add_argument('--metafile', '-m', action='store', - help='name of metadata file to convert', - required=True) +method = parser.add_mutually_exclusive_group(required=True) +method.add_argument('--config', '-c', action='store', + help='path to CCPP prebuild configuration file') +method.add_argument('--metafile', '-m', action='store', + help='name of metadata file to convert (requires -o)') parser.add_argument('--outputdir', '-o', action='store', help='directory where to write the html files', - required=True) + required='--metafile' in sys.argv or '-m' in sys.argv) -attributes = [ 'local_name', 'standard_name', 'long_name', 'units', +# List and order of variable attributes to output to HTML +ATTRIBUTES = [ 'local_name', 'standard_name', 'long_name', 'units', 'type', 'dimensions', 'kind', 'intent', 'optional' ] +############################################################################### +# Functions and subroutines # +############################################################################### + def parse_arguments(): """Parse command line arguments.""" args = parser.parse_args() + config = args.config filename = args.metafile outdir = args.outputdir - return (filename, outdir) + return (config, filename, outdir) + +def import_config(configfile, logger): + """Import the configuration from a given configuration file""" + + if not os.path.isfile(configfile): + raise Exception("Configuration file {0} not found".format(configfile)) + + # Import the host-model specific CCPP prebuild config; + # split into path and module name for import + configpath = os.path.abspath(os.path.split(configfile)[0]) + configmodule = os.path.split(configfile)[1].rstrip('.py') + sys.path.append(configpath) + ccpp_prebuild_config = __import__(configmodule) + + config = {} + # Definitions in host-model dependent CCPP prebuild config script + config['variable_definition_files'] = ccpp_prebuild_config.VARIABLE_DEFINITION_FILES + config['scheme_files'] = ccpp_prebuild_config.SCHEME_FILES + # Add model-independent, CCPP-internal variable definition files + config['variable_definition_files'].append(CCPP_INTERNAL_VARIABLE_DEFINITON_FILE) + # Output directory for converted metadata tables + config['metadata_html_output_dir'] = ccpp_prebuild_config.METADATA_HTML_OUTPUT_DIR + + return config + +def get_metadata_files_from_config(config, logger): + """Create a list of metadata filenames for a CCPP prebuild configuration""" + filenames = [] + for sourcefile in config['variable_definition_files'] + config['scheme_files'].keys(): + metafile = os.path.splitext(sourcefile)[0]+'.meta' + if os.path.isfile(metafile): + filenames.append(metafile) + else: + # DH* Warn for now, raise exception later when + # old metadata format is no longer supported + logger.warn("Metadata file {} for source file {} not found, assuming old metadata format".format( + metafile, sourcefile)) + return filenames + +def get_output_directory_from_config(config, logger): + """Return the html output directory for a CCPP prebuild configuration""" + outdir = config['metadata_html_output_dir'] + if not os.path.isdir(outdir): + raise Exception("Output directory {} for converted metadata tables does not exist".format(outdir)) + return outdir def convert_to_html(filename_in, outdir, logger): """Convert a metadata file into html (one html file for each table)""" @@ -34,7 +92,7 @@ def convert_to_html(filename_in, outdir, logger): logger.info("Converting file {} to HTML".format(filename_in)) metadata_headers = MetadataHeader.parse_metadata_file(filename_in) for metadata_header in metadata_headers: - filename_out = metadata_header.to_html(outdir, attributes) + filename_out = metadata_header.to_html(outdir, ATTRIBUTES) if filename_out: logger.info(" ... wrote {}".format(filename_out)) @@ -43,8 +101,15 @@ def main(): logger = init_log('metadata2html') set_log_level(logger, logging.INFO) # Convert metadata file - (filename, outdir) = parse_arguments() - convert_to_html(filename, outdir, logger) + (configfile, filename, outdir) = parse_arguments() + if configfile: + config = import_config(configfile, logger) + filenames = get_metadata_files_from_config(config, logger) + outdir = get_output_directory_from_config(config, logger) + for filename in filenames: + convert_to_html(filename, outdir, logger) + else: + convert_to_html(filename, outdir, logger) if __name__ == '__main__': main() From 258bc8e135585d88678165848612e2e309e1e0ca Mon Sep 17 00:00:00 2001 From: climbfuji Date: Thu, 12 Sep 2019 08:43:48 -0600 Subject: [PATCH 3/3] doc/CCPPtechnical/source/ScientificDocRules.inc: update documentation on metadata2html.py --- .../source/ScientificDocRules.inc | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/doc/CCPPtechnical/source/ScientificDocRules.inc b/doc/CCPPtechnical/source/ScientificDocRules.inc index 0f507aab..c766c093 100644 --- a/doc/CCPPtechnical/source/ScientificDocRules.inc +++ b/doc/CCPPtechnical/source/ScientificDocRules.inc @@ -539,7 +539,7 @@ script from the directory above where the CCPP is installed is: .. code-block:: fortran - ccpp/framework/scripts/metadata2html.py -m ccpp/physics/physics/file.meta -o ccpp/physics/physics/docs + ./ccpp/framework/scripts/metadata2html.py -m ccpp/physics/physics/file.meta -o ccpp/physics/physics/docs where ``-m`` is used to specify a file with metadata information and ``-o`` is used to specify the directory for output. Note that a single input file (``.meta``) may have more than one CCPP entrypoint @@ -549,8 +549,20 @@ Note that the ``.meta`` files are supplied with the CCPP Physics, and that there each Fortran file that contains one or more CCPP entrypoint schemes. The ``.meta`` files are located in the same directory as the scheme Fortran files (``ccpp/physics/physics``). -To generate a complete Scientific Documentation, documentation, script ``/ccpp/framework/scripts/metadata2html.py`` -must be run separately for each ``.meta`` file available in ccpp/physics/physics. +To generate a complete Scientific Documentation, documentation, script ``./ccpp/framework/scripts/metadata2html.py`` +must be run separately for each ``.meta`` file available in ``ccpp/physics/physics``. Alternatively, a batch mode exists +that converts all metadata files associated with schemes and variable definitions in the CCPP prebuild config: + +.. code-block:: fortran + + ./ccpp/framework/scripts/metadata2html.py -c ccpp/config/ccpp_prebuild_config.py + +Note that the options ``-c`` and ``-m`` are mutually exclusive, but that one of them is required. Option ``-m`` also requires +to specify ``-o``, while option ``-c`` will ignore ``-o``. For more information, use + +.. code-block:: fortran + + ./ccpp/framework/scripts/metadata2html.py --help Using Doxygen -------------------------------