Add script to check fortran vs metadata without a host model

scripts/ccpp_capgen.py

@@ -312,17 +312,30 @@ def compare_fheader_to_mheader(meta_header, fort_header, logger):
         # end if
         for mind, mvar in enumerate(mlist):
             lname = mvar.get_prop_value('local_name')
+            mname = mvar.get_prop_value('standard_name')
             arrayref = is_arrayspec(lname)
             fvar, find = find_var_in_list(lname, flist)
             # Check for consistency between optional variables in metadata and
             # optional variables in fortran. Error if optional attribute is
             # missing from fortran declaration.
+            # first check: if metadata says the variable is optional, does the fortran match?
             mopt  = mvar.get_prop_value('optional')
             if find and mopt:
                 fopt = fvar.get_prop_value('optional')
                 if (not fopt):
-                    errmsg = 'Missing optional attribute in fortran declaration for variable {}, in file {}'
-                    errors_found = add_error(errors_found, errmsg.format(mname,title))
+                    errmsg = f'Missing "optional" attribute in fortran declaration for variable {mname}, ' \
+                            'for {title}'
+                    errors_found = add_error(errors_found, errmsg)
+                # end if
+            # end if
+            # now check: if fortran says the variable is optional, does the metadata match?
+            if fvar:
+                fopt = fvar.get_prop_value('optional')
+                mopt = mvar.get_prop_value('optional')
+                if (fopt and not mopt):
+                    errmsg = f'Missing "optional" metadata property for variable {mname}, ' \
+                            'for {title}'
+                    errors_found = add_error(errors_found, errmsg)
                 # end if
             # end if
             if mind >= flen:
@@ -511,7 +524,7 @@ def parse_host_model_files(host_filenames, host_name, run_env):
     return host_model
 
 ###############################################################################
-def parse_scheme_files(scheme_filenames, run_env):
+def parse_scheme_files(scheme_filenames, run_env, skip_ddt_check=False):
 ###############################################################################
     """
     Gather information from scheme files (e.g., init, run, and finalize
@@ -520,11 +533,13 @@ def parse_scheme_files(scheme_filenames, run_env):
     table_dict = {} # Duplicate check and for dependencies processing
     header_dict = {} # To check for duplicates
     known_ddts = list()
+    # end if
     logger = run_env.logger
     for filename in scheme_filenames:
         logger.info('Reading CCPP schemes from {}'.format(filename))
         # parse metadata file
-        mtables = parse_metadata_file(filename, known_ddts, run_env)
+        mtables = parse_metadata_file(filename, known_ddts, run_env,
+                                      skip_ddt_check=skip_ddt_check)
         fort_file = find_associated_fortran_file(filename)
         ftables = parse_fortran_file(fort_file, run_env)
         # Check Fortran against metadata (will raise an exception on error)

scripts/fortran_tools/offline_check_fortran_vs_metadata.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+"""
+Recursively compare all fortran and metadata files in user-supplied directory, and report any problems
+USAGE:
+    ./offline_check_fortran_vs_metadata.py --directory <full path to directory with scheme files> (--debug)
+"""
+
+
+import sys
+import os
+import glob
+import logging
+import argparse
+import site
+# Enable imports from parent directory
+site.addsitedir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+# CCPP framework imports
+from framework_env import CCPPFrameworkEnv
+from fortran_tools import parse_fortran_file
+from metadata_table import parse_metadata_file
+from ccpp_capgen import find_associated_fortran_file
+from ccpp_capgen import check_fortran_against_metadata, parse_scheme_files
+from parse_tools import init_log, set_log_level
+from parse_tools import register_fortran_ddt_name
+from parse_tools import CCPPError, ParseInternalError
+
+_LOGGER = init_log(os.path.basename(__file__))
+_DUMMY_RUN_ENV = CCPPFrameworkEnv(_LOGGER, ndict={'host_files':'',
+                                                    'scheme_files':'',
+                                                    'suites':''})
+
+def find_files_to_compare(directory):
+    metadata_files = []
+    for file in glob.glob(os.path.join(directory,'**','*.meta'), recursive=True):
+        metadata_files.append(file)
+    # end for
+    return metadata_files
+
+def compare_fortran_and_metadata(scheme_directory, run_env):
+    ## Check for files
+    metadata_files = find_files_to_compare(scheme_directory)
+    # Perform checks
+    parse_scheme_files(metadata_files, run_env, skip_ddt_check=True)
+
+def parse_command_line(arguments, description):
+    """Parse command-line arguments"""
+    parser = argparse.ArgumentParser(description=description,
+                                     formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument("--directory", type=str, required=True,
+                        metavar='top-level directory to analyze - REQUIRED',
+                        help="""Full path to scheme directory""")
+    parser.add_argument("--debug", action='store_true', default=False,
+                        help="""turn on debug mode for additional verbosity""")
+    pargs = parser.parse_args(arguments)
+    return pargs
+
+def _main_func():
+    """Parse command line, then parse indicated host, scheme, and suite files.
+    Finally, generate code to allow host model to run indicated CCPP suites."""
+    pargs = parse_command_line(sys.argv[1:], __doc__)
+    logger = _LOGGER
+    if pargs.debug:
+        set_log_level(logger, logging.DEBUG)
+    else:
+        set_log_level(logger, logging.INFO)
+    # end if
+    compare_fortran_and_metadata(pargs.directory, _DUMMY_RUN_ENV)
+    print('All checks passed!')
+
+###############################################################################
+
+if __name__ == "__main__":
+    try:
+        _main_func()
+        sys.exit(0)
+    except ParseInternalError as pie:
+        _LOGGER.exception(pie)
+        sys.exit(-1)
+    except CCPPError as ccpp_err:
+        if _LOGGER.getEffectiveLevel() <= logging.DEBUG:
+            _LOGGER.exception(ccpp_err)
+        else:
+            _LOGGER.error(ccpp_err)
+        # end if
+        sys.exit(1)
+    finally:
+        logging.shutdown()
+    # end try
+

scripts/metadata_table.py

@@ -179,7 +179,7 @@ def _parse_config_line(line, context):
 
 ########################################################################
 
-def parse_metadata_file(filename, known_ddts, run_env):
+def parse_metadata_file(filename, known_ddts, run_env, skip_ddt_check=False):
     """Parse <filename> and return list of parsed metadata tables"""
     # Read all lines of the file at once
     meta_tables = []
@@ -196,7 +196,8 @@ def parse_metadata_file(filename, known_ddts, run_env):
     while curr_line is not None:
         if MetadataTable.table_start(curr_line):
             new_table = MetadataTable(run_env, parse_object=parse_obj,
-                                      known_ddts=known_ddts)
+                                      known_ddts=known_ddts,
+                                      skip_ddt_check=skip_ddt_check)
             ntitle = new_table.table_name
             if ntitle not in table_titles:
                 meta_tables.append(new_table)
@@ -271,7 +272,8 @@ class MetadataTable():
 
     def __init__(self, run_env, table_name_in=None, table_type_in=None,
                  dependencies=None, relative_path=None, known_ddts=None,
-                 var_dict=None, module=None, parse_object=None):
+                 var_dict=None, module=None, parse_object=None,
+                 skip_ddt_check=False):
         """Initialize a MetadataTable, either with a name, <table_name_in>, and
         type, <table_type_in>, or with information from a file (<parse_object>).
         if <parse_object> is None, <dependencies> and <relative_path> are
@@ -317,7 +319,8 @@ def __init__(self, run_env, table_name_in=None, table_type_in=None,
                 sect = MetadataSection(self.table_name, self.table_type,
                                        run_env, title=stitle,
                                        type_in=self.table_type, module=module,
-                                       var_dict=var_dict, known_ddts=known_ddts)
+                                       var_dict=var_dict, known_ddts=known_ddts,
+                                       skip_ddt_check=skip_ddt_check)
                 self.__sections.append(sect)
             # end if
         else:
@@ -342,10 +345,10 @@ def __init__(self, run_env, table_name_in=None, table_type_in=None,
                 known_ddts = []
             # end if
             self.__start_context = ParseContext(context=self.__pobj)
-            self.__init_from_file(known_ddts, self.__run_env)
+            self.__init_from_file(known_ddts, self.__run_env, skip_ddt_check=skip_ddt_check)
         # end if
 
-    def __init_from_file(self, known_ddts, run_env):
+    def __init_from_file(self, known_ddts, run_env, skip_ddt_check=False):
         """ Read the table preamble, assume the caller already figured out
         the first line of the header using the header_start method."""
         curr_line, _ = self.__pobj.next_line()
@@ -407,7 +410,8 @@ def __init_from_file(self, known_ddts, run_env):
                     skip_rest_of_section = False
                     section = MetadataSection(self.table_name, self.table_type,
                                               run_env, parse_object=self.__pobj,
-                                              known_ddts=known_ddts)
+                                              known_ddts=known_ddts,
+                                              skip_ddt_check=skip_ddt_check)
                     # Some table types only allow for one associated section
                     if ((len(self.__sections) == 1) and
                         (self.table_type in _SINGLETON_TABLE_TYPES)):
@@ -623,7 +627,7 @@ class MetadataSection(ParseSource):
 
     def __init__(self, table_name, table_type, run_env, parse_object=None,
                  title=None, type_in=None, module=None, process_type=None,
-                 var_dict=None, known_ddts=None):
+                 var_dict=None, known_ddts=None, skip_ddt_check=False):
         """Initialize a new MetadataSection object.
         If <parse_object> is not None, initialize from the current file and
         location in <parse_object>.
@@ -693,7 +697,8 @@ def __init__(self, table_name, table_type, run_env, parse_object=None,
                 known_ddts = []
             # end if
             self.__start_context = ParseContext(context=self.__pobj)
-            self.__init_from_file(table_name, table_type, known_ddts, run_env)
+            self.__init_from_file(table_name, table_type, known_ddts, run_env,
+                                  skip_ddt_check=skip_ddt_check)
         # end if
         # Register this header if it is a DDT
         if self.header_type == 'ddt':
@@ -724,7 +729,7 @@ def _default_module(self):
         # end if
         return def_mod
 
-    def __init_from_file(self, table_name, table_type, known_ddts, run_env):
+    def __init_from_file(self, table_name, table_type, known_ddts, run_env, skip_ddt_check=False):
         """ Read the section preamble, assume the caller already figured out
         the first line of the header using the header_start method."""
         start_ctx = context_string(self.__pobj)
@@ -809,7 +814,8 @@ def __init_from_file(self, table_name, table_type, known_ddts, run_env):
         valid_lines = True
         self.__variables = VarDictionary(self.title, run_env)
         while valid_lines:
-            newvar, curr_line = self.parse_variable(curr_line, known_ddts)
+            newvar, curr_line = self.parse_variable(curr_line, known_ddts,
+                                                    skip_ddt_check=skip_ddt_check)
             valid_lines = newvar is not None
             if valid_lines:
                 if run_env.verbose:
@@ -828,7 +834,7 @@ def __init_from_file(self, table_name, table_type, known_ddts, run_env):
             # end if
         # end while
 
-    def parse_variable(self, curr_line, known_ddts):
+    def parse_variable(self, curr_line, known_ddts, skip_ddt_check=False):
         """Parse a new metadata variable beginning on <curr_line>.
         The header line has the format [ <valid_fortran_symbol> ].
         """
@@ -872,7 +878,10 @@ def parse_variable(self, curr_line, known_ddts):
                     pval_str = prop[1].strip()
                     if ((pname == 'type') and
                         (not check_fortran_intrinsic(pval_str, error=False))):
-                        if pval_str in known_ddts:
+                        if skip_ddt_check or pval_str in known_ddts:
+                            if skip_ddt_check:
+                                register_fortran_ddt_name(pval_str)
+                            # end if
                             pval = pval_str
                             pname = 'ddt_type'
                         else:

test/capgen_test/temp_adjust.F90

@@ -27,7 +27,7 @@ subroutine temp_adjust_run(foo, timestep, temp_prev, temp_layer, qv, ps,    &
     REAL(kind_phys),           intent(in)    :: temp_prev(:)
     REAL(kind_phys),           intent(inout) :: temp_layer(foo)
     character(len=512),        intent(out)   :: errmsg
-    integer,         optional, intent(out)   :: errflg
+    integer,                   intent(out)   :: errflg
     real(kind_phys), optional, intent(in)    :: innie
     real(kind_phys), optional, intent(out)   :: outie
     real(kind_phys), optional, intent(inout) :: optsie
@@ -36,9 +36,7 @@ subroutine temp_adjust_run(foo, timestep, temp_prev, temp_layer, qv, ps,    &
     integer :: col_index
 
     errmsg = ''
-    if (present(errflg)) then
-       errflg = 0
-    end if
+    errflg = 0
 
     do col_index = 1, foo
        temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index)