diff --git a/scripts/ccpp_track_variables.py b/scripts/ccpp_track_variables.py new file mode 100755 index 00000000..71671a9d --- /dev/null +++ b/scripts/ccpp_track_variables.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 + +# Standard modules +import os +import argparse +import logging +import glob + +# CCPP framework imports +from metadata_table import find_scheme_names, parse_metadata_file +from ccpp_prebuild import import_config, gather_variable_definitions +from mkstatic import Suite +from parse_checkers import registered_fortran_ddt_names +from parse_tools import init_log, set_log_level +from framework_env import CCPPFrameworkEnv + +############################################################################### +# Set up the command line argument parser and other global variables # +############################################################################### + +############################################################################### +# Functions and subroutines # +############################################################################### + +def parse_arguments(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument('-s', '--sdf', help='suite definition file to parse', required=True) + parser.add_argument('-m', '--metadata_path', + help='path to CCPP scheme metadata files', required=True) + parser.add_argument('-c', '--config', + help='path to CCPP prebuild configuration file', required=True) + parser.add_argument('-v', '--variable', help='variable to track through CCPP suite', + required=True) + parser.add_argument('--debug', action='store_true', help='enable debugging output', + default=False) + + args = parser.parse_args() + + return(args) + +def setup_logging(debug): + """Sets up the logging module and logging level.""" + + #Use capgen logging tools + logger = init_log('ccpp_track_variables') + + if debug: + set_log_level(logger, logging.DEBUG) + logger.info('Logging level set to DEBUG') + else: + set_log_level(logger, logging.WARNING) + return logger + +def parse_suite(sdf, run_env): + """Reads the provided sdf, parses into a Suite data structure, including the "call tree": + the ordered list of schemes for the suite specified by the provided sdf""" + run_env.logger.info(f'Reading sdf {sdf} and populating Suite object') + suite = Suite(sdf_name=sdf) + run_env.logger.info(f'Reading sdf {sdf} and populating Suite object') + success = suite.parse(make_call_tree=True) + if not success: + raise Exception(f'Parsing suite definition file {sdf} failed.') + run_env.logger.info(f'Successfully read sdf {suite.sdf_name}') + return suite + +def create_metadata_filename_dict(metapath): + """Given a path, read all .meta files in that directory and add them to a dictionary: the keys + are the name of the scheme, and the values are the filename of the .meta file associated + with that scheme""" + + metadata_dict = {} + scheme_filenames = glob.glob(os.path.join(metapath, "*.meta")) + if not scheme_filenames: + raise Exception(f'No files found in {metapath} with ".meta" extension') + + for scheme_fn in scheme_filenames: + schemes = find_scheme_names(scheme_fn) + # The above returns a list of schemes in each filename, but + # we want a dictionary of schemes associated with filenames: + for scheme in schemes: + metadata_dict[scheme] = scheme_fn + + return metadata_dict + + +def create_var_graph(suite, var, config, metapath, run_env): + """Given a suite, variable name, a 'config' dictionary, and a path to .meta files: + 1. Creates a dictionary associating schemes with their .meta files + 2. Loops through the call tree of the provided suite + 3. For each scheme, reads .meta file for said scheme, checks for variable within that + scheme, and if it exists, adds an entry to a list of tuples, where each tuple includes + the name of the scheme and the intent of the variable within that scheme""" + + # Create a list of tuples that will hold the in/out information for each scheme + var_graph = [] + + run_env.logger.debug(f"reading .meta files in path:\n {metapath}") + metadata_dict=create_metadata_filename_dict(metapath) + + run_env.logger.debug(f"reading metadata files for schemes defined in config file: " + f"{config['scheme_files']}") + + # Loop through call tree, find matching filename for scheme via dictionary schemes_in_files, + # then parse that metadata file to find variable info + partial_matches = {} + for scheme in suite.call_tree: + run_env.logger.debug(f"reading meta file for scheme {scheme} ") + + if scheme in metadata_dict: + scheme_filename = metadata_dict[scheme] + else: + raise Exception(f"Error, scheme '{scheme}' from suite '{suite.sdf_name}' " + f"not found in metadata files in {metapath}") + + run_env.logger.debug(f"reading metadata file {scheme_filename} for scheme {scheme}") + + new_metadata_headers = parse_metadata_file(scheme_filename, + known_ddts=registered_fortran_ddt_names(), run_env=run_env) + for scheme_metadata in new_metadata_headers: + for section in scheme_metadata.sections(): + found_var = [] + intent = '' + for scheme_var in section.variable_list(): + exact_match = False + if var == scheme_var.get_prop_value('standard_name'): + run_env.logger.debug(f"Found variable {var} in scheme {section.title}") + found_var = var + exact_match = True + intent = scheme_var.get_prop_value('intent') + break + scheme_var_standard_name = scheme_var.get_prop_value('standard_name') + if scheme_var_standard_name.find(var) != -1: + run_env.logger.debug(f"{var} matches {scheme_var_standard_name}") + found_var.append(scheme_var_standard_name) + if not found_var: + run_env.logger.debug(f"Did not find variable {var} in scheme {section.title}") + elif exact_match: + run_env.logger.debug(f"Exact match found for variable {var} in scheme {section.title}," + f" intent {intent}") + var_graph.append((section.title,intent)) + else: + run_env.logger.debug(f"Found inexact matches for variable(s) {var} " + f"in scheme {section.title}:\n{found_var}") + partial_matches[section.title] = found_var + if var_graph: + success = True + run_env.logger.debug(f"Successfully generated variable graph for sdf {suite.sdf_name}\n") + else: + success = False + run_env.logger.error(f"Variable {var} not found in any suites for sdf {suite.sdf_name}\n") + if partial_matches: + print("Did find partial matches that may be of interest:\n") + for key in partial_matches: + print(f"In {key} found variable(s) {partial_matches[key]}") + + return (success,var_graph) + +def main(): + """Main routine that traverses a CCPP suite and outputs the list of schemes that use given variable""" + + args = parse_arguments() + + logger = setup_logging(args.debug) + + #Use new capgen class CCPPFrameworkEnv + run_env = CCPPFrameworkEnv(logger, host_files="", scheme_files="", suites="") + + suite = parse_suite(args.sdf,run_env) + + (success, config) = import_config(args.config, None) + if not success: + raise Exception('Call to import_config failed.') + + # Variables defined by the host model; this call is necessary because it converts some old + # metadata formats so they can be used later in the script + (success, _, _) = gather_variable_definitions(config['variable_definition_files'], + config['typedefs_new_metadata']) + if not success: + raise Exception('Call to gather_variable_definitions failed.') + + (success, var_graph) = create_var_graph(suite, args.variable, config, args.metadata_path, run_env) + if success: + print(f"For suite {suite.sdf_name}, the following schemes (in order) " + f"use the variable {args.variable}:") + for entry in var_graph: + print(f"{entry[0]} (intent {entry[1]})") + + +if __name__ == '__main__': + main() diff --git a/scripts/metadata_parser.py b/scripts/metadata_parser.py index cf6aa6ea..60abc1ec 100755 --- a/scripts/metadata_parser.py +++ b/scripts/metadata_parser.py @@ -26,7 +26,7 @@ # Output: This routine converts the argument tables for all subroutines / typedefs / kind / module variables # into dictionaries suitable to be used with ccpp_prebuild.py (which generates the fortran code for the caps) -# Items in this dictionary are used for checking valid entries in metadata tables. For columsn with no keys/keys +# Items in this dictionary are used for checking valid entries in metadata tables. For columns with no keys/keys # commented out, no check is performed. This is the case for 'type' and 'kind' right now, since models use their # own derived data types and kind types. VALID_ITEMS = { diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 7c94a04e..83b3dcfd 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -504,6 +504,7 @@ def __init__(self, **kwargs): self._sdf_name = None self._all_schemes_called = None self._all_subroutines_called = None + self._call_tree = [] self._caps = None self._module = None self._subroutines = None @@ -545,7 +546,7 @@ def update_cap(self): def update_cap(self, value): self._update_cap = value - def parse(self): + def parse(self, make_call_tree=False): '''Parse the suite definition file.''' success = True @@ -568,6 +569,10 @@ def parse(self): self._all_schemes_called = [] self._all_subroutines_called = [] + if make_call_tree: + # Call tree of all schemes in SDF (with duplicates and subcycles) + self._call_tree = [] + # Build hierarchical structure as in SDF self._groups = [] for group_xml in suite_xml: @@ -594,14 +599,22 @@ def parse(self): loop=int(subcycle_xml.get('loop')) for ccpp_stage in CCPP_STAGES: self._all_subroutines_called.append(scheme_xml.text + '_' + CCPP_STAGES[ccpp_stage]) + subcycles.append(Subcycle(loop=loop, schemes=schemes)) + if make_call_tree: + # Populate call tree from SDF's heirarchical structure, including multiple calls in subcycle loops + for loop in range(0,int(subcycle_xml.get('loop'))): + for scheme_xml in subcycle_xml: + self._call_tree.append(scheme_xml.text) + self._groups.append(Group(name=group_xml.get('name'), subcycles=subcycles, suite=self._name)) # Remove duplicates from list of all subroutines an schemes self._all_schemes_called = list(set(self._all_schemes_called)) self._all_subroutines_called = list(set(self._all_subroutines_called)) + return success def print_debug(self): @@ -618,6 +631,11 @@ def all_schemes_called(self): '''Get the list of all schemes.''' return self._all_schemes_called + @property + def call_tree(self): + '''Get the call tree of the suite (all schemes, in order, with duplicates and loops).''' + return self._call_tree + @property def all_subroutines_called(self): '''Get the list of all subroutines.'''