diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index b5005ed9..9e96e4e4 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -25,3 +25,7 @@ if(BUILD_DOCUMENTATION) VERBATIM) endif() +set(gmtb_sty_in ${CMAKE_CURRENT_SOURCE_DIR}/DevelopersGuide/gmtb.sty) +set(gmtb_sty ${CMAKE_CURRENT_BINARY_DIR}/DevelopersGuide/gmtb.sty) + +configure_file(${gmtb_sty_in} ${gmtb_sty} @ONLY) diff --git a/doc/common/gmtb.sty b/doc/DevelopersGuide/gmtb.sty similarity index 100% rename from doc/common/gmtb.sty rename to doc/DevelopersGuide/gmtb.sty diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py index 1e3dc561..68353ed3 100755 --- a/scripts/ccpp_prebuild.py +++ b/scripts/ccpp_prebuild.py @@ -188,15 +188,6 @@ def parse_suites(suites_dir, sdfs): suites.append(suite) return (success, suites) -def check_unique_pset_per_scheme(scheme_files): - """Check that each scheme belongs to one and only one physics set""" - success = True - for scheme_file in scheme_files.keys(): - if len(scheme_files[scheme_file])>1: - logging.error("Scheme file {0} belongs to multiple physics sets: {1}".format(scheme_file, ','.join(scheme_files[scheme_file]))) - success = False - return success - def convert_local_name_from_new_metadata(metadata, standard_name, typedefs_new_metadata, converted_variables): """Convert local names in new metadata format (no old-style DDT references, array references as standard names) to old metadata format (with old-style DDT references, array references as local names).""" @@ -319,33 +310,24 @@ def collect_physics_subroutines(scheme_files): # Parse all scheme files metadata_request = {} arguments_request = {} - pset_request = {} - pset_schemes = {} - for scheme_file in scheme_files.keys(): + for scheme_file in scheme_files: (scheme_filepath, scheme_filename) = os.path.split(os.path.abspath(scheme_file)) # Change to directory where scheme_file lives os.chdir(scheme_filepath) (metadata, arguments) = parse_scheme_tables(scheme_filename) - # The different psets for the variables used by schemes in scheme_file - pset = { var_name : scheme_files[scheme_file] for var_name in metadata.keys() } - # The different psets for the schemes in scheme_file - for scheme_name in arguments.keys(): - pset_schemes[scheme_name] = scheme_files[scheme_file] - # Merge metadata and pset, append to arguments + # Merge metadata, append to arguments metadata_request = merge_dictionaries(metadata_request, metadata) - pset_request = merge_dictionaries(pset_request, pset) arguments_request.update(arguments) os.chdir(BASEDIR) # Return to BASEDIR os.chdir(BASEDIR) - return (success, metadata_request, pset_request, arguments_request, pset_schemes) + return (success, metadata_request, arguments_request) -def filter_metadata(metadata, pset, arguments, suites): +def filter_metadata(metadata, arguments, suites): """Remove all variables from metadata that are not used in the given suite""" success = True # Output: filtered dictionaries metadata_filtered = {} - pset_filtered = {} arguments_filtered = {} # Loop through all variables and check if the calling subroutine is in list of subroutines for var_name in sorted(metadata.keys()): @@ -361,7 +343,6 @@ def filter_metadata(metadata, pset, arguments, suites): break if keep: metadata_filtered[var_name] = metadata[var_name] - pset_filtered[var_name] = pset[var_name] else: logging.info("filtering out variable {0}".format(var_name)) for scheme in arguments.keys(): @@ -370,7 +351,7 @@ def filter_metadata(metadata, pset, arguments, suites): arguments_filtered[scheme] = arguments[scheme] break - return (success, metadata_filtered, pset_filtered, arguments_filtered) + return (success, metadata_filtered, arguments_filtered) def check_optional_arguments(metadata, arguments, optional_arguments): """Check if for each subroutine with optional arguments, an entry exists in the @@ -381,6 +362,46 @@ def check_optional_arguments(metadata, arguments, optional_arguments): for each subroutine.""" logging.info('Checking optional arguments in physics schemes ...') success = True + + # First make sure that the CCPP prebuild config entry doesn't contain any variables that are unknown + # (by standard name), or that it lists variables as optional arguments that aren't declared as optional + for module_name in optional_arguments.keys(): + # Skip modules that have been filtered out (because they are not used by the selected suites, for example) + if module_name in arguments.keys(): + # DH* 2020-05-26 + # This is a test/workaround for some legacy code. It is actually required that + # module_name == scheme_name (see metadata_parser.py, around line 528). + for scheme_name in arguments[module_name].keys(): + if not scheme_name == module_name: + raise Exception("Found example for scheme_name /= module_name: {} vs. {}".format(module_name, scheme_name)) + scheme_name = module_name + # *DH 2020-05-26 + for subroutine_name in optional_arguments[module_name].keys(): + # If optional arguments are listed individually, check each of them + if type(optional_arguments[module_name][subroutine_name]) is list: + for var_name in optional_arguments[module_name][subroutine_name]: + if not var_name in arguments[module_name][scheme_name][subroutine_name]: + raise Exception("Explicitly requested optional argument '{}' not known to {}/{}".format( + var_name, module_name, subroutine_name)) + else: + for var in metadata[var_name][:]: + for item in var.container.split(' '): + subitems = item.split('_') + if subitems[0] == 'MODULE': + module_name_test = '_'.join(subitems[1:]) + elif subitems[0] == 'SCHEME': + scheme_name_test = '_'.join(subitems[1:]) + elif subitems[0] == 'SUBROUTINE': + subroutine_name_test = '_'.join(subitems[1:]) + else: + success = False + logging.error('Invalid identifier {0} in container value {1} of requested variable {2}'.format( + subitems[0], var.container, var_name)) + if module_name_test == module_name and scheme_name_test == scheme_name \ + and subroutine_name_test == subroutine_name and not var.optional in ['t', 'T']: + raise Exception("Variable {} in {} / {}".format(var_name, module_name, subroutine_name) + \ + " is not an optional argument, but listed as such in the CCPP prebuild config") + for var_name in sorted(metadata.keys()): # The notation metadata[var_name][:] is a convenient way to make a copy # of the metadata[var_name] list, which allows removing items as we go @@ -395,14 +416,12 @@ def check_optional_arguments(metadata, arguments, optional_arguments): elif subitems[0] == 'SUBROUTINE': subroutine_name = '_'.join(subitems[1:]) else: - success = False - logging.error('Invalid identifier {0} in container value {1} of requested variable {2}'.format( - subitems[0], var.container, var_name)) + raise Exception('Invalid identifier {0} in container value {1} of requested variable {2}'.format( + subitems[0], var.container, var_name)) if not module_name in optional_arguments.keys() or not \ subroutine_name in optional_arguments[module_name].keys(): - success = False - logging.error('No entry found in optional_arguments dictionary for optional argument ' + \ - '{0} to subroutine {1} in module {2}'.format(var_name, subroutine_name, module_name)) + raise Exception('No entry found in optional_arguments dictionary for optional argument ' + \ + '{0} to subroutine {1} in module {2}'.format(var_name, subroutine_name, module_name)) if type(optional_arguments[module_name][subroutine_name]) is list: if var_name in optional_arguments[module_name][subroutine_name]: logging.debug('Optional argument {0} to subroutine {1} in module {2} is required, keep in list'.format( @@ -427,7 +446,7 @@ def check_optional_arguments(metadata, arguments, optional_arguments): return (success, metadata, arguments) -def compare_metadata(metadata_define, metadata_request, pset_request, psets_merged): +def compare_metadata(metadata_define, metadata_request): """Compare the requested metadata to the defined one. For each requested entry, a single (i.e. non-ambiguous entry) must be present in the defined entries. All optional arguments that are still in the list of required variables for a scheme are needed, @@ -435,7 +454,7 @@ def compare_metadata(metadata_define, metadata_request, pset_request, psets_merg logging.info('Comparing metadata for requested and provided variables ...') success = True - modules = { x : [] for x in psets_merged } + modules = [] metadata = {} for var_name in sorted(metadata_request.keys()): # Check that variable is provided by the model @@ -486,9 +505,8 @@ def compare_metadata(metadata_define, metadata_request, pset_request, psets_merg for item in var.container.split(' '): subitems = item.split('_') if subitems[0] == 'MODULE': - # Add to list of required modules for each pset the requested variable falls under - for pset in pset_request[var_name]: - modules[pset].append('_'.join(subitems[1:])) + # Add to list of required modules + modules.append('_'.join(subitems[1:])) elif subitems[0] == 'TYPE': pass else: @@ -502,16 +520,15 @@ def compare_metadata(metadata_define, metadata_request, pset_request, psets_merg for var in metadata[var_name]: var.target = target logging.debug('Requested variable {0} in {1} matched to target {2} in module {3}'.format( - var_name, var.container, target, modules[pset_request[var_name][0]][-1])) + var_name, var.container, target, modules[-1])) # Update len=* for character variables if var.type == 'character' and var.kind == 'len=*': logging.debug('Update kind information for requested variable {0} in {1} from {2} to {3}'.format(var_name, var.container, var.kind, kind)) var.kind = kind - # Remove duplicated from list of modules - for pset in psets_merged: - modules[pset] = sorted(list(set(modules[pset]))) + # Remove duplicates from list of modules + modules = sorted(list(set(modules))) return (success, modules, metadata) def generate_suite_and_group_caps(suites, metadata_request, metadata_define, arguments, caps_dir): @@ -720,13 +737,6 @@ def main(): if not success: raise Exception('Parsing suite definition files failed.') - # Check that each scheme only belongs to one set of physics - # this is required for using the optimized version of ccpp_field_get - # that supplies the build-time derived index in the array - success = check_unique_pset_per_scheme(config['scheme_files']) - if not success: - raise Exception('Call to check_unique_pset_per_scheme failed.') - # Variables defined by the host model (success, metadata_define) = gather_variable_definitions(config['variable_definition_files'], config['typedefs_new_metadata']) if not success: @@ -738,13 +748,12 @@ def main(): raise Exception('Call to metadata_to_html failed.') # Variables requested by the CCPP physics schemes - (success, metadata_request, pset_request, arguments_request, pset_schemes) = collect_physics_subroutines(config['scheme_files']) + (success, metadata_request, arguments_request) = collect_physics_subroutines(config['scheme_files']) if not success: raise Exception('Call to collect_physics_subroutines failed.') - # Filter metadata/pset/arguments - remove whatever is not included in suite definition files - (success, metadata_request, pset_request, arguments_request) = filter_metadata(metadata_request, pset_request, - arguments_request, suites) + # Filter metadata/arguments - remove whatever is not included in suite definition files + (success, metadata_request, arguments_request) = filter_metadata(metadata_request, arguments_request, suites) if not success: raise Exception('Call to filter_metadata failed.') @@ -755,15 +764,12 @@ def main(): raise Exception('Call to check_optional_arguments failed.') # Create a LaTeX table with all variables requested by the pool of physics and/or provided by the host model - success = metadata_to_latex(metadata_define, metadata_request, pset_request, config['host_model'], config['latex_vartable_file']) + success = metadata_to_latex(metadata_define, metadata_request, config['host_model'], config['latex_vartable_file']) if not success: raise Exception('Call to metadata_to_latex failed.') - # Flatten list of list of psets for all variables - psets_merged = list(set(itertools.chain(*pset_request.values()))) - # Check requested against defined arguments to generate metadata (list/dict of variables for CCPP) - (success, modules, metadata) = compare_metadata(metadata_define, metadata_request, pset_request, psets_merged) + (success, modules, metadata) = compare_metadata(metadata_define, metadata_request) if not success: raise Exception('Call to compare_metadata failed.') @@ -774,7 +780,7 @@ def main(): raise Exception('Call to generate_typedefs_makefile failed.') # Add filenames of schemes to makefile/cmakefile/shell script - add dependencies for schemes - success = generate_schemes_makefile(config['scheme_files_dependencies'] + list(config['scheme_files'].keys()), + success = generate_schemes_makefile(config['scheme_files_dependencies'] + config['scheme_files'], config['schemes_makefile'], config['schemes_cmakefile'], config['schemes_sourcefile']) if not success: diff --git a/scripts/common.py b/scripts/common.py index f638547a..abffa107 100755 --- a/scripts/common.py +++ b/scripts/common.py @@ -11,8 +11,18 @@ CCPP_ERROR_MSG_VARIABLE = 'ccpp_error_message' CCPP_LOOP_COUNTER = 'ccpp_loop_counter' CCPP_BLOCK_NUMBER = 'ccpp_block_number' +CCPP_BLOCK_COUNT = 'ccpp_block_count' +CCPP_BLOCK_SIZES = 'ccpp_block_sizes' CCPP_THREAD_NUMBER = 'ccpp_thread_number' +CCPP_HORIZONTAL_LOOP_EXTENT = 'horizontal_loop_extent' +CCPP_HORIZONTAL_DIMENSION = 'horizontal_dimension' + +FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '(', ')', '==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.', + '.true.', '.false.', '.lt.', '.le.', '.eq.', '.ge.', '.gt.', '.ne.', + '.not.', '.and.', '.or.', '.xor.'] +FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([word.replace('(','\(').replace(')', '\)') for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) + CCPP_TYPE = 'ccpp_t' # SCRIPTDIR is the directory where ccpp_prebuild.py and its Python modules are located @@ -136,7 +146,8 @@ def escape_tex(text): """Substitutes characters for generating LaTeX sources files from Python.""" return text.replace( '%', '\%').replace( - '_', '\_') + '_', '\_').replace( + '&', '\&') def isstring(s): """Return true if a variable is a string""" diff --git a/scripts/conversion_tools/unit_conversion.py b/scripts/conversion_tools/unit_conversion.py index 811bee2d..b61d670a 100755 --- a/scripts/conversion_tools/unit_conversion.py +++ b/scripts/conversion_tools/unit_conversion.py @@ -91,6 +91,30 @@ def d__to__s(): """Convert day to second""" return '8.64E+4{kind}*{var}' +################## +# Temperature # +################## + +def K__to__C(): + """Convert Kelvin to Celcius""" + return '{var}-273.15{kind}' + +def C__to__K(): + """Convert Celcius to Kelvin""" + return '{var}+273.15{kind}' + +################## +# Mass # +################## + +def kg_kg_minus_1__to__g_kg_minus_1(): + """Convert kilogram per kilogram to gram per kilogram""" + return '1.0E+3{kind}*{var}' + +def g_kg_minus_1__to__kg_kg_minus_1(): + """Convert gram per kilogram to kilogram per kilogram""" + return '{var}/1.0E+3{kind}' + ################## # Composed units # ################## diff --git a/scripts/metadata_parser.py b/scripts/metadata_parser.py index 4f8077e2..60056c3d 100755 --- a/scripts/metadata_parser.py +++ b/scripts/metadata_parser.py @@ -34,20 +34,24 @@ long_name = 'error message for error handling in CCPP', units = 'none', type = 'character', + dimensions = [], rank = '', kind = 'len=*', intent = 'out', - optional = 'F' + optional = 'F', + active = 'T', ), 'ccpp_error_flag' : Var(local_name = 'ierr', standard_name = 'ccpp_error_flag', long_name = 'error flag for error handling in CCPP', units = 'flag', type = 'integer', + dimensions = [], rank = '', kind = '', intent = 'out', - optional = 'F' + optional = 'F', + active = 'T', ), } @@ -120,22 +124,56 @@ def read_new_metadata(filename, module_name, table_name, scheme_name = None, sub container = encode_container(module_name, scheme_name, table_name) for new_var in new_metadata_header.variable_list(): standard_name = new_var.get_prop_value('standard_name') - rank = len(new_var.get_prop_value('dimensions')) + # DH* 2020-05-26 + # Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent). + # Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't + # make sense), we swap the standard name and add a note to the long name + legacy_note = '' + if standard_name == 'horizontal_loop_extent' and scheme_name and \ + (table_name.endswith("_init") or table_name.endswith("finalize")): + logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \ + " with 'horizontal_dimension' in table {}".format(table_name)) + standard_name = 'horizontal_dimension' + legacy_note = ' replaced by horizontal dimension (legacy extension)' + elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith("_run"): + logging.warn("Legacy extension - replacing variable 'horizontal_dimension' " + \ + "with 'horizontal_loop_extent' in table {}".format(table_name)) + standard_name = 'horizontal_loop_extent' + legacy_note = ' replaced by horizontal loop extent (legacy extension)' + # Adjust dimensions + dimensions = new_var.get_prop_value('dimensions') + if scheme_name and (table_name.endswith("_init") or table_name.endswith("finalize")) \ + and 'horizontal_loop_extent' in dimensions: + logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ + "for variable {} in table {}".format(standard_name,table_name)) + dimensions = ['horizontal_dimension' if x=='horizontal_loop_extent' else x for x in dimensions] + elif scheme_name and table_name.endswith("_run") and 'horizontal_dimension' in dimensions: + logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ + "for variable {} in table {}".format(standard_name,table_name)) + dimensions = ['horizontal_loop_extent' if x=='horizontal_dimension' else x for x in dimensions] + # *DH 2020-05-26 + if new_var.get_prop_value('active').lower() == '.true.': + active = 'T' + elif new_var.get_prop_value('active').lower() == '.false.': + active = 'F' + else: + # Replace multiple whitespaces and use lowercase throughout + active = ' '.join(new_var.get_prop_value('active').lower().split()) var = Var(standard_name = standard_name, - long_name = new_var.get_prop_value('long_name'), + long_name = new_var.get_prop_value('long_name') + legacy_note, units = new_var.get_prop_value('units'), local_name = new_var.get_prop_value('local_name'), type = new_var.get_prop_value('type'), + dimensions = dimensions, container = container, kind = new_var.get_prop_value('kind'), intent = new_var.get_prop_value('intent'), optional = 'T' if new_var.get_prop_value('optional') else 'F', + active = active, ) - # Set rank using integer-setter method - var.rank = rank # Check for duplicates in same table if standard_name in metadata.keys(): - raise Exception("Error, multiple definitions of standard name {0} in new metadata table {1}".format(standard_name, table_name)) + raise Exception("Error, multiple definitions of standard name {0} in new metadata table {1}".format(standard_name, table_name)) metadata[standard_name] = [var] return metadata @@ -269,7 +307,6 @@ def parse_variable_tables(filename): line_counter = 0 in_table = False in_type = False - new_metadata = False for line in lines[startline:endline]: current_line_number = startline + line_counter @@ -296,7 +333,6 @@ def parse_variable_tables(filename): if 'htmlinclude' in line.lower(): words = line.split() if words[0] == '!!' and words[1] == '\\htmlinclude' and len(words) == 3: - new_metadata = True filename_parts = filename.split('.') metadata_filename = '.'.join(filename_parts[0:len(filename_parts)-1]) + '.meta' this_metadata = read_new_metadata(metadata_filename, module_name, table_name) @@ -347,65 +383,19 @@ def parse_variable_tables(filename): #raise Exception("Old metadata table found for table {}".format(table_name)) # *DH continue - elif current_line_number == header_line_number + 1 and not new_metadata: - # Skip over separator line - line_counter += 1 - continue else: if len(words) == 1: # End of table if words[0].strip() == '!!': - if new_metadata and not current_line_number == header_line_number+1: + if not current_line_number == header_line_number+1: raise Exception("Invalid definition of new metadata format in file {0}".format(filename)) in_table = False - new_metadata = False line_counter += 1 continue else: raise Exception('Encountered invalid line "{0}" in argument table {1}'.format(line, table_name)) else: - if new_metadata: - raise Exception("Invalid definition of new metadata format in file {0}: {1}".format(filename, words)) - var_items = [x.strip() for x in words[1:-1]] - if not len(var_items) == len(table_header): - raise Exception('Error parsing variable entry "{0}" in argument table {1}'.format(var_items, table_name)) - var_name = var_items[standard_name_index] - # Skip variables without a standard_name (i.e. empty cell in column standard_name) - if var_name: - # Enforce CF standards: no dashes, no dots (underscores instead) - if "-" in var_name: - raise Exception("Invalid character '-' found in standard name {0} in table {1}".format(var_name, table_name)) - elif "." in var_name: - raise Exception("Invalid character '.' found in standard name {0} in table {1}".format(var_name, table_name)) - # - var = Var.from_table(table_header,var_items) - if table_name == module_name: - container = encode_container(module_name) - else: - container = encode_container(module_name, table_name) - var.container = container - # Check for incompatible definitions with CCPP mandatory variables - if var_name in CCPP_MANDATORY_VARIABLES.keys() and not CCPP_MANDATORY_VARIABLES[var_name].compatible(var): - raise Exception('Entry for variable {0}'.format(var_name) + \ - ' in argument table {0}'.format(table_name) +\ - ' is incompatible with mandatory variable:\n' +\ - ' existing: {0}\n'.format(CCPP_MANDATORY_VARIABLES[var_name].print_debug()) +\ - ' vs. new: {0}'.format(var.print_debug())) - # Add variable to metadata dictionary - if not var_name in metadata.keys(): - metadata[var_name] = [var] - else: - for existing_var in metadata[var_name]: - if not existing_var.compatible(var): - raise Exception('New entry for variable {0}'.format(var_name) + \ - ' in argument table {0}'.format(table_name) +\ - ' is incompatible with existing entry:\n' +\ - ' existing: {0}\n'.format(existing_var.print_debug()) +\ - ' vs. new: {0}'.format(var.print_debug())) - - metadata[var_name].append(var) - #else: - # logging.debug('Skipping variable entry "{0}" without a standard_name'.format(var_items)) + raise Exception("Invalid definition of metadata in file {0}: {1}".format(filename, words)) line_counter += 1 @@ -606,7 +596,6 @@ def parse_scheme_tables(filename): if 'htmlinclude' in lines[header_line_number].lower(): words = lines[header_line_number].split() if words[0] == '!!' and words[1] == '\\htmlinclude' and len(words) == 3: - new_metadata = True filename_parts = filename.split('.') metadata_filename = '.'.join(filename_parts[0:len(filename_parts)-1]) + '.meta' this_metadata = read_new_metadata(metadata_filename, module_name, table_name, @@ -640,88 +629,23 @@ def parse_scheme_tables(filename): else: raise Exception('Encountered invalid format "{0}" of new metadata table hook in table {1}'.format(line, table_name)) line_number += 1 - continue - # Separate the table headers - table_header = lines[header_line_number].split('|') - # Check for blank table - if len(table_header) <= 1: - logging.debug('Skipping blank table {0}'.format(table_name)) - table_found = False - continue - # Extract table header - table_header = [x.strip() for x in table_header[1:-1]] - # Check that only valid table headers are used - for item in table_header: - if not item in VALID_ITEMS['header']: - raise Exception('Invalid column header {0} in argument table {1}'.format(item, table_name)) - # Locate mandatory column 'standard_name' - try: - standard_name_index = table_header.index('standard_name') - except ValueError: - raise Exception('Mandatory column standard_name not found in argument table {0}'.format(table_name)) - # DH* warn or raise error for old metadata format - logging.warn("Old metadata table found for table {}".format(table_name)) - #raise Exception("Old metadata table found for table {}".format(table_name)) - # *DH - # Get all of the variable information in table - end_of_table = False - line_number = header_line_number + 2 - while not end_of_table: - line = lines[line_number] - words = line.split('|') - if len(words) == 1: - if words[0].strip() == '!!': - end_of_table = True - else: - raise Exception('Encountered invalid line "{0}" in argument table {1}'.format(line, table_name)) + else: + words = lines[header_line_number].split() + if len(words) == 1 and words[0].strip() == '!!': + logging.info("Legacy extension - skip empty table for {}".format(table_name)) + end_of_table = True + line_number += 1 else: - var_items = [x.strip() for x in words[1:-1]] - if not len(var_items) == len(table_header): - raise Exception('Error parsing variable entry "{0}" in argument table {1}'.format(var_items, table_name)) - var_name = var_items[standard_name_index] - # Column standard_name cannot be left blank in scheme_tables - if not var_name: - raise Exception('Encountered line "{0}" without standard name in argument table {1}'.format(line, table_name)) - # Enforce CF standards: no dashes, no dots (underscores instead) - if "-" in var_name: - raise Exception("Invalid character '-' found in standard name {0} in table {1}".format(var_name, table_name)) - elif "." in var_name: - raise Exception("Invalid character '.' found in standard name {0} in table {1}".format(var_name, table_name)) - # - # Add standard_name to argument list for this subroutine - arguments[module_name][scheme_name][subroutine_name].append(var_name) - var = Var.from_table(table_header,var_items) - # Check for incompatible definitions with CCPP mandatory variables - if var_name in CCPP_MANDATORY_VARIABLES.keys() and not CCPP_MANDATORY_VARIABLES[var_name].compatible(var): - raise Exception('Entry for variable {0}'.format(var_name) + \ - ' in argument table of subroutine {0}'.format(subroutine_name) +\ - ' is incompatible with mandatory variable:\n' +\ - ' existing: {0}\n'.format(CCPP_MANDATORY_VARIABLES[var_name].print_debug()) +\ - ' vs. new: {0}'.format(var.print_debug())) - # Record the location of this variable: module, scheme, table - container = encode_container(module_name, scheme_name, table_name) - var.container = container - # Add variable to metadata dictionary - if not var_name in metadata.keys(): - metadata[var_name] = [var] - else: - for existing_var in metadata[var_name]: - if not existing_var.compatible(var): - raise Exception('New entry for variable {0}'.format(var_name) + \ - ' in argument table of subroutine {0}'.format(subroutine_name) +\ - ' is incompatible with existing entry:\n' +\ - ' existing: {0}\n'.format(existing_var.print_debug()) +\ - ' vs. new: {0}'.format(var.print_debug())) - metadata[var_name].append(var) - - line_number += 1 - - # After parsing entire metadata table for the subroutine, check that all mandatory CCPP variables are present - for var_name in CCPP_MANDATORY_VARIABLES.keys(): - if not var_name in arguments[module_name][scheme_name][subroutine_name]: - raise Exception('Mandatory CCPP variable {0} not declared in metadata table of subroutine {1}'.format( - var_name, subroutine_name)) + raise Exception("Invalid definition of metadata in file {0}: {1}".format(filename, words)) + + # After parsing entire metadata table for the subroutine, check that all + # mandatory CCPP variables are present - skip empty tables. + if arguments[module_name][scheme_name][subroutine_name]: + for var_name in CCPP_MANDATORY_VARIABLES.keys(): + if not var_name in arguments[module_name][scheme_name][subroutine_name]: + raise Exception('Mandatory CCPP variable {0} not declared in metadata table of subroutine {1}'.format( + var_name, subroutine_name)) # For CCPP-compliant files (i.e. files with metadata tables, perform additional checks) if len(metadata.keys()) > 0: diff --git a/scripts/metavar.py b/scripts/metavar.py index 27484691..9311af3c 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -340,6 +340,10 @@ class Var(object): >>> Var.get_prop('dimensions').valid_value(['Bob', 'Ray']) ['Bob', 'Ray'] + >>> Var.get_prop('active') + '.true.' + >>> Var.get_prop('active').valid_value('flag_for_aerosol_physics') + 'flag_for_aerosol_physics' >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm/s', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext())).get_prop_value('long_name') 'Hi mom' >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm/s', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext())).get_prop_value('intent') @@ -381,7 +385,9 @@ class Var(object): optional_in=True, default_in=False), VariableProperty('persistence', str, optional_in=True, valid_values_in=['timestep', 'run'], - default_in='timestep')] + default_in='timestep'), + VariableProperty('active', str, optional_in=True, + default_in='.true.')] # __var_props contains properties which are not in __spec_props __var_props = [VariableProperty('optional', bool, diff --git a/scripts/mkcap.py b/scripts/mkcap.py index 6eb38ccb..9170424c 100755 --- a/scripts/mkcap.py +++ b/scripts/mkcap.py @@ -28,11 +28,12 @@ def __init__(self, **kwargs): self._units = None self._local_name = None self._type = None - self._rank = None + self._dimensions = [] self._container = None self._kind = None self._intent = None self._optional = None + self._active = None self._target = None self._actions = { 'in' : None, 'out' : None } for key, value in kwargs.items(): @@ -83,19 +84,24 @@ def type(self): def type(self, value): self._type = value + @property + def dimensions(self): + '''Get the dimensions of the variable.''' + return self._dimensions + + @dimensions.setter + def dimensions(self, value): + if not type(value) is list: + raise TypeError('Invalid type for variable property dimensions, must be a list') + self._dimensions = value + @property def rank(self): '''Get the rank of the variable.''' - return self._rank - - @rank.setter - def rank(self, value): - if not isinstance(value, int): - raise TypeError('Invalid type for variable property rank, must be integer') - if (value == 0): - self._rank = '' + if len(self._dimensions) == 0: + return '' else: - self._rank = '('+ ','.join([':'] * value) +')' + return '('+ ','.join([':'] * len(self._dimensions)) +')' @property def kind(self): @@ -119,7 +125,7 @@ def intent(self, value): @property def optional(self): - '''Get the optional of the variable.''' + '''Get the optional attribute of the variable.''' return self._optional @optional.setter @@ -128,6 +134,17 @@ def optional(self, value): raise ValueError('Invalid value {0} for variable property optional'.format(value)) self._optional = value + @property + def active(self): + '''Get the active attribute of the variable.''' + return self._active + + @active.setter + def active(self, value): + if not isinstance(value, str): + raise ValueError('Invalid value {0} for variable property active, must be a string'.format(value)) + self._active = value + @property def target(self): '''Get the target of the variable.''' @@ -153,7 +170,7 @@ def actions(self): @actions.setter def actions(self, values): - if type(value)==dict: + if type(values)==dict: for key in values.keys(): if key in ['in', 'out'] and isstring(values[key]): self._actions[key] = values[key] @@ -260,29 +277,17 @@ def print_debug(self): units = {s.units} * local_name = {s.local_name} type = {s.type} * + dimensions = {s.dimensions} rank = {s.rank} * kind = {s.kind} * intent = {s.intent} optional = {s.optional} + active = {s.active} target = {s.target} container = {s.container} actions = {s.actions}''' return str.format(s=self) - @classmethod - def from_table(cls, columns, data): - var = cls() - var.standard_name = data[columns.index('standard_name')] - var.long_name = data[columns.index('long_name')] - var.units = data[columns.index('units')] - var.local_name = data[columns.index('local_name')] - var.rank = int(data[columns.index('rank')]) - var.type = data[columns.index('type')] - var.kind = data[columns.index('kind')] - var.intent = data[columns.index('intent')] - var.optional = data[columns.index('optional')] - return var - class CapsMakefile(object): header=''' diff --git a/scripts/mkdoc.py b/scripts/mkdoc.py index 45a7daf7..0f55f2aa 100755 --- a/scripts/mkdoc.py +++ b/scripts/mkdoc.py @@ -76,7 +76,7 @@ def metadata_to_html(metadata, model, filename): return success -def metadata_to_latex(metadata_define, metadata_request, pset_request, model, filename): +def metadata_to_latex(metadata_define, metadata_request, model, filename): """Create a LaTeX document with a table that lists each variable provided and/or requested. Uses the GMTB LaTeX templates and style definitons in gmtb.sty.""" @@ -132,11 +132,6 @@ def metadata_to_latex(metadata_define, metadata_request, pset_request, model, fi requested = '\\newline '.join(sorted(requested_list)) else: requested = 'NOT REQUESTED' - if var_name in pset_request.keys(): - pset_list = [ escape_tex(c) for c in pset_request[var_name] ] - pset = '\\newline '.join(sorted(pset_list)) - else: - pset = '' # Create output text = ''' @@ -151,7 +146,6 @@ def metadata_to_latex(metadata_define, metadata_request, pset_request, model, fi \\execout{{source }} & \\execout{{{target} }} \\\\ \\execout{{local\_name}} & \\execout{{{local_name} }} \\\\ \\execout{{requested }} & \\execout{{\\vtop{{{requested}}}}} \\\\ -\\execout{{physics set}} & \\execout{{\\vtop{{{set} }}}} \\\\ \\end{{tabular}} \\vspace{{4pt}} \\end{{samepage}}'''.format(standard_name=escape_tex(var.standard_name), standard_name_ref=var.standard_name, @@ -162,8 +156,7 @@ def metadata_to_latex(metadata_define, metadata_request, pset_request, model, fi kind=escape_tex(var.kind), target=target, local_name=local_name, - requested=requested, - set=pset) + requested=requested) latex += text # Footer latex += ''' diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index ed6fd6fd..287cec2a 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -7,12 +7,16 @@ import filecmp import logging import os +import re import sys import types import xml.etree.ElementTree as ET from common import encode_container from common import CCPP_ERROR_FLAG_VARIABLE, CCPP_ERROR_MSG_VARIABLE, CCPP_LOOP_COUNTER +from common import CCPP_BLOCK_NUMBER, CCPP_BLOCK_COUNT, CCPP_BLOCK_SIZES, CCPP_INTERNAL_VARIABLES +from common import CCPP_HORIZONTAL_DIMENSION, CCPP_HORIZONTAL_LOOP_EXTENT +from common import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX from common import CCPP_TYPE, STANDARD_VARIABLE_TYPES, STANDARD_CHARACTER_TYPE from common import CCPP_STATIC_API_MODULE, CCPP_STATIC_SUBROUTINE_NAME from mkcap import Var @@ -803,7 +807,8 @@ class Group(object): implicit none - integer :: ierr + ! Error handling + integer :: ierr {var_defs} @@ -893,7 +898,13 @@ def write(self, metadata_request, metadata_define, arguments): tmpvars = {} # body = '' + # Variable definitions automatically added for subroutines var_defs = '' + # List of manual variable definitions, for example for handling blocked data structures + var_defs_manual = [] + # Conditionals for variables (used or allocated only under certain conditions) + conditionals = {} + # for subcycle in self._subcycles: if subcycle.loop > 1 and ccpp_stage == 'run': body += ''' @@ -915,17 +926,95 @@ def write(self, metadata_request, metadata_define, arguments): error_check = '' args = '' length = 0 - # Extract all variables needed (including indices for components/slices of arrays) + + # First identify all dimensions needed to handle the arguments + # and add them to the list of required variables for the cap + additional_variables_required = [] for var_standard_name in arguments[module_name][scheme_name][subroutine_name]: + if not var_standard_name in metadata_define.keys(): + raise Exception('Variable {standard_name} not defined in host model metadata'.format( + standard_name=var_standard_name)) + var = metadata_define[var_standard_name][0] + # dim can be 'A', '1', '1:A', ... + for dim_expression in var.dimensions: + dims = dim_expression.split(':') + for dim in dims: + try: + dim = int(dim) + except ValueError: + if not dim in local_vars.keys() and \ + not dim in additional_variables_required + arguments[module_name][scheme_name][subroutine_name]: + logging.debug("Adding dimension {} for variable {}".format(dim, var_standard_name)) + additional_variables_required.append(dim) + + # If blocked data structures need to be converted, add necessary variables + if ccpp_stage in ['init', 'finalize'] and CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in var.local_name: + if not CCPP_BLOCK_COUNT in local_vars.keys() \ + and not CCPP_BLOCK_COUNT in additional_variables_required + arguments[module_name][scheme_name][subroutine_name]: + logging.debug("Adding variable {} for handling blocked data structures".format(CCPP_BLOCK_COUNT)) + additional_variables_required.append(CCPP_BLOCK_COUNT) + if not CCPP_HORIZONTAL_LOOP_EXTENT in local_vars.keys() \ + and not CCPP_HORIZONTAL_LOOP_EXTENT in additional_variables_required + arguments[module_name][scheme_name][subroutine_name]: + logging.debug("Adding variable {} for handling blocked data structures".format(CCPP_HORIZONTAL_LOOP_EXTENT)) + additional_variables_required.append(CCPP_HORIZONTAL_LOOP_EXTENT) + if not CCPP_HORIZONTAL_DIMENSION in local_vars.keys() \ + and not CCPP_HORIZONTAL_DIMENSION in additional_variables_required + arguments[module_name][scheme_name][subroutine_name]: + logging.debug("Adding variable {} for handling blocked data structures".format(CCPP_HORIZONTAL_DIMENSION)) + additional_variables_required.append(CCPP_HORIZONTAL_DIMENSION) + + # If the variable is only active/used under certain conditions, add necessary variables + # also record the conditional for later use in unit conversions / blocked data conversions. + if var.active == 'T': + conditional = '.true.' + elif var.active == 'F': + conditional = '.false.' + else: + # Convert conditional expression in standard_name format to local names known to the host model + conditional = '' + # Find all words in the conditional, for each of them look for a matching + # standard name in the list of known variables + items = FORTRAN_CONDITIONAL_REGEX.findall(var.active.lower()) + for item in items: + if item in FORTRAN_CONDITIONAL_REGEX_WORDS: + conditional += item + else: + # Detect integers, following Python's "easier to ask forgiveness than permission" mentality + try: + int(item) + conditional += item + except ValueError: + if not item in metadata_define.keys(): + raise Exception("Variable {} used in conditional for {} not known to host model".format( + item, var_standard_name)) + var2 = metadata_define[item][0] + conditional += var2.local_name + # Add to list of required variables for the cap + if not item in local_vars.keys() \ + and not item in additional_variables_required + arguments[module_name][scheme_name][subroutine_name]: + logging.debug("Adding variable {} for handling conditionals".format(item)) + additional_variables_required.append(item) + # Conditionals are identical per requirement, no need to test for consistency again + if not var_standard_name in conditionals.keys(): + conditionals[var_standard_name] = conditional + + # Extract all variables needed (including indices for components/slices of arrays) + for var_standard_name in additional_variables_required + arguments[module_name][scheme_name][subroutine_name]: # Pick the correct variable for this module/scheme/subroutine - # from the list of requested variable - for var in metadata_request[var_standard_name]: - if container == var.container: - break + # from the list of requested variable, if it is in that list + if var_standard_name in arguments[module_name][scheme_name][subroutine_name]: + for var in metadata_request[var_standard_name]: + if container == var.container: + break + # This is a dimension or required variable added automatically (e.g. for handling blocked data) + else: + # Create a copy of the variable in the metadata dictionary + # of host model variables and set necessary default values + var = copy.deepcopy(metadata_define[var_standard_name][0]) + var.intent = 'in' + var.optional = 'F' + if not var_standard_name in local_vars.keys(): - if not var_standard_name in metadata_define.keys(): - raise Exception('Variable {standard_name} not defined in host model metadata'.format( - standard_name=var_standard_name)) + # The full name of the variable as known to the host model var_local_name_define = metadata_define[var_standard_name][0].local_name # Break apart var_local_name_define into the different components (members of DDTs) @@ -933,11 +1022,13 @@ def write(self, metadata_request, metadata_define, arguments): (parent_local_name_define, parent_local_names_define_indices) = \ extract_parents_and_indices_from_local_name(var_local_name_define) + parent_standard_name = None + parent_var = None # Check for each of the derived parent local names as defined by the host model # if they are registered (i.e. if there is a standard name for it). Note that # the output of extract_parents_and_indices_from_local_name is stripped of any # array subset information, i.e. a local name 'Atm(:)%...' will produce a - # parent local name 'Atm'. Since the rank of tha parent variable is not known + # parent local name 'Atm'. Since the rank of the parent variable is not known # at this point and since the local name in the host model metadata table could # contain '(:)', '(:,:)', ... (up to the rank of the array), we search for the # maximum number of dimensions allowed by the Fortran standard. @@ -990,68 +1081,249 @@ def write(self, metadata_request, metadata_define, arguments): else: self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - # Record this information in the local_vars dictionary - local_vars[var_standard_name] = { - 'name' : metadata_define[var_standard_name][0].local_name, - 'kind' : metadata_define[var_standard_name][0].kind, - 'parent_standard_name' : parent_standard_name - } + # Record the parent information for this variable (with standard name var_standard_name) + if local_name_define == parent_local_name_define: + local_vars[var_standard_name] = { + 'name' : metadata_define[var_standard_name][0].local_name, + 'kind' : metadata_define[var_standard_name][0].kind, + 'parent_standard_name' : parent_standard_name + } - else: + # Reset parent to actual parent of the variable with standard name var_standard_name + if local_vars[var_standard_name]['parent_standard_name']: + parent_standard_name = local_vars[var_standard_name]['parent_standard_name'] + parent_var = metadata_define[parent_standard_name][0] + + elif local_vars[var_standard_name]['parent_standard_name']: parent_standard_name = local_vars[var_standard_name]['parent_standard_name'] + parent_var = metadata_define[parent_standard_name][0] # Update intent information if necessary if self.parents[ccpp_stage][parent_standard_name].intent == 'in' and not var.intent == 'in': self.parents[ccpp_stage][parent_standard_name].intent = 'inout' elif self.parents[ccpp_stage][parent_standard_name].intent == 'out' and not var.intent == 'out': self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - # Add necessary actions before/after while populating the subroutine's argument list + # The remainder of this loop deals with adding variables to the argument list + # for this subroutine, not required for the additional dimensions and variables + if not var_standard_name in arguments[module_name][scheme_name][subroutine_name]: + continue + + # kind_string is used for automated unit conversions, i.e. foo_kind_phys kind_string = '_' + local_vars[var_standard_name]['kind'] if local_vars[var_standard_name]['kind'] else '' - if var.actions['out']: + + # Convert blocked data in init and finalize steps + if ccpp_stage in ['init', 'finalize'] and CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name']: + # Reuse existing temporary variable, if possible if local_vars[var_standard_name]['name'] in tmpvars.keys(): # If the variable already has a local variable (tmpvar), reuse it tmpvar = tmpvars[local_vars[var_standard_name]['name']] + actions_in = tmpvar.actions['in'] + actions_out = tmpvar.actions['out'] + # DH* 2020-05-26 + raise Exception("Reusing temporary variables used for blocking has not been tested yet") + # *DH 2020-05-26 else: # Add a local variable (tmpvar) for this variable tmpvar_cnt += 1 tmpvar = copy.deepcopy(var) tmpvar.local_name = 'tmpvar{0}'.format(tmpvar_cnt) + # Only variables that contain a horizontal dimension are supported at this time. + if not tmpvar.dimensions: + #actions_in = ??? + #actions_out = ??? + raise Exception("Cannot handle blocked data for variables w/o a horizontal dimension: {}".format( + var_standard_name)) + else: + # Create string for allocating the temporary array by converting the dimensions + # (in standard_name format) to local names as known to the host model + alloc_dimensions = [] + for dim in tmpvar.dimensions: + # This is not supported/implemented: tmpvar would have one dimension less + # than the original array, and the metadata requesting the variable would + # not pass the initial test that host model variables and scheme variables + # have the same rank. + if dim == CCPP_BLOCK_NUMBER: + raise Exception("{} cannot be part of the dimensions of variable {}".format( + CCPP_BLOCK_NUMBER, var_standard_name)) + else: + # Handle dimensions like "A:B", "A:3", "-1:Z" + if ':' in dim: + dims = dim.split(':') + try: + dim0 = int(dims[0]) + except ValueError: + dim0 = metadata_define[dims[0]][0].local_name + try: + dim1 = int(dims[1]) + except ValueError: + dim1 = metadata_define[dims[1]][0].local_name + # Single dimensions + else: + dim0 = 1 + try: + dim1 = int(dim) + except ValueError: + dim1 = metadata_define[dim][0].local_name + alloc_dimensions.append('{}:{}'.format(dim0,dim1)) + + # Padding of additional dimensions - before and after the horizontal dimension; + # same as for scalars, variables w/o a horizontal dimension are not supported. + try: + hdim_index = tmpvar.dimensions.index(CCPP_HORIZONTAL_DIMENSION) + except ValueError: + raise Exception("Cannot handle blocked data for variables w/o a horizontal dimension: {}".format( + var_standard_name)) + dimpad_before = '' + ':,'*(len(tmpvar.dimensions[:hdim_index])) + dimpad_after = '' + ',:'*(len(tmpvar.dimensions[hdim_index+1:])) + # Add necessary local variables for looping over blocks + var_defs_manual.append('integer :: ib, nb') + # Define actions before, depending on intent + if var.intent in [ 'in', 'inout' ]: + actions_in = ''' + allocate({tmpvar}({dims})) + ib = 1 + do nb=1,{block_count} + {tmpvar}({dimpad_before}ib:ib+{block_size}-1{dimpad_after}) = {var} + ib = ib+{block_size} + end do +'''.format(tmpvar=tmpvar.local_name, + block_count=metadata_define[CCPP_BLOCK_COUNT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), + block_size=metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), + var=tmpvar.target.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), + dims=','.join(alloc_dimensions), + dimpad_before=dimpad_before, + dimpad_after=dimpad_after, + ) + else: + actions_in = ''' + allocate({tmpvar}({dims})) +'''.format(tmpvar=tmpvar.local_name, + dims=','.join(alloc_dimensions), + ) + # Define actions after, depending on intent + if var.intent in [ 'inout', 'out' ]: + actions_out = ''' + ib = 1 + do nb=1,{block_count} + {var} = {tmpvar}({dimpad_before}ib:ib+{block_size}-1{dimpad_after}) + ib = ib+{block_size} + end do + deallocate({tmpvar}) +'''.format(tmpvar=tmpvar.local_name, + block_count=metadata_define[CCPP_BLOCK_COUNT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), + block_size=metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), + var=tmpvar.target.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), + dimpad_before=dimpad_before, + dimpad_after=dimpad_after, + ) + else: + actions_out = ''' + deallocate({tmpvar}) +'''.format(tmpvar=tmpvar.local_name) + + # Set/update actions for this temporary variable + tmpvar.actions = {'in' : actions_in, 'out' : actions_out} tmpvars[local_vars[var_standard_name]['name']] = tmpvar - if var.rank: + + # Add unit conversions, if necessary + if var.actions['in']: + # Add unit conversion before entering the subroutine, after allocating the temporary + # array holding the non-blocked data and copying the blocked data to it + actions_in = actions_in + \ + ' {t} = {c}\n'.format(t=tmpvar.local_name, + c=var.actions['in'].format(var=tmpvar.local_name, + kind=kind_string)) + + if var.actions['out']: + # Add unit conversion after returning from the subroutine, before copying the non-blocked + # data back to the blocked data and deallocating the temporary array + actions_out = ' {t} = {c}\n'.format(t=tmpvar.local_name, + c=var.actions['out'].format(var=tmpvar.local_name, + kind=kind_string)) + \ + actions_out + + # Add the conditionals for the "before" operations + actions_before += ''' + if ({conditional}) then +{actions_in} + end if +'''.format(conditional=conditionals[var_standard_name], + actions_in=actions_in.rstrip('\n')) + # Add the conditionals for the "after" operations + actions_after += ''' + if ({conditional}) then +{actions_out} + end if +'''.format(conditional=conditionals[var_standard_name], + actions_out=actions_out.rstrip('\n')) + + # Add to argument list if required + if var_standard_name in arguments[module_name][scheme_name][subroutine_name]: + arg = '{local_name}={var_name},'.format(local_name=var.local_name, var_name=tmpvar.local_name) + + # Unit conversions without converting blocked data structures + elif var.actions['in'] or var.actions['out']: + actions_in = '' + actions_out = '' + if local_vars[var_standard_name]['name'] in tmpvars.keys(): + # If the variable already has a local variable (tmpvar), reuse it + tmpvar = tmpvars[local_vars[var_standard_name]['name']] + else: + # Add a local variable (tmpvar) for this variable + tmpvar_cnt += 1 + tmpvar = copy.deepcopy(var) + tmpvar.local_name = 'tmpvar{0}'.format(tmpvar_cnt) + tmpvars[local_vars[var_standard_name]['name']] = tmpvar + if tmpvar.rank: # Add allocate statement if the variable has a rank > 0 - actions_before += ' allocate({t}, source={v})\n'.format(t=tmpvar.local_name, - v=local_vars[var_standard_name]['name']) + actions_in += ' allocate({t}, source={v})\n'.format(t=tmpvar.local_name, + v=tmpvar.target) if var.actions['in']: # Add unit conversion before entering the subroutine - actions_before += ' {t} = {c}\n'.format(t=tmpvar.local_name, - c=var.actions['in'].format(var=local_vars[var_standard_name]['name'], + actions_in += ' {t} = {c}\n'.format(t=tmpvar.local_name, + c=var.actions['in'].format(var=tmpvar.target, + kind=kind_string)) + if var.actions['out']: + # Add unit conversion after returning from the subroutine + actions_out += ' {v} = {c}\n'.format(v=tmpvar.target, + c=var.actions['out'].format(var=tmpvar.local_name, kind=kind_string)) - # Add unit conversion after returning from the subroutine - actions_after += ' {v} = {c}\n'.format(v=local_vars[var_standard_name]['name'], - c=var.actions['out'].format(var=tmpvar.local_name, - kind=kind_string)) - if var.rank: + if tmpvar.rank: # Add deallocate statement if the variable has a rank > 0 - actions_after += ' deallocate({t})\n'.format(t=tmpvar.local_name) - # Add to argument list - arg = '{local_name}={var_name},'.format(local_name=var.local_name, - var_name=tmpvar.local_name) - elif var.actions['in']: - # Add to argument list, call action in argument list - action = var.actions['in'].format(var=local_vars[var_standard_name]['name'], - kind=kind_string) - arg = '{local_name}={action},'.format(local_name=var.local_name, action=action) - else: - # Add to argument list + actions_out += ' deallocate({t})\n'.format(t=tmpvar.local_name) + + # Add the conditionals for the "before" operations + actions_before += ''' + if ({conditional}) then +{actions_in} + end if +'''.format(conditional=conditionals[var_standard_name], + actions_in=actions_in.rstrip('\n')) + # Add the conditionals for the "after" operations + actions_after += ''' + if ({conditional}) then +{actions_out} + end if +'''.format(conditional=conditionals[var_standard_name], + actions_out=actions_out.rstrip('\n')) + + # Add to argument list if required + if var_standard_name in arguments[module_name][scheme_name][subroutine_name]: + arg = '{local_name}={var_name},'.format(local_name=var.local_name, var_name=tmpvar.local_name) + + # Ordinary variables, no blocked data or unit conversions + elif var_standard_name in arguments[module_name][scheme_name][subroutine_name]: + # Add to argument list if required arg = '{local_name}={var_name},'.format(local_name=var.local_name, var_name=local_vars[var_standard_name]['name']) + else: + arg = '' args += arg length += len(arg) # Split args so that lines don't exceed 260 characters (for PGI) if length > 70 and not var_standard_name == arguments[module_name][scheme_name][subroutine_name][-1]: args += ' &\n ' length = 0 - args = args.rstrip(',') subroutine_call = ''' {actions_before} @@ -1084,6 +1356,10 @@ def write(self, metadata_request, metadata_define, arguments): self.parents[ccpp_stage], metadata_define, tmpvars.values()) sub_argument_list = create_argument_list_wrapped(self.arguments[ccpp_stage]) + # Remove duplicates from additional manual variable definitions + var_defs_manual = list(set(var_defs_manual)) + + # Write cap subroutine = self._suite + '_' + self._name + '_' + ccpp_stage + '_cap' self._subroutines.append(subroutine) # Test and set blocks for initialization status @@ -1101,7 +1377,7 @@ def write(self, metadata_request, metadata_define, arguments): module_use='\n '.join(sub_module_use), initialized_test_block=initialized_test_block, initialized_set_block=initialized_set_block, - var_defs='\n '.join(sub_var_defs), + var_defs='\n '.join(sub_var_defs + var_defs_manual), body=body) # Write output to stdout or file