RRTMGP in CCPP (updates to ccpp_prebuild.py)

scripts/ccpp_prebuild.py

@@ -13,10 +13,12 @@
 # CCPP framework imports
 from common import encode_container, decode_container, decode_container_as_dict, execute
 from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE
+from common import STANDARD_VARIABLE_TYPES, STANDARD_INTEGER_TYPE, CCPP_TYPE
 from common import split_var_name_and_array_reference
 from metadata_parser import merge_dictionaries, parse_scheme_tables, parse_variable_tables
 from mkcap import Cap, CapsMakefile, CapsCMakefile, CapsSourcefile, \
-                  SchemesMakefile, SchemesCMakefile, SchemesSourcefile
+                  SchemesMakefile, SchemesCMakefile, SchemesSourcefile, \
+                  TypedefsMakefile, TypedefsCMakefile, TypedefsSourcefile
 from mkdoc import metadata_to_html, metadata_to_latex
 from mkstatic import API, Suite, Group
 
@@ -80,6 +82,9 @@ def import_config(configfile, builddir):
 
     # Definitions in host-model dependent CCPP prebuild config script
     config['variable_definition_files'] = ccpp_prebuild_config.VARIABLE_DEFINITION_FILES
+    config['typedefs_makefile']         = ccpp_prebuild_config.TYPEDEFS_MAKEFILE.format(build_dir=builddir)
+    config['typedefs_cmakefile']        = ccpp_prebuild_config.TYPEDEFS_CMAKEFILE.format(build_dir=builddir)
+    config['typedefs_sourcefile']       = ccpp_prebuild_config.TYPEDEFS_SOURCEFILE.format(build_dir=builddir)
     config['scheme_files']              = ccpp_prebuild_config.SCHEME_FILES
     config['scheme_files_dependencies'] = ccpp_prebuild_config.SCHEME_FILES_DEPENDENCIES
     config['schemes_makefile']          = ccpp_prebuild_config.SCHEMES_MAKEFILE.format(build_dir=builddir)
@@ -629,6 +634,63 @@ def generate_static_api(suites, static_api_dir):
     os.chdir(BASEDIR)
     return (success, api)
 
+def generate_typedefs_makefile(metadata_define, typedefs_makefile, typedefs_cmakefile, typedefs_sourcefile):
+    """Generate list of Fortran modules containing CCPP type/kind definitions,
+       and create makefile/cmakefile snippets for host model build system"""
+    logging.info('Generating list of Fortran modules containing CCPP type definitions ...')
+    success = True
+    #
+    typedefs = []
+    # (1) Search for type definitions in the metadata, defined by:
+    #    (a) the type not being a standard type, and
+    #    (b) the type not being the CCPP framework internal type
+    #    (c) the standard_name being identical to the type name
+    # (2) Search for kind definitions in the metadata, defined by:
+    #    (a) the standard_name starting with "kind_"
+    #    (b) the type being integer and the units being none
+    for key in metadata_define.keys():
+        # derived data types
+        if not metadata_define[key][0].type in STANDARD_VARIABLE_TYPES and \
+                not metadata_define[key][0].type == CCPP_TYPE and \
+                metadata_define[key][0].type == metadata_define[key][0].standard_name:
+            container = decode_container_as_dict(metadata_define[key][0].container)
+            if not 'MODULE' in container.keys():
+                logging.error("Invalid type definition for type {}: {}".format(metadata_define[key][0].type, metadata_define[key][0].print_debug()))
+                success = False
+                continue
+            # Fortran modules are lowercase and have the ending ".mod"
+            typedef_fortran_module = "{}.mod".format(container['MODULE']).lower()
+            if not typedef_fortran_module in typedefs:
+                typedefs.append(typedef_fortran_module)
+        # kind definitions
+        elif metadata_define[key][0].standard_name.startswith("kind_") and \
+                metadata_define[key][0].type == STANDARD_INTEGER_TYPE and \
+                metadata_define[key][0].units == 'none':
+            container = decode_container_as_dict(metadata_define[key][0].container)
+            if not 'MODULE' in container.keys():
+                logging.error("Invalid kind definition for kind {}: {}".format(metadata_define[key][0].type, metadata_define[key][0].print_debug()))
+                success = False
+                continue
+            # Fortran modules are lowercase and have the ending ".mod"
+            typedef_fortran_module = "{}.mod".format(container['MODULE']).lower()
+            if not typedef_fortran_module in typedefs:
+                typedefs.append(typedef_fortran_module)
+
+    logging.info('Generating typedefs makefile/cmakefile snippet ...')
+    # Write the Fortran modules without path - the build system knows where they are
+    makefile = TypedefsMakefile()
+    makefile.filename = typedefs_makefile
+    cmakefile = TypedefsCMakefile()
+    cmakefile.filename = typedefs_cmakefile
+    sourcefile = TypedefsSourcefile()
+    sourcefile.filename = typedefs_sourcefile
+    makefile.write(typedefs)
+    cmakefile.write(typedefs)
+    sourcefile.write(typedefs)
+    logging.info('Added {0} typedefs to {1}, {2}, {3}'.format(
+           len(typedefs), makefile.filename, cmakefile.filename, sourcefile.filename))
+    return success
+
 def generate_schemes_makefile(schemes, schemes_makefile, schemes_cmakefile, schemes_sourcefile):
     """Generate makefile/cmakefile snippets for all schemes."""
     logging.info('Generating schemes makefile/cmakefile snippet ...')
@@ -639,20 +701,13 @@ def generate_schemes_makefile(schemes, schemes_makefile, schemes_cmakefile, sche
     cmakefile.filename = schemes_cmakefile
     sourcefile = SchemesSourcefile()
     sourcefile.filename = schemes_sourcefile
-    # Adjust relative file path to schemes from caps makefile
-    schemes_with_path = []
-    schemes_with_abspath = []
-    schemes_makefile_dir = os.path.split(os.path.abspath(schemes_makefile))[0]
-    for scheme in schemes:
-        (scheme_filepath, scheme_filename) = os.path.split(os.path.abspath(scheme))
-        relative_path = './{0}'.format(os.path.relpath(scheme_filepath, schemes_makefile_dir))
-        schemes_with_path.append(os.path.join(relative_path, scheme_filename))
-        schemes_with_abspath.append(os.path.abspath(scheme))
+    # Generate list of schemes with absolute path
+    schemes_with_abspath = [ os.path.abspath(scheme) for scheme in schemes ]
     makefile.write(schemes_with_abspath)
     cmakefile.write(schemes_with_abspath)
     sourcefile.write(schemes_with_abspath)
     logging.info('Added {0} schemes to {1}, {2}, {3}'.format(
-           len(schemes_with_path), makefile.filename, cmakefile.filename, sourcefile.filename))
+           len(schemes_with_abspath), makefile.filename, cmakefile.filename, sourcefile.filename))
     return success
 
 def generate_caps_makefile(caps, caps_makefile, caps_cmakefile, caps_sourcefile, caps_dir):
@@ -665,16 +720,13 @@ def generate_caps_makefile(caps, caps_makefile, caps_cmakefile, caps_sourcefile,
     cmakefile.filename = caps_cmakefile
     sourcefile = CapsSourcefile()
     sourcefile.filename = caps_sourcefile
-    # Adjust relative file path to schemes from caps makefile
-    caps_makefile_dir = os.path.split(os.path.abspath(caps_makefile))[0]
-    relative_path = './{0}'.format(os.path.relpath(caps_dir, caps_makefile_dir))
-    caps_with_path = [ os.path.join(relative_path, cap) for cap in caps]
-    caps_with_abspath = [ os.path.abspath(os.path.join(caps_dir, cap)) for cap in caps]
+    # Generate list of caps with absolute path
+    caps_with_abspath = [ os.path.abspath(os.path.join(caps_dir, cap)) for cap in caps ]
     makefile.write(caps_with_abspath)
     cmakefile.write(caps_with_abspath)
     sourcefile.write(caps_with_abspath)
-    logging.info('Added {0} auto-generated caps to {1} and {2}'.format(
-                          len(caps_with_path), makefile.filename, cmakefile.filename))
+    logging.info('Added {0} auto-generated caps to {1} and {2}, {3}'.format(
+           len(caps_with_abspath), makefile.filename, cmakefile.filename, sourcefile.filename))
     return success
 
 def main():
@@ -780,8 +832,14 @@ def main():
             if not success:
                 raise Exception('Call to generate_include_files failed.')
 
-    # Add filenames of schemes to makefile - add dependencies for schemes
-    success = generate_schemes_makefile(config['scheme_files_dependencies'] + config['scheme_files'].keys(),
+    # Add Fortran module files of typedefs to makefile/cmakefile/shell script
+    success = generate_typedefs_makefile(metadata_define, config['typedefs_makefile'],
+                                         config['typedefs_cmakefile'], config['typedefs_sourcefile'])
+    if not success:
+        raise Exception('Call to generate_typedefs_makefile failed.')
+
+    # Add filenames of schemes to makefile/cmakefile/shell script - add dependencies for schemes
+    success = generate_schemes_makefile(config['scheme_files_dependencies'] + list(config['scheme_files'].keys()),
                                         config['schemes_makefile'], config['schemes_cmakefile'],
                                         config['schemes_sourcefile'])
     if not success:
@@ -808,7 +866,7 @@ def main():
         if not success:
             raise Exception('Call to generate_scheme_caps failed.')
 
-    # Add filenames of caps to makefile
+    # Add filenames of caps to makefile/cmakefile/shell script
     if static:
         all_caps = suite_and_group_caps
     else:

scripts/common.py

@@ -33,8 +33,9 @@
     CCPP_THREAD_NUMBER       : 'cdata%thrd_no',
     }
 
-STANDARD_VARIABLE_TYPES = [ 'character', 'integer', 'logical', 'real' ]
 STANDARD_CHARACTER_TYPE = 'character'
+STANDARD_INTEGER_TYPE = 'integer'
+STANDARD_VARIABLE_TYPES = [ STANDARD_CHARACTER_TYPE, STANDARD_INTEGER_TYPE, 'logical', 'real' ]
 
 # For static build
 CCPP_STATIC_API_MODULE = 'ccpp_static_api'
@@ -126,7 +127,7 @@ def decode_container(container):
     items = container.split(' ')
     if not len(items) in [1, 2, 3]:
         raise Exception("decode_container not implemented for {0} items".format(len(items)))
-    for i in xrange(len(items)):
+    for i in range(len(items)):
         items[i] = items[i][:items[i].find('_')] + ' ' + items[i][items[i].find('_')+1:]
     return ' '.join(items)
 
@@ -139,7 +140,7 @@ def decode_container_as_dict(container):
     if not len(items) in [1, 2, 3]:
         raise Exception("decode_container not implemented for {0} items".format(len(items)))
     itemsdict = {}
-    for i in xrange(len(items)):
+    for i in range(len(items)):
         key, value = (items[i][:items[i].find('_')], items[i][items[i].find('_')+1:])
         itemsdict[key] = value
     return itemsdict

scripts/conversion_tools/__init__.py

@@ -2,7 +2,49 @@
 """
 
 __all__ = [
-    'units',
+    'mm__to__m',
+    'm__to__mm',
+    'um__to__m',
+    'm__to__um',
+    'm__to__km',
+    'km__to__m',
+    'mm__to__km',
+    'km__to__mm',
+    's__to__min',
+    'min__to__s',
+    's__to__h',
+    'h__to__s',
+    'h__to__d',
+    'd__to__h',
+    's__to__d',
+    'd__to__s',
+    'Pa__to__hPa',
+    'hPa__to__Pa',
+    'm_s_minus_1__to__km_h_minus_1',
+    'km_h_minus_1__to__m_s_minus_1',
+    'W_m_minus_2__to__erg_cm_minus_2_s_minus_1',
+    'erg_cm_minus_2_s_minus_1__to__W_m_minus_2',
     ]
 
-import unit_conversion
+from .unit_conversion import mm__to__m
+from .unit_conversion import m__to__mm
+from .unit_conversion import um__to__m
+from .unit_conversion import m__to__um
+from .unit_conversion import m__to__km
+from .unit_conversion import km__to__m
+from .unit_conversion import mm__to__km
+from .unit_conversion import km__to__mm
+from .unit_conversion import s__to__min
+from .unit_conversion import min__to__s
+from .unit_conversion import s__to__h
+from .unit_conversion import h__to__s
+from .unit_conversion import h__to__d
+from .unit_conversion import d__to__h
+from .unit_conversion import s__to__d
+from .unit_conversion import d__to__s
+from .unit_conversion import Pa__to__hPa
+from .unit_conversion import hPa__to__Pa
+from .unit_conversion import m_s_minus_1__to__km_h_minus_1
+from .unit_conversion import km_h_minus_1__to__m_s_minus_1
+from .unit_conversion import W_m_minus_2__to__erg_cm_minus_2_s_minus_1
+from .unit_conversion import erg_cm_minus_2_s_minus_1__to__W_m_minus_2

scripts/metadata_parser.py

@@ -190,7 +190,7 @@ def parse_variable_tables(filename):
 
     lines = []
     buffer = ''
-    for i in xrange(len(file_lines)):
+    for i in range(len(file_lines)):
         line = file_lines[i].rstrip('\n').strip()
         # Skip empty lines
         if line == '' or line == '&':
@@ -498,7 +498,7 @@ def parse_scheme_tables(filename):
     lines = []
     original_line_numbers = []
     buffer = ''
-    for i in xrange(len(file_lines)):
+    for i in range(len(file_lines)):
         line = file_lines[i].rstrip('\n').strip()
         # Skip empty lines
         if line == '' or line == '&':

scripts/metadata_table.py

@@ -517,7 +517,7 @@ def parse_metadata_file(cls, filename):
         mheaders = list()
         with open(filename, 'r') as file:
             fin_lines = file.readlines()
-            for index in xrange(len(fin_lines)):
+            for index in range(len(fin_lines)):
                 fin_lines[index] = fin_lines[index].rstrip('\n')
             # End for
         # End with