ccpp_prebuild: ccpp stub and various bugfixes

scripts/ccpp_prebuild.py

@@ -95,13 +95,14 @@ def import_config(configfile, builddir):
     config['caps_cmakefile']            = ccpp_prebuild_config.CAPS_CMAKEFILE.format(build_dir=builddir)
     config['caps_sourcefile']           = ccpp_prebuild_config.CAPS_SOURCEFILE.format(build_dir=builddir)
     config['caps_dir']                  = ccpp_prebuild_config.CAPS_DIR.format(build_dir=builddir)
-    config['suites_dir']                = ccpp_prebuild_config.SUITES_DIR
+    config['suites_dir']                = ccpp_prebuild_config.SUITES_DIR.format(build_dir=builddir)
     config['host_model']                = ccpp_prebuild_config.HOST_MODEL_IDENTIFIER
     config['html_vartable_file']        = ccpp_prebuild_config.HTML_VARTABLE_FILE.format(build_dir=builddir)
     config['latex_vartable_file']       = ccpp_prebuild_config.LATEX_VARTABLE_FILE.format(build_dir=builddir)
-    # Location of static API file, and shell script to source
+    # Location of static API file, shell script to source, cmake include file
     config['static_api_dir']            = ccpp_prebuild_config.STATIC_API_DIR.format(build_dir=builddir)
-    config['static_api_srcfile']        = ccpp_prebuild_config.STATIC_API_SRCFILE.format(build_dir=builddir)
+    config['static_api_sourcefile']     = ccpp_prebuild_config.STATIC_API_SOURCEFILE.format(build_dir=builddir)
+    config['static_api_cmakefile']      = ccpp_prebuild_config.STATIC_API_CMAKEFILE.format(build_dir=builddir)
 
     # To handle new metadata: import DDT references (if exist)
     try:
@@ -146,7 +147,7 @@ def clean_files(config):
         config['latex_vartable_file'],
         os.path.join(config['caps_dir'], 'ccpp_*_cap.F90'),
         os.path.join(config['static_api_dir'], '{api}.F90'.format(api=CCPP_STATIC_API_MODULE)),
-        config['static_api_srcfile'],
+        config['static_api_sourcefile'],
         ]
     # Not very pythonic, but the easiest way w/o importing another Python module
     cmd = 'rm -vf {0}'.format(' '.join(files_to_remove))
@@ -328,6 +329,19 @@ def collect_physics_subroutines(scheme_files):
     os.chdir(BASEDIR)
     return (success, metadata_request, arguments_request, dependencies_request, schemes_in_files)
 
+def check_schemes_in_suites(arguments, suites):
+    """Check that all schemes that are requested in the suites exist"""
+    success = True
+    logging.info("Checking for existence of schemes in suites ...")
+    for suite in suites:
+        for group in suite.groups:
+            for subcycle in group.subcycles:
+                for scheme_name in subcycle.schemes:
+                    if not scheme_name in arguments.keys():
+                        success = False
+                        logging.critical("Scheme {} in suite {} cannot be found".format(scheme_name, suite.name))
+    return success
+
 def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites):
     """Remove all variables from metadata that are not used in the given suite;
     also remove information on argument lists, dependencies and schemes in files"""
@@ -698,6 +712,11 @@ def main():
     if not success:
         raise Exception('Call to collect_physics_subroutines failed.')
 
+    # Check that the schemes requested in the suites exist
+    success = check_schemes_in_suites(arguments_request, suites)
+    if not success:
+        raise Exception('Call to check_schemes_in_suites failed.')
+
     # Filter metadata/arguments - remove whatever is not included in suite definition files
     (success, metadata_request, arguments_request, dependencies_request, schemes_in_files) = filter_metadata(
                          metadata_request, arguments_request, dependencies_request, schemes_in_files, suites)
@@ -739,12 +758,16 @@ def main():
         raise Exception('Call to generate_suite_and_group_caps failed.')
 
     (success, api) = generate_static_api(suites, config['static_api_dir'])
-    if not success: 
+    if not success:
         raise Exception('Call to generate_static_api failed.')
 
-    success = api.write_sourcefile(config['static_api_srcfile'])
-    if not success: 
-        raise Exception("Writing API sourcefile {sourcefile} failed".format(sourcefile=config['static_api_srcfile']))
+    success = api.write_includefile(config['static_api_sourcefile'], type='shell')
+    if not success:
+        raise Exception("Writing API sourcefile {sourcefile} failed".format(sourcefile=config['static_api_sourcefile']))
+
+    success = api.write_includefile(config['static_api_cmakefile'], type='cmake')
+    if not success:
+        raise Exception("Writing API cmakefile {cmakefile} failed".format(cmakefile=config['static_api_cmakefile']))
 
     # Add filenames of caps to makefile/cmakefile/shell script
     all_caps = suite_and_group_caps

scripts/mkstatic.py

@@ -406,7 +406,7 @@ def write(self):
                 self.update_api = True
         return
 
-    def write_sourcefile(self, source_filename):
+    def write_includefile(self, source_filename, type):
         success = True
         filepath = os.path.split(source_filename)[0]
         if filepath and not os.path.isdir(filepath):
@@ -422,14 +422,30 @@ def write_sourcefile(self, source_filename):
         else:
             write_to_test_file = False
             f = open(source_filename, 'w')
-        # Contents of shell/source file
-        contents = """# The CCPP static API is defined here.
+
+        if type == 'shell':
+            # Contents of shell/source file
+            contents = """# The CCPP static API is defined here.
 #
 # This file is auto-generated using ccpp_prebuild.py
 # at compile time, do not edit manually.
 #
 export CCPP_STATIC_API=\"{filename}\"
 """.format(filename=os.path.abspath(os.path.join(self.directory,self.filename)))
+        elif type == 'cmake':
+            # Contents of cmake include file
+            contents = """# The CCPP static API is defined here.
+#
+# This file is auto-generated using ccpp_prebuild.py
+# at compile time, do not edit manually.
+#
+set(API \"{filename}\")
+""".format(filename=os.path.abspath(os.path.join(self.directory,self.filename)))
+        else:
+            logging.error('Encountereed unknown type of file "{type}" when writing include file for static API'.format(type=type))
+            success = False
+            return
+
         f.write(contents)
         f.close()
         # See comment above on updating the API or not
@@ -903,7 +919,10 @@ def write(self, metadata_request, metadata_define, arguments, debug):
             standard_name_by_local_name_define[metadata_define[standard_name][0].local_name] = standard_name
 
         # First get target names of standard CCPP variables for subcycling and error handling
-        ccpp_loop_counter_target_name = metadata_request[CCPP_LOOP_COUNTER][0].target
+        if CCPP_LOOP_COUNTER in metadata_request.keys():
+            ccpp_loop_counter_target_name = metadata_request[CCPP_LOOP_COUNTER][0].target
+        else:
+            ccpp_loop_counter_target_name = None
         if CCPP_LOOP_EXTENT in metadata_request.keys():
             ccpp_loop_extent_target_name = metadata_request[CCPP_LOOP_EXTENT][0].target
         else:
@@ -1483,7 +1502,7 @@ def write(self, metadata_request, metadata_define, arguments, debug):
       end do
       end associate
 '''
-                    else:
+                    elif ccpp_loop_counter_target_name:
                         subcycle_body_prefix += '''
       {loop_var_name} = 1\n'''.format(loop_var_name=ccpp_loop_counter_target_name)
 

stub/CMakeLists.txt

@@ -0,0 +1,84 @@
+#------------------------------------------------------------------------------
+cmake_minimum_required(VERSION 3.0)
+
+project(ccppstub
+        VERSION 1.0.0
+        LANGUAGES Fortran)
+
+#------------------------------------------------------------------------------
+# Request a static build
+option(BUILD_SHARED_LIBS "Build a shared library" OFF)
+
+#------------------------------------------------------------------------------
+# Set the sources: physics type definitions
+set(TYPEDEFS $ENV{CCPP_TYPEDEFS})
+if(TYPEDEFS)
+  message(STATUS "Got CCPP TYPEDEFS from environment variable: ${TYPEDEFS}")
+else(TYPEDEFS)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/CCPP_TYPEDEFS.cmake)
+  message(STATUS "Got CCPP TYPEDEFS from cmakefile include file: ${TYPEDEFS}")
+endif(TYPEDEFS)
+
+# Generate list of Fortran modules from the CCPP type
+# definitions that need need to be installed
+foreach(typedef_module ${TYPEDEFS})
+    list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module})
+endforeach()
+
+#------------------------------------------------------------------------------
+# Set the sources: physics schemes
+set(SCHEMES $ENV{CCPP_SCHEMES})
+if(SCHEMES)
+  message(STATUS "Got CCPP SCHEMES from environment variable: ${SCHEMES}")
+else(SCHEMES)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/CCPP_SCHEMES.cmake)
+  message(STATUS "Got CCPP SCHEMES from cmakefile include file: ${SCHEMES}")
+endif(SCHEMES)
+
+# Set the sources: physics scheme caps
+set(CAPS $ENV{CCPP_CAPS})
+if(CAPS)
+  message(STATUS "Got CCPP CAPS from environment variable: ${CAPS}")
+else(CAPS)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/CCPP_CAPS.cmake)
+  message(STATUS "Got CCPP CAPS from cmakefile include file: ${CAPS}")
+endif(CAPS)
+
+# Set the sources: physics scheme caps
+set(API $ENV{CCPP_API})
+if(API)
+  message(STATUS "Got CCPP API from environment variable: ${API}")
+else(API)
+  include(${CMAKE_CURRENT_SOURCE_DIR}/CCPP_API.cmake)
+  message(STATUS "Got CCPP API from cmakefile include file: ${API}")
+endif(API)
+
+#------------------------------------------------------------------------------
+add_library(ccppstub STATIC ${SCHEMES} ${CAPS} ${API})
+# Generate list of Fortran modules from defined sources
+foreach(source_f90 ${CAPS} ${API})
+    get_filename_component(tmp_source_f90 ${source_f90} NAME)
+    string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90})
+    string(TOLOWER ${tmp_module_f90} module_f90)
+    list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90})
+endforeach()
+
+set_target_properties(ccppstub PROPERTIES VERSION ${PROJECT_VERSION}
+                      SOVERSION ${PROJECT_VERSION_MAJOR})
+
+# Define where to install the library
+install(TARGETS ccppstub
+        EXPORT ccppstub-targets
+        ARCHIVE DESTINATION lib
+        LIBRARY DESTINATION lib
+        RUNTIME DESTINATION lib
+)
+# Export our configuration
+install(EXPORT ccppstub-targets
+        FILE ccppstub-config.cmake
+        DESTINATION lib/cmake
+)
+# Define where to install the C headers and Fortran modules
+#install(FILES ${HEADERS_C} DESTINATION include)
+install(FILES ${MODULES_F90} DESTINATION include)
+

stub/README.md

@@ -0,0 +1,12 @@
+# How to build the stub
+
+Note: build is in-source for now
+
+1. Set compiler environment as appropriate for your system
+2. Run the following commands:
+```
+cd stub
+../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py
+cmake . 2>&1 | tee log.cmake
+make 2>&1 | tee log.make
+```

stub/ccpp_prebuild_config.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+# CCPP prebuild config for GFDL Finite-Volume Cubed-Sphere Model (FV3)
+
+
+###############################################################################
+# Definitions                                                                 #
+###############################################################################
+
+HOST_MODEL_IDENTIFIER = "FV3"
+
+# Add all files with metadata tables on the host model side and in CCPP,
+# relative to basedir = top-level directory of host model. This includes
+# kind and type definitions used in CCPP physics. Also add any internal
+# dependencies of these files to the list.
+VARIABLE_DEFINITION_FILES = [
+    # actual variable definition files
+    '../src/ccpp_types.F90',
+    'data.F90',
+    ]
+
+TYPEDEFS_NEW_METADATA = {
+    'ccpp_types' : {
+        'ccpp_types' : '',
+        'ccpp_t' : 'ccpp_data',
+        },
+    }
+
+# Add all physics scheme files relative to basedir
+SCHEME_FILES = [
+    'stub.F90',
+    ]
+
+# Default build dir, relative to current working directory,
+# if not specified as command-line argument
+DEFAULT_BUILD_DIR = '.'
+
+# Auto-generated makefile/cmakefile snippets that contain all type definitions
+TYPEDEFS_MAKEFILE   = '{build_dir}/CCPP_TYPEDEFS.mk'
+TYPEDEFS_CMAKEFILE  = '{build_dir}/CCPP_TYPEDEFS.cmake'
+TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh'
+
+# Auto-generated makefile/cmakefile snippets that contain all schemes
+SCHEMES_MAKEFILE   = '{build_dir}/CCPP_SCHEMES.mk'
+SCHEMES_CMAKEFILE  = '{build_dir}/CCPP_SCHEMES.cmake'
+SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh'
+
+# Auto-generated makefile/cmakefile snippets that contain all caps
+CAPS_MAKEFILE   = '{build_dir}/CCPP_CAPS.mk'
+CAPS_CMAKEFILE  = '{build_dir}/CCPP_CAPS.cmake'
+CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh'
+
+# Directory where to put all auto-generated physics caps
+CAPS_DIR = '{build_dir}'
+
+# Directory where the suite definition files are stored
+SUITES_DIR = '{build_dir}'
+
+# Optional arguments - only required for schemes that use
+# optional arguments. ccpp_prebuild.py will throw an exception
+# if it encounters a scheme subroutine with optional arguments
+# if no entry is made here. Possible values are: 'all', 'none',
+# or a list of standard_names: [ 'var1', 'var3' ].
+OPTIONAL_ARGUMENTS = {}
+
+# Directory where to write static API to
+STATIC_API_DIR = '{build_dir}'
+STATIC_API_CMAKEFILE  = '{build_dir}/CCPP_API.cmake'
+STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh'
+
+# Directory for writing HTML pages generated from metadata files
+METADATA_HTML_OUTPUT_DIR = '{build_dir}'
+
+# HTML document containing the model-defined CCPP variables
+HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_STUB.html'
+
+# LaTeX document containing the provided vs requested CCPP variables
+LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_STUB.tex'

stub/data.F90

@@ -0,0 +1,17 @@
+module data
+
+!! \section arg_table_data Argument Table
+!! \htmlinclude data.html
+!!
+
+    use ccpp_types,    only: ccpp_t
+
+    implicit none
+
+    private
+
+    public ccpp_data
+
+    type(ccpp_t), save, target :: ccpp_data
+
+end module data

stub/data.meta

@@ -0,0 +1,14 @@
+[ccpp-table-properties]
+  name = data
+  type = module
+  dependencies =
+
+[ccpp-arg-table]
+  name = data
+  type = module
+[ccpp_data]
+  standard_name = ccpp_t_instance
+  long_name = instance of derived data type ccpp_t
+  units = DDT
+  dimensions = ()
+  type = ccpp_t

stub/stub.F90

@@ -0,0 +1,35 @@
+!>\file stub.F90
+!! This file contains a stub CCPP scheme that does nothing
+!! except requesting the minimum, mandatory variables.
+
+module stub
+
+   implicit none
+   private
+   public :: stub_init, stub_finalize
+
+   contains
+
+!! \section arg_table_stub_init Argument Table
+!! \htmlinclude stub_init.html
+!!
+   subroutine stub_init(errmsg, errflg)
+      character(len=*), intent(out) :: errmsg
+      integer,          intent(out) :: errflg
+      ! Initialize CCPP error handling variables
+      errmsg = ''
+      errflg = 0
+   end subroutine stub_init
+
+!! \section arg_table_stub_finalize Argument Table
+!! \htmlinclude stub_finalize.html
+!!
+   subroutine stub_finalize(errmsg, errflg)
+      character(len=*), intent(out) :: errmsg
+      integer,          intent(out) :: errflg
+      ! Initialize CCPP error handling variables
+      errmsg = ''
+      errflg = 0
+   end subroutine stub_finalize
+
+end module stub