From e55992f7e301afa46dc509ad1928e4ab63d942ab Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 23 Feb 2022 20:44:47 -0700 Subject: [PATCH 1/2] Minor cleanup and documentation updates for CCPP; split CCPP finalize into physics_finalize and (framework) finalize --- .gitmodules | 6 ++-- atmos_model.F90 | 5 ++- ccpp/CMakeLists.txt | 13 -------- ccpp/config/ccpp_prebuild_config.py | 3 +- ccpp/data/GFS_typedefs.F90 | 4 +-- ccpp/driver/CCPP_driver.F90 | 48 +++++++++++++---------------- ccpp/framework | 2 +- 7 files changed, 35 insertions(+), 46 deletions(-) diff --git a/.gitmodules b/.gitmodules index 6bb663df1..f0e00047d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,8 +4,10 @@ branch = dev/emc [submodule "ccpp/framework"] path = ccpp/framework - url = https://github.com/NCAR/ccpp-framework - branch = main + #url = https://github.com/NCAR/ccpp-framework + #branch = main + url = https://github.com/climbfuji/ccpp-framework + branch = feature/fix_mkdoc [submodule "ccpp/physics"] path = ccpp/physics url = https://github.com/NCAR/ccpp-physics diff --git a/atmos_model.F90 b/atmos_model.F90 index 291c2bf69..132b4f403 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -975,7 +975,10 @@ subroutine atmos_model_end (Atmos) endif ! Fast physics (from dynamics) are finalized in atmosphere_end above; -! standard/slow physics (from CCPP) are finalized in CCPP_step 'finalize'. +! standard/slow physics (from CCPP) are finalized in CCPP_step 'physics_finalize'. + call CCPP_step (step="physics_finalize", nblks=Atm_block%nblks, ierr=ierr) + if (ierr/=0) call mpp_error(FATAL, 'Call to CCPP physics_finalize step failed') + ! The CCPP framework for all cdata structures is finalized in CCPP_step 'finalize'. call CCPP_step (step="finalize", nblks=Atm_block%nblks, ierr=ierr) if (ierr/=0) call mpp_error(FATAL, 'Call to CCPP finalize step failed') diff --git a/ccpp/CMakeLists.txt b/ccpp/CMakeLists.txt index 7f1af3301..8ee867056 100644 --- a/ccpp/CMakeLists.txt +++ b/ccpp/CMakeLists.txt @@ -13,10 +13,6 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "Coverage") endif() -#------------------------------------------------------------------------------ -# CMake Modules -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/framework/cmake") - #------------------------------------------------------------------------------ # Call to CCPP code generator if(DEBUG) @@ -54,14 +50,6 @@ if(MPI) add_definitions(-DMPI) endif() -#------------------------------------------------------------------------------ -# Set additional flags for debug build -if(DEBUG) - if(CMAKE_Fortran_COMPILER_ID MATCHES "Intel") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -init=snan,arrays") - endif() -endif() - #------------------------------------------------------------------------------ # Set flag for 32bit dynamics build if(32BIT) @@ -117,7 +105,6 @@ add_library( # Compile GFS_diagnostics.F90 without optimization, this leads to out of memory errors on wcoss_dell_p3 set_property(SOURCE driver/GFS_diagnostics.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-O0") - target_link_libraries(fv3ccpp PUBLIC ccpp_framework) target_link_libraries(fv3ccpp PUBLIC ccpp_physics) diff --git a/ccpp/config/ccpp_prebuild_config.py b/ccpp/config/ccpp_prebuild_config.py index bbc7da433..92622e425 100755 --- a/ccpp/config/ccpp_prebuild_config.py +++ b/ccpp/config/ccpp_prebuild_config.py @@ -213,7 +213,7 @@ # Default build dir, relative to current working directory, # if not specified as command-line argument -DEFAULT_BUILD_DIR = 'FV3' +DEFAULT_BUILD_DIR = 'build' # Auto-generated makefile/cmakefile snippets that contain all type definitions TYPEDEFS_MAKEFILE = '{build_dir}/physics/CCPP_TYPEDEFS.mk' @@ -241,6 +241,7 @@ STATIC_API_SRCFILE = '{build_dir}/physics/CCPP_STATIC_API.sh' # Directory for writing HTML pages generated from metadata files +# used by metadata2html.py for generating scientific documentation METADATA_HTML_OUTPUT_DIR = '{build_dir}/physics/physics/docs' # HTML document containing the model-defined CCPP variables diff --git a/ccpp/data/GFS_typedefs.F90 b/ccpp/data/GFS_typedefs.F90 index f33bcca16..790c130fc 100644 --- a/ccpp/data/GFS_typedefs.F90 +++ b/ccpp/data/GFS_typedefs.F90 @@ -22,8 +22,8 @@ module GFS_typedefs implicit none - ! To ensure that these values match what's in the physics, - ! array sizes are compared during model init in GFS_rrtmg_setup_init() + ! To ensure that these values match what's in the physics, array + ! sizes are compared in the auto-generated physics caps in debug mode private :: NF_AESW, NF_AELW, NSPC, NSPC1, NF_CLDS, NF_VGAS, NF_ALBD, ntrcaerm ! from module_radiation_aerosols integer, parameter :: NF_AESW = 3 diff --git a/ccpp/driver/CCPP_driver.F90 b/ccpp/driver/CCPP_driver.F90 index 392b37151..fa2c8fa54 100644 --- a/ccpp/driver/CCPP_driver.F90 +++ b/ccpp/driver/CCPP_driver.F90 @@ -101,8 +101,8 @@ subroutine CCPP_step (step, nblks, ierr) else if (trim(step)=="physics_init") then ! Since the physics init step is independent of the blocking structure, - ! we can use cdata_domain here. Since we don't use threading on the outside, - ! we can allow threading inside the physics init routines. + ! we can use cdata_domain. And since we don't use threading on the host + ! model side, we can allow threading inside the physics init routines. GFS_control%nthreads = nthrds call ccpp_physics_init(cdata_domain, suite_name=trim(ccpp_suite), ierr=ierr) @@ -116,8 +116,8 @@ subroutine CCPP_step (step, nblks, ierr) else if (trim(step)=="timestep_init") then ! Since the physics timestep init step is independent of the blocking structure, - ! we can use cdata_domain here. Since we don't use threading on the outside, - ! we can allow threading inside the timestep init (time_vary) routines. + ! we can use cdata_domain. And since we don't use threading on the host + ! model side, we can allow threading inside the timestep init (time_vary) routines. GFS_control%nthreads = nthrds call ccpp_physics_timestep_init(cdata_domain, suite_name=trim(ccpp_suite), group_name="time_vary", ierr=ierr) @@ -159,11 +159,11 @@ subroutine CCPP_step (step, nblks, ierr) ! *DH 20210104 ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! Radiation and stochastic physics + ! Radiation, physics and and stochastic physics - threaded regions using blocked data structures else if (trim(step)=="radiation" .or. trim(step)=="physics" .or. trim(step)=="stochastics") then ! Set number of threads available to physics schemes to one, - ! because threads are used on the outside for blocking + ! because threads are used on the host model side for blocking GFS_control%nthreads = 1 !$OMP parallel num_threads (nthrds) & @@ -188,8 +188,8 @@ subroutine CCPP_step (step, nblks, ierr) call ccpp_physics_run(cdata_block(nb,ntX), suite_name=trim(ccpp_suite), group_name=trim(step), ierr=ierr2) if (ierr2/=0) then write(0,'(2a,3(a,i4),a)') "An error occurred in ccpp_physics_run for group ", trim(step), & - ", block ", nb, " and thread ", nt, " (ntX=", ntX, ")" - write(0,'(a)') trim(cdata_block(nb,nt)%errmsg) + ", block ", nb, " and thread ", nt, " (ntX=", ntX, "):" + write(0,'(a)') trim(cdata_block(nb,ntX)%errmsg) ierr = ierr + ierr2 end if end do @@ -202,7 +202,7 @@ subroutine CCPP_step (step, nblks, ierr) else if (trim(step)=="timestep_finalize") then ! Since the physics timestep finalize step is independent of the blocking structure, - ! we can use cdata_domain here. Since we don't use threading on the outside, + ! we can use cdata_domain. And since we don't use threading on the host model side, ! we can allow threading inside the timestep finalize (time_vary) routines. GFS_control%nthreads = nthrds @@ -213,27 +213,23 @@ subroutine CCPP_step (step, nblks, ierr) return end if - ! Finalize - else if (trim(step)=="finalize") then + ! Physics finalize + else if (trim(step)=="physics_finalize") then - ! Loop over blocks, don't use threading on the outside but allowing threading - ! inside the finalization, similar to what is done for the initialization + ! Since the physics finalize step is independent of the blocking structure, + ! we can use cdata_domain. And since we don't use threading on the host + ! model side, we can allow threading inside the physics finalize routines. GFS_control%nthreads = nthrds - ! Fast physics are finalized in atmosphere_end, loop over - ! all blocks and threads to finalize all other physics - do nt=1,nthrdsX - do nb=1,nblks - !--- Finalize CCPP physics - call ccpp_physics_finalize(cdata_block(nb,nt), suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(0,'(a,i4,a,i4)') "An error occurred in ccpp_physics_finalize for block ", nb, " and thread ", nt - write(0,'(a)') trim(cdata_block(nb,nt)%errmsg) - return - end if - end do - end do + call ccpp_physics_finalize(cdata_domain, suite_name=trim(ccpp_suite), ierr=ierr) + if (ierr/=0) then + write(0,'(a)') "An error occurred in ccpp_physics_finalize" + write(0,'(a)') trim(cdata_domain%errmsg) + return + end if + ! Finalize + else if (trim(step)=="finalize") then ! Deallocate cdata structure for blocks and threads if (allocated(cdata_block)) deallocate(cdata_block) diff --git a/ccpp/framework b/ccpp/framework index a55457fe3..bd87694c7 160000 --- a/ccpp/framework +++ b/ccpp/framework @@ -1 +1 @@ -Subproject commit a55457fe3ef66e1651c94f99e72aba3362b105a2 +Subproject commit bd87694c75a62fe506845138d22a322a6a3a0f7a From ec66f4d703f2455eadfbef4e2129c93313ed7336 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 10 Mar 2022 18:55:28 -0700 Subject: [PATCH 2/2] Revert .gitmodules and update submodule pointer for ccpp-framework --- .gitmodules | 6 ++---- ccpp/framework | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index f0e00047d..6bb663df1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,10 +4,8 @@ branch = dev/emc [submodule "ccpp/framework"] path = ccpp/framework - #url = https://github.com/NCAR/ccpp-framework - #branch = main - url = https://github.com/climbfuji/ccpp-framework - branch = feature/fix_mkdoc + url = https://github.com/NCAR/ccpp-framework + branch = main [submodule "ccpp/physics"] path = ccpp/physics url = https://github.com/NCAR/ccpp-physics diff --git a/ccpp/framework b/ccpp/framework index bd87694c7..8577ea3b8 160000 --- a/ccpp/framework +++ b/ccpp/framework @@ -1 +1 @@ -Subproject commit bd87694c75a62fe506845138d22a322a6a3a0f7a +Subproject commit 8577ea3b865a6ff7406c6d26c7dd3ce835b7c101