diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..21c5ba4f4 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# This is a comment. +# Each line is a file pattern followed by one or more owners. +# Order is important; the last matching pattern takes the most +# precedence. + +# The components are owned by their respective teams +/FV3/ @noaa-emc/fv3atm-team diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..faa664ab3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,56 @@ +--- +name: Bug report +about: Create a report to fix bugs +title: '' +labels: bug +assignees: '' + +--- + +## Description +Provide a clear and concise description of what the bug is. +Also give a description of how to fix the bug. + + +### To Reproduce: +What compilers/machines are you seeing this with? +Give explicit steps to reproduce the behavior. +1. do this +2. then that +3. then, oops, look at the bug + + +## Additional context +Add any other context about the problem here. +Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example: +- needs to be fixed also in noaa-emc/nems/issues/ +- needed for noaa-emc/fv3atm/pull/ + + +## Output + +**Screenshots** +If applicable, drag and drop screenshots to help explain your problem. + +**output logs** +If applicable, include relevant output logs. +Either drag and drop the entire log file here (if a long log) or + +``` +paste the code here (if a short section of log) +``` + +## Testing: + +1. Have you tested the code changes? On what platforms? + +2. Have you run regression test in ufs-weather-model or ufs-s2s-model with code changes? +- Will the baseline results change? +- If the baseline results change, is it expected? Please give brief explanation. + +## Dependent PRs: + +Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example: +- required to support noaa-emc/GFDL_atmos_cubed_sphere/issues/ +- ncar/ccpp-physics/pull/ +- associated ufs-weather-model/pull/ diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..12db08eda --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,32 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +## Description +Provide a clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +## Solution +Add a clear and concise description of what solution you provide. + +## Alternatives +If applicable, add a description of any alternative solutions or features you've considered. + +## Testing: + +1. Have you tested the code changes? On what platforms? +2. Have you run regression test in ufs-weather-model or ufs-s2s-model with code changes? +- Will the baseline results change? +- If yes, please give brief explanation of your code updates on the regression test. + +## Dependent PRs: + +Directly reference any issues or PRs in this or other repositories that this is related to, and describe how they are related. Example: +- required to support noaa-emc/GFDL_atmos_cubed_sphere/issues/ +- ncar/ccpp-physics/pull/ +- associated ufs-weather-model/pull/ + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..e4d67029f --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,37 @@ +## Description + +(Instructions: this, and all subsequent sections of text should be removed and filled in as appropriate.) +Provide a detailed description of what this PR does. +What bug does it fix, or what feature does it add? +Is a change of answers expected from this PR? + + + +### Issue(s) addressed + +Link the issues to be closed with this PR, whether in this repository, or in another repository. +(Remember, issues should always be created before starting work on a PR branch!) +- fixes # +- fixes noaa-emc/fv3atm/issues/ + + + +## Testing + +How were these changes tested? +What compilers / HPCs was it tested with? +Are the changes covered by regression tests? (If not, why? Do new tests need to be added?) +Have the ufs-weather-model regression test been run? On what platform? +- Will the code updates change regression test baseline? If yes, why? Please show the baseline directory below. +- Please commit the regression test log files in your ufs-weather-model branch + + +## Dependencies + +If testing this branch requires non-default branches in other repositories, list them. +Those branches should have matching names (ideally) + +Do PRs in upstream repositories need to be merged first? +If so add the "waiting for other repos" label and list the upstream PRs +- waiting on noaa-emc/nems/pull/ +- waiting on noaa-emc/fv3atm/pull/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 0af2ea9a7..1eaae5928 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ if(CCPP) - add_definitions(-DCCPP) + add_definitions(-DCCPP) endif() add_subdirectory(cpl) @@ -7,14 +7,15 @@ add_subdirectory(gfsphysics) add_subdirectory(ipd) add_subdirectory(io) -# fv3core library - +############################################################################### +### fv3dycore +############################################################################### if(NOT CCPP) -set(FAST_PHYSICS_SRCS_F90 atmos_cubed_sphere/model/fv_cmp.F90) + set(FAST_PHYSICS_SRCS_F90 atmos_cubed_sphere/model/fv_cmp.F90) endif() add_library( - fv3core + fv3dycore atmos_cubed_sphere/model/a2b_edge.F90 atmos_cubed_sphere/model/multi_gases.F90 @@ -61,37 +62,46 @@ add_library( set_property(SOURCE atmos_cubed_sphere/model/nh_utils.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "${FAST}") set_property(SOURCE atmos_cubed_sphere/model/fv_mapz.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "${FAST}") -target_link_libraries(fv3core PRIVATE fms) -target_link_libraries(fv3core PRIVATE gfsphysics) -target_link_libraries(fv3core PRIVATE ipd) +set_target_properties(fv3dycore PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) +target_include_directories(fv3dycore PUBLIC $) -target_include_directories(fv3core PRIVATE atmos_cubed_sphere ${ESMF_MOD}) +target_link_libraries(fv3dycore PRIVATE fms) +target_link_libraries(fv3dycore PRIVATE gfsphysics) +target_link_libraries(fv3dycore PRIVATE ipd) +target_link_libraries(fv3dycore PRIVATE esmf) -# end of fv3core +target_include_directories(fv3dycore PRIVATE atmos_cubed_sphere ${ESMF_MOD}) ############################################################################### ### ccpp ############################################################################### if(CCPP) - add_subdirectory(ccpp) - add_subdirectory(ccpp/driver) - add_dependencies(gfsphysics ccpp ccppphys) - add_dependencies(ccppdriver ccpp ccppphys) - add_dependencies(ccppphys ccpp) - - target_include_directories(fv3core PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src) - target_include_directories(fv3core PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/driver) -endif() + add_subdirectory(ccpp) + add_subdirectory(ccpp/driver) + + add_dependencies(gfsphysics ccpp ccppphys) + add_dependencies(ccppdriver ccpp ccppphys) + add_dependencies(ccppphys ccpp) -set_target_properties(fv3core PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) -target_include_directories(fv3core PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod) + target_include_directories(fv3dycore PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src) + target_include_directories(fv3dycore PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/driver) + target_link_libraries(ccppphys PRIVATE sp::sp_d) + target_link_libraries(ccppphys PRIVATE w3nco::w3nco_d) +endif() + +############################################################################### +### stochastic_physics +############################################################################### # When specifying an out-of-tree source a binary directory must be explicitly specified. #add_subdirectory(../stochastic_physics stochastic_physics) add_subdirectory(stochastic_physics) +############################################################################### +### fv3atm +############################################################################### add_library( - fv3cap + fv3atm atmos_model.F90 fv3_cap.F90 @@ -100,21 +110,46 @@ add_library( time_utils.F90 ) +set_target_properties(fv3atm PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) +target_include_directories(fv3atm PUBLIC $) + if(CCPP) -target_include_directories(fv3cap PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/ccpp/driver/mod) -add_dependencies(fv3cap ccppdriver ccppphys) + target_include_directories(fv3atm PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/ccpp/driver/mod) + set(CCPP_LIBRARIES ccppdriver ccppphys ccpp) + add_dependencies(fv3atm ccppdriver ccppphys ccpp) + target_link_libraries(fv3atm PUBLIC ccppdriver ccppphys ccpp) +endif() +target_include_directories(fv3atm PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/stochastic_physics) +target_compile_definitions(fv3atm PRIVATE -DESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR}) + +target_link_libraries(fv3atm PUBLIC fms) +target_link_libraries(fv3atm PUBLIC cpl) +target_link_libraries(fv3atm PUBLIC gfsphysics) +target_link_libraries(fv3atm PUBLIC ipd) +target_link_libraries(fv3atm PUBLIC io) +target_link_libraries(fv3atm PUBLIC fv3dycore) +target_link_libraries(fv3atm PUBLIC stochastic_physics) +target_link_libraries(fv3atm PUBLIC stochastic_physics_wrapper) + +target_link_libraries(fv3atm PUBLIC + nemsio::nemsio w3emc::w3emc_d w3nco::w3nco_d sp::sp_d bacio::bacio_4) +if(INLINE_POST) + target_link_libraries(fv3atm PUBLIC nceppost::nceppost) endif() -target_include_directories(fv3cap PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/stochastic_physics) -target_compile_definitions(fv3cap PRIVATE -DESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR}) -target_include_directories(fv3cap PRIVATE ${ESMF_MOD}) - -target_link_libraries(fv3cap PRIVATE fms) -target_link_libraries(fv3cap PRIVATE fv3cpl) -target_link_libraries(fv3cap PRIVATE gfsphysics) -target_link_libraries(fv3cap PRIVATE ipd) -target_link_libraries(fv3cap PRIVATE io) -target_link_libraries(fv3cap PRIVATE fv3core) -target_link_libraries(fv3cap PRIVATE stochastic_physics) - -set_target_properties(fv3cap PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) -target_include_directories(fv3cap PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod) + +target_link_libraries(fv3atm PUBLIC esmf) + +############################################################################### +### Install +############################################################################### +install( + TARGETS fv3atm fv3dycore io ipd gfsphysics ${CCPP_LIBRARIES} cpl stochastic_physics stochastic_physics_wrapper + EXPORT fv3atm-config + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib) +install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX}) + +install(EXPORT fv3atm-config + DESTINATION lib/cmake +) + diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index 664d0dc29..e8896bc0f 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit 664d0dc291e880043ff0415b5a8acabe1ade52fc +Subproject commit e8896bc0f582c7c825eaa198cbbb83c206a29d43 diff --git a/atmos_model.F90 b/atmos_model.F90 index 8637ed838..ec8e8a9a4 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -104,9 +104,7 @@ module atmos_model_mod use physics_abstraction_layer, only: time_vary_step, radiation_step1, physics_step1, physics_step2 #endif -use stochastic_physics, only: init_stochastic_physics, & - run_stochastic_physics -use stochastic_physics_sfc, only: run_stochastic_physics_sfc +use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper use FV3GFS_io_mod, only: FV3GFS_restart_read, FV3GFS_restart_write, & FV3GFS_IPD_checksum, & @@ -251,26 +249,17 @@ module atmos_model_mod ! subroutine update_atmos_radiation_physics (Atmos) -#ifdef OPENMP - use omp_lib -#endif !----------------------------------------------------------------------- type (atmos_data_type), intent(in) :: Atmos !--- local variables--- integer :: nb, jdat(8), rc procedure(IPD_func0d_proc), pointer :: Func0d => NULL() procedure(IPD_func1d_proc), pointer :: Func1d => NULL() - integer :: nthrds + ! #ifdef CCPP integer :: ierr #endif -#ifdef OPENMP - nthrds = omp_get_max_threads() -#else - nthrds = 1 -#endif - if (mpp_pe() == mpp_root_pe() .and. debug) write(6,*) "statein driver" !--- get atmospheric state from the dynamic core call set_atmosphere_pelist() @@ -307,24 +296,7 @@ subroutine update_atmos_radiation_physics (Atmos) #endif !--- call stochastic physics pattern generation / cellular automata - if (IPD_Control%do_sppt .OR. IPD_Control%do_shum .OR. IPD_Control%do_skeb .OR. IPD_Control%do_sfcperts) then - call run_stochastic_physics(IPD_Control, IPD_Data(:)%Grid, IPD_Data(:)%Coupling, nthrds) - end if - - if(IPD_Control%do_ca)then - if(IPD_Control%ca_sgs)then - call cellular_automata_sgs(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, & - IPD_Control%nca,IPD_Control%ncells,IPD_Control%nlives,IPD_Control%nfracseed,& - IPD_Control%nseed,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,& - IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1)) - endif - if(IPD_Control%ca_global)then - call cellular_automata_global(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, & - IPD_Control%nca_g,IPD_Control%ncells_g,IPD_Control%nlives_g,IPD_Control%nfracseed,& - IPD_Control%nseed_g,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,& - IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1),IPD_Control%nsmooth,IPD_Control%ca_amplitude) - endif - endif + call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block) !--- if coupled, assign coupled fields if( IPD_Control%cplflx .or. IPD_Control%cplwav ) then @@ -649,45 +621,11 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step) call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, Init_parm) #endif - if (IPD_Control%do_sppt .OR. IPD_Control%do_shum .OR. IPD_Control%do_skeb .OR. IPD_Control%do_sfcperts) then - ! Initialize stochastic physics - call init_stochastic_physics(IPD_Control, Init_parm, mpp_npes(), nthrds) - if(IPD_Control%me == IPD_Control%master) print *,'do_skeb=',IPD_Control%do_skeb - end if +!--- Initialize stochastic physics pattern generation / cellular automata for first time step + call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block) Atmos%Diag => IPD_Diag - if (IPD_Control%do_sfcperts) then - ! Get land surface perturbations here (move to GFS_time_vary - ! step if wanting to update each time-step) - call run_stochastic_physics_sfc(IPD_Control, IPD_Data(:)%Grid, IPD_Data(:)%Coupling) - end if - - ! Initialize cellular automata - if(IPD_Control%do_ca)then - ! DH* The current implementation of cellular_automata assumes that all blocksizes are the - ! same - abort if this is not the case, otherwise proceed with Atm_block%blksz(1) below - if (.not. minval(Atm_block%blksz)==maxval(Atm_block%blksz)) then - call mpp_error(FATAL, 'Logic errror: cellular_automata not compatible with non-uniform blocksizes') - end if - ! *DH - if(IPD_Control%do_ca)then - if(IPD_Control%ca_sgs)then - call cellular_automata_sgs(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, & - IPD_Control%nca,IPD_Control%ncells,IPD_Control%nlives,IPD_Control%nfracseed,& - IPD_Control%nseed,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,& - IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1)) - endif - if(IPD_Control%ca_global)then - call cellular_automata_global(IPD_Control%kdt,IPD_Data(:)%Statein,IPD_Data(:)%Coupling,IPD_Data(:)%Intdiag,Atm_block%nblks,IPD_Control%levs, & - IPD_Control%nca_g,IPD_Control%ncells_g,IPD_Control%nlives_g,IPD_Control%nfracseed,& - IPD_Control%nseed_g,IPD_Control%nthresh,IPD_Control%ca_global,IPD_Control%ca_sgs,IPD_Control%iseed_ca,& - IPD_Control%ca_smooth,IPD_Control%nspinup,Atm_block%blksz(1),IPD_Control%nsmooth,IPD_Control%ca_amplitude) - endif - - endif - endif - Atm(mygrid)%flagstruct%do_skeb = IPD_Control%do_skeb ! initialize the IAU module diff --git a/ccpp/build_ccpp.sh b/ccpp/build_ccpp.sh index fe1f64fe7..2e9aa9a11 100755 --- a/ccpp/build_ccpp.sh +++ b/ccpp/build_ccpp.sh @@ -6,8 +6,9 @@ set -eu # List of valid/tested machines VALID_MACHINES=( wcoss_cray wcoss_dell_p3 gaea.intel jet.intel \ hera.intel hera.gnu orion.intel \ - cheyenne.intel cheyenne.intel-impi cheyenne.gnu endeavor.intel \ - stampede.intel supermuc_phase2.intel macosx.gnu \ + cheyenne.intel cheyenne.gnu \ + endeavor.intel stampede.intel \ + macosx.gnu \ linux.intel linux.gnu linux.pgi ) ################################################################################################### diff --git a/ccpp/config/ccpp_prebuild_config.py b/ccpp/config/ccpp_prebuild_config.py index cb9a0ade4..a1aa5cd16 100755 --- a/ccpp/config/ccpp_prebuild_config.py +++ b/ccpp/config/ccpp_prebuild_config.py @@ -162,6 +162,7 @@ 'FV3/ccpp/physics/physics/sfcsub.F', 'FV3/ccpp/physics/physics/sflx.f', 'FV3/ccpp/physics/physics/set_soilveg.f', + 'FV3/ccpp/physics/physics/flake.F90', 'FV3/ccpp/physics/physics/surface_perturbation.F90', 'FV3/ccpp/physics/physics/cu_gf_deep.F90', 'FV3/ccpp/physics/physics/cu_gf_sh.F90', @@ -289,6 +290,7 @@ 'FV3/ccpp/physics/physics/sfc_diff.f', 'FV3/ccpp/physics/physics/sfc_drv.f', 'FV3/ccpp/physics/physics/sfc_noahmp_drv.f', + 'FV3/ccpp/physics/physics/flake_driver.F90', 'FV3/ccpp/physics/physics/sfc_nst.f', 'FV3/ccpp/physics/physics/sfc_ocean.F', 'FV3/ccpp/physics/physics/sfc_sice.f', diff --git a/ccpp/driver/CMakeLists.txt b/ccpp/driver/CMakeLists.txt index 654eb972e..5960d92da 100644 --- a/ccpp/driver/CMakeLists.txt +++ b/ccpp/driver/CMakeLists.txt @@ -24,4 +24,4 @@ target_link_libraries(ccppdriver ccppphys) target_include_directories(ccppdriver PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src) set_target_properties(ccppdriver PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) -target_include_directories(ccppdriver PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod) +target_include_directories(ccppdriver PUBLIC $) diff --git a/ccpp/driver/makefile b/ccpp/driver/makefile index ccbae2c50..c64441135 100644 --- a/ccpp/driver/makefile +++ b/ccpp/driver/makefile @@ -64,8 +64,6 @@ clean: MKDEPENDS = ../../mkDepends.pl include ../../conf/make.rules -include ./depend - # do not include 'depend' file if the target contains string 'clean' ifneq (clean,$(findstring clean,$(MAKECMDGOALS))) -include depend diff --git a/ccpp/framework b/ccpp/framework index c5ede565f..0b84becf3 160000 --- a/ccpp/framework +++ b/ccpp/framework @@ -1 +1 @@ -Subproject commit c5ede565f5e9ff434619498bfea77c19cbb21a20 +Subproject commit 0b84becf3ca7f541b37fc9cf8d11a45aa2b14407 diff --git a/ccpp/physics b/ccpp/physics index eab936e42..273557081 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit eab936e42875d35b4e38b3cfe6cb2e099fd9c88d +Subproject commit 2735570811a9d91fd7909e785735e4ea4349dd4e diff --git a/ccpp/set_compilers.sh b/ccpp/set_compilers.sh index 23143a672..894017cb0 100755 --- a/ccpp/set_compilers.sh +++ b/ccpp/set_compilers.sh @@ -60,13 +60,6 @@ case "$MACHINE_ID" in export F77=mpif77 export F90=mpif90 ;; - cheyenne.intel-impi) - export CC=mpiicc - export CXX=mpiicpc - export FC=mpiifort - export F77=mpiifort - export F90=mpiifort - ;; cheyenne.gnu) export CC=mpicc export CXX=mpicxx @@ -88,13 +81,6 @@ case "$MACHINE_ID" in export F77=mpif77 export F90=mpif90 ;; - supermuc_phase2.intel) - export CC=mpiicc - export CXX=mpiicpc - export FC=mpiifort - export F77=mpiifort - export F90=mpiifort - ;; macosx.gnu) # set in generic modulefile ;; diff --git a/ccpp/suites/suite_FV3_GFS_2017_fv3wam.xml b/ccpp/suites/suite_FV3_GFS_2017_fv3wam.xml index 89f21fd7c..fac22314c 100644 --- a/ccpp/suites/suite_FV3_GFS_2017_fv3wam.xml +++ b/ccpp/suites/suite_FV3_GFS_2017_fv3wam.xml @@ -60,9 +60,9 @@ rayleigh_damp GFS_suite_stateout_update ozphys - GFS_DCNV_generic_pre get_phi_fv3 GFS_suite_interstitial_3 + GFS_DCNV_generic_pre samfdeepcnv GFS_DCNV_generic_post GFS_SCNV_generic_pre diff --git a/ccpp/suites/suite_FV3_GFS_v16beta_flake.xml b/ccpp/suites/suite_FV3_GFS_v16beta_flake.xml new file mode 100644 index 000000000..dec1a76a2 --- /dev/null +++ b/ccpp/suites/suite_FV3_GFS_v16beta_flake.xml @@ -0,0 +1,94 @@ + + + + + + + fv_sat_adj + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + GFS_rrtmg_pre + rrtmg_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw_pre + rrtmg_lw + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + GFS_surface_composites_pre + dcyc2t3 + GFS_surface_composites_inter + GFS_suite_interstitial_2 + + + + sfc_diff + GFS_surface_loop_control_part1 + sfc_nst_pre + sfc_nst + sfc_nst_post + lsm_noah + flake_driver + sfc_sice + GFS_surface_loop_control_part2 + + + + GFS_surface_composites_post + sfc_diag + sfc_diag_post + GFS_surface_generic_post + GFS_PBL_generic_pre + satmedmfvdifq + GFS_PBL_generic_post + GFS_GWD_generic_pre + cires_ugwp + cires_ugwp_post + GFS_GWD_generic_post + rayleigh_damp + GFS_suite_stateout_update + ozphys_2015 + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_DCNV_generic_pre + samfdeepcnv + GFS_DCNV_generic_post + GFS_SCNV_generic_pre + samfshalcnv + GFS_SCNV_generic_post + GFS_suite_interstitial_4 + cnvc90 + GFS_MP_generic_pre + gfdl_cloud_microphys + GFS_MP_generic_post + maximum_hourly_diagnostics + + + + + GFS_stochastics + + + + diff --git a/ccpp/suites/suite_FV3_GSD_noah_mynnsfc.xml b/ccpp/suites/suite_FV3_GSD_noah_mynnsfc.xml index 72f1b659f..f2da9d4e8 100644 --- a/ccpp/suites/suite_FV3_GSD_noah_mynnsfc.xml +++ b/ccpp/suites/suite_FV3_GSD_noah_mynnsfc.xml @@ -1,6 +1,6 @@ - + diff --git a/ccpp/suites/suite_FV3_GSD_v0_mynnsfc.xml b/ccpp/suites/suite_FV3_GSD_v0_mynnsfc.xml index 50680a893..f1c22a774 100644 --- a/ccpp/suites/suite_FV3_GSD_v0_mynnsfc.xml +++ b/ccpp/suites/suite_FV3_GSD_v0_mynnsfc.xml @@ -1,6 +1,6 @@ - + diff --git a/ccpp/suites/suite_FV3_RRFS_v1beta.xml b/ccpp/suites/suite_FV3_RRFS_v1beta.xml new file mode 100644 index 000000000..3bff7b39d --- /dev/null +++ b/ccpp/suites/suite_FV3_RRFS_v1beta.xml @@ -0,0 +1,82 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + sgscloud_radpre + GFS_rrtmg_pre + rrtmg_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw_pre + rrtmg_lw + sgscloud_radpost + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + GFS_surface_composites_pre + dcyc2t3 + GFS_surface_composites_inter + GFS_suite_interstitial_2 + + + + mynnsfc_wrapper + GFS_surface_loop_control_part1 + sfc_nst_pre + sfc_nst + sfc_nst_post + noahmpdrv + sfc_sice + GFS_surface_loop_control_part2 + + + + GFS_surface_composites_post + sfc_diag + sfc_diag_post + GFS_surface_generic_post + mynnedmf_wrapper + GFS_GWD_generic_pre + drag_suite + GFS_GWD_generic_post + rayleigh_damp + GFS_suite_stateout_update + ozphys_2015 + h2ophys + get_phi_fv3 + GFS_suite_interstitial_3 + GFS_suite_interstitial_4 + GFS_MP_generic_pre + mp_thompson_pre + mp_thompson + mp_thompson_post + GFS_MP_generic_post + maximum_hourly_diagnostics + + + + + GFS_stochastics + + + + diff --git a/cpl/CMakeLists.txt b/cpl/CMakeLists.txt index c810ed1b3..74271f917 100644 --- a/cpl/CMakeLists.txt +++ b/cpl/CMakeLists.txt @@ -1,12 +1,11 @@ add_library( - fv3cpl + cpl module_cplfields.F90 module_cap_cpl.F90 ) +target_link_libraries(cpl PRIVATE esmf) -target_include_directories(fv3cpl PRIVATE ${ESMF_MOD}) - -set_target_properties(fv3cpl PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) -target_include_directories(fv3cpl PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod) +set_target_properties(cpl PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) +target_include_directories(cpl PUBLIC $) diff --git a/gfsphysics/CMakeLists.txt b/gfsphysics/CMakeLists.txt index 6b9d5ae8e..34faf6f4c 100644 --- a/gfsphysics/CMakeLists.txt +++ b/gfsphysics/CMakeLists.txt @@ -10,7 +10,7 @@ if(CMAKE_Fortran_COMPILER_ID MATCHES "Intel") string (REPLACE "-i4 -real-size 32" "-i4 -real-size 64 -no-prec-div -no-prec-sqrt" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") endif() elseif(CMAKE_Fortran_COMPILER_ID MATCHES "GNU") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8") endif() endif() @@ -202,6 +202,7 @@ if(CCPP) add_library(gfsphysics ${CCPP_SOURCES}) target_include_directories(gfsphysics PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src) target_include_directories(gfsphysics PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/physics) + target_link_libraries(gfsphysics PRIVATE ccppphys ccpp) else() add_library(gfsphysics ${IPD_SOURCES}) endif() @@ -216,7 +217,7 @@ endif() set_property(SOURCE GFS_layer/GFS_diagnostics.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-O0") target_compile_definitions(gfsphysics PRIVATE -DNEMS_GSM) -target_include_directories(gfsphysics PRIVATE ${NETCDF_INC_DIR}) +target_link_libraries(gfsphysics PRIVATE NetCDF::NetCDF_Fortran) set_target_properties(gfsphysics PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) -target_include_directories(gfsphysics PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod) +target_include_directories(gfsphysics PUBLIC $) diff --git a/gfsphysics/GFS_layer/GFS_typedefs.F90 b/gfsphysics/GFS_layer/GFS_typedefs.F90 index 967a4cc64..70a565a36 100644 --- a/gfsphysics/GFS_layer/GFS_typedefs.F90 +++ b/gfsphysics/GFS_layer/GFS_typedefs.F90 @@ -140,7 +140,6 @@ module GFS_typedefs #ifdef CCPP !--- restart information logical :: restart !< flag whether this is a coldstart (.false.) or a warmstart/restart (.true.) - logical :: cycling !< flag whether this is a coldstart (.false.) or a cycled run (.true.) !--- hydrostatic/non-hydrostatic flag logical :: hydrostatic !< flag whether this is a hydrostatic or non-hydrostatic run #endif @@ -235,6 +234,7 @@ module GFS_typedefs real (kind=kind_phys), pointer :: oceanfrac(:) => null() !< ocean fraction [0:1] real (kind=kind_phys), pointer :: landfrac(:) => null() !< land fraction [0:1] real (kind=kind_phys), pointer :: lakefrac(:) => null() !< lake fraction [0:1] + real (kind=kind_phys), pointer :: lakedepth(:) => null() !< lake depth [ m ] real (kind=kind_phys), pointer :: tsfc (:) => null() !< surface air temperature in K !< [tsea in gbphys.f] real (kind=kind_phys), pointer :: tsfco (:) => null() !< sst in K @@ -803,6 +803,9 @@ module GFS_typedefs logical :: use_ufo !< flag for gcycle surface option +!--- flake model parameters + integer :: lkm !< flag for flake model + !--- tuning parameters for physical parameterizations logical :: ras !< flag for ras convection scheme logical :: flipv !< flag for vertical direction flip (ras) @@ -1125,7 +1128,6 @@ module GFS_typedefs #ifdef CCPP logical :: first_time_step !< flag signaling first time step for time integration routine logical :: restart !< flag whether this is a coldstart (.false.) or a warmstart/restart (.true.) - logical :: cycling !< flag whether this is a coldstart (.false.) or a cycled run (.true.) logical :: hydrostatic !< flag whether this is a hydrostatic or non-hydrostatic run #endif integer :: jdat(1:8) !< current forecast date and time @@ -2208,6 +2210,7 @@ subroutine sfcprop_create (Sfcprop, IM, Model) allocate (Sfcprop%oceanfrac(IM)) allocate (Sfcprop%landfrac (IM)) allocate (Sfcprop%lakefrac (IM)) + allocate (Sfcprop%lakedepth(IM)) allocate (Sfcprop%tsfc (IM)) allocate (Sfcprop%tsfco (IM)) allocate (Sfcprop%tsfcl (IM)) @@ -2225,6 +2228,7 @@ subroutine sfcprop_create (Sfcprop, IM, Model) Sfcprop%oceanfrac = clear_val Sfcprop%landfrac = clear_val Sfcprop%lakefrac = clear_val + Sfcprop%lakedepth = clear_val Sfcprop%tsfc = clear_val Sfcprop%tsfco = clear_val Sfcprop%tsfcl = clear_val @@ -2890,7 +2894,6 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & logical :: aux2d_time_avg(1:naux2dmax) = .false. !< flags for time averaging of auxiliary 2d arrays logical :: aux3d_time_avg(1:naux3dmax) = .false. !< flags for time averaging of auxiliary 3d arrays - logical :: cycling = .false. !< flag to activate extra cycling procedures real(kind=kind_phys) :: fhcyc = 0. !< frequency for surface data cycling (hours) integer :: thermodyn_id = 1 !< valid for GFS only for get_prs/phi integer :: sfcpress_id = 1 !< valid for GFS only for get_prs/phi @@ -3059,6 +3062,9 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & logical :: use_ufo = .false. !< flag for gcycle surface option +!--- flake model parameters + integer :: lkm = 0 !< flag for flake model + !--- tuning parameters for physical parameterizations logical :: ras = .false. !< flag for ras convection scheme logical :: flipv = .true. !< flag for vertical direction flip (ras) @@ -3352,6 +3358,10 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & ! Noah MP options iopt_dveg,iopt_crs,iopt_btr,iopt_run,iopt_sfc, iopt_frz, & iopt_inf, iopt_rad,iopt_alb,iopt_snf,iopt_tbot,iopt_stc, & + + !--- lake model control + lkm, & + !--- physical parameterizations ras, trans_trac, old_monin, cnvgwd, mstrat, moist_adj, & cscnv, cal_pre, do_aw, do_shoc, shocaftcnv, shoc_cld, & @@ -3796,6 +3806,9 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & Model%isot = isot Model%use_ufo = use_ufo +!--- flake model parameters + Model%lkm = lkm + ! Noah MP options from namelist ! Model%iopt_dveg = iopt_dveg @@ -4164,7 +4177,6 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & #ifdef CCPP Model%first_time_step = .true. Model%restart = restart - Model%cycling = cycling Model%hydrostatic = hydrostatic #endif Model%jdat(1:8) = jdat(1:8) @@ -4328,14 +4340,18 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, & stop endif - if (Model%lsm == Model%lsm_noahmp .and. Model%iopt_snf == 4) then - if (Model%imp_physics /= Model%imp_physics_gfdl) stop 'iopt_snf == 4 must use GFDL MP' - endif +! if (Model%lsm == Model%lsm_noahmp .and. Model%iopt_snf == 4) then +! if (Model%imp_physics /= Model%imp_physics_gfdl) stop 'iopt_snf == 4 must use GFDL MP' +! endif print *,' nst_anl=',Model%nst_anl,' use_ufo=',Model%use_ufo,' frac_grid=',Model%frac_grid,& ' ignore_lake=',ignore_lake print *,' min_lakeice=',Model%min_lakeice,' min_seaice=',Model%min_seaice, & 'min_lake_height=',Model%min_lake_height + + print *, 'flake model parameters' + print *, 'lkm : ', Model%lkm + if (Model%nstf_name(1) > 0 ) then print *,' NSSTM is active ' print *,' nstf_name(1)=',Model%nstf_name(1) @@ -4920,6 +4936,9 @@ subroutine control_print(Model) print *, ' use_ufo : ', Model%use_ufo print *, ' ' + print *, 'flake model parameters' + print *, 'lkm : ', Model%lkm + print *, ' ' print *, 'tuning parameters for physical parameterizations' print *, ' ras : ', Model%ras if (Model%ras) then @@ -5128,7 +5147,6 @@ subroutine control_print(Model) print *, ' sec : ', Model%sec print *, ' first_time_step : ', Model%first_time_step print *, ' restart : ', Model%restart - print *, ' cycling : ', Model%cycling print *, ' hydrostatic : ', Model%hydrostatic #endif endif @@ -5783,7 +5801,6 @@ end subroutine diag_create subroutine diag_rad_zero(Diag, Model) class(GFS_diag_type) :: Diag type(GFS_control_type), intent(in) :: Model - integer :: i Diag%fluxr = zero Diag%topfsw%upfxc = zero @@ -5806,7 +5823,6 @@ subroutine diag_phys_zero (Diag, Model, linit, iauwindow_center) logical,optional, intent(in) :: linit, iauwindow_center logical set_totprcp - integer :: i !--- In/Out Diag%srunoff = zero diff --git a/gfsphysics/GFS_layer/GFS_typedefs.meta b/gfsphysics/GFS_layer/GFS_typedefs.meta index 337b3fce8..49023a1df 100644 --- a/gfsphysics/GFS_layer/GFS_typedefs.meta +++ b/gfsphysics/GFS_layer/GFS_typedefs.meta @@ -470,6 +470,13 @@ dimensions = (horizontal_dimension) type = real kind = kind_phys +[lakedepth] + standard_name = lake_depth + long_name = lake depth + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys [tsfc] standard_name = surface_skin_temperature long_name = surface skin temperature @@ -2975,6 +2982,12 @@ units = index dimensions = () type = integer +[lkm] + standard_name = flag_for_lake_surface_scheme + long_name = flag for lake surface model + units = flag + dimensions = () + type = integer [ras] standard_name = flag_for_ras_deep_convection long_name = flag for ras convection scheme @@ -4010,7 +4023,7 @@ [slag] standard_name = equation_of_time long_name = equation of time (radian) - units = radians + units = radian dimensions = () type = real kind = kind_phys @@ -4074,12 +4087,6 @@ units = flag dimensions = () type = logical -[cycling] - standard_name = flag_for_cycling - long_name = flag for cycling or coldstart - units = flag - dimensions = () - type = logical [hydrostatic] standard_name = flag_for_hydrostatic_solver long_name = flag for hydrostatic solver from dynamics @@ -4319,14 +4326,14 @@ [xlat] standard_name = latitude long_name = latitude - units = radians + units = radian dimensions = (horizontal_dimension) type = real kind = kind_phys [xlon] standard_name = longitude long_name = longitude - units = radians + units = radian dimensions = (horizontal_dimension) type = real kind = kind_phys @@ -4345,9 +4352,9 @@ type = real kind = kind_phys [xlat_d] - standard_name = latitude_degree - long_name = latitude in degrees - units = degree + standard_name = latitude_in_degree + long_name = latitude in degree north + units = degree_north dimensions = (horizontal_dimension) type = real kind = kind_phys @@ -8150,7 +8157,7 @@ type = real kind = kind_phys [save_tcp] - standard_name = air_temperature_save_from_cumulus_paramterization + standard_name = air_temperature_save_from_convective_parameterization long_name = air temperature after cumulus parameterization units = K dimensions = (horizontal_dimension,vertical_dimension) @@ -8344,7 +8351,7 @@ [theta] standard_name = angle_from_east_of_maximum_subgrid_orographic_variations long_name = angle with_respect to east of maximum subgrid orographic variations - units = degrees + units = degree dimensions = (horizontal_dimension) type = real kind = kind_phys @@ -9482,7 +9489,7 @@ [con_pi] standard_name = pi long_name = ratio of a circle's circumference to its diameter - units = radians + units = none dimensions = () type = real kind = kind_phys @@ -9502,7 +9509,7 @@ kind = kind_phys [con_t0c] standard_name = temperature_at_zero_celsius - long_name = temperature at 0 degrees Celsius + long_name = temperature at 0 degree Celsius units = K dimensions = () type = real diff --git a/gfsphysics/makefile b/gfsphysics/makefile index 9f25780c8..2ae58e317 100644 --- a/gfsphysics/makefile +++ b/gfsphysics/makefile @@ -266,8 +266,6 @@ clean: MKDEPENDS = ../mkDepends.pl include ../conf/make.rules -include ./depend - # do not include 'depend' file if the target contains string 'clean' ifneq (clean,$(findstring clean,$(MAKECMDGOALS))) -include depend diff --git a/io/CMakeLists.txt b/io/CMakeLists.txt index 8ec8bcf00..1fc2ea77e 100644 --- a/io/CMakeLists.txt +++ b/io/CMakeLists.txt @@ -23,20 +23,21 @@ add_library( ${POST_SRC} ) -set_source_files_properties(post_gfs.F90 post_nems_routines.F90 - PROPERTIES - COMPILE_FLAGS "-I${POST_INC}") +target_link_libraries(io PRIVATE fms) +target_link_libraries(io PRIVATE gfsphysics) +target_link_libraries(io PRIVATE ipd) -target_link_libraries(io fms) -target_link_libraries(io gfsphysics) -target_link_libraries(io ipd) +if(INLINE_POST) + target_link_libraries(io PRIVATE nceppost::nceppost) +endif() if(CCPP) target_include_directories(io PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src) target_include_directories(io PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/physics) endif() -target_include_directories(io PRIVATE ${ESMF_MOD}) -target_include_directories(io PRIVATE ${NEMSIO_INC}) + +target_link_libraries(io PRIVATE nemsio::nemsio) +target_link_libraries(io PRIVATE esmf) set_target_properties(io PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) -target_include_directories(io PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod) +target_include_directories(io PUBLIC $) diff --git a/io/FV3GFS_io.F90 b/io/FV3GFS_io.F90 index 4b2738b23..b86cd0295 100644 --- a/io/FV3GFS_io.F90 +++ b/io/FV3GFS_io.F90 @@ -640,6 +640,9 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain) Sfcprop(nb)%oro_uf(ix) = oro_var2(i,j,16) Sfcprop(nb)%landfrac(ix) = oro_var2(i,j,17) !land frac [0:1] Sfcprop(nb)%lakefrac(ix) = oro_var2(i,j,18) !lake frac [0:1] + + Sfcprop(nb)%lakedepth(ix) = oro_var2(i,j,19) !lake depth [m] !YWu + enddo enddo diff --git a/ipd/CMakeLists.txt b/ipd/CMakeLists.txt index 7c9e59b16..3009fbfa5 100644 --- a/ipd/CMakeLists.txt +++ b/ipd/CMakeLists.txt @@ -20,11 +20,12 @@ add_library( IPD_driver.F90 IPD_typedefs.F90 ) -target_link_libraries(ipd gfsphysics) +target_link_libraries(ipd PRIVATE gfsphysics) if(CCPP) target_include_directories(ipd PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src) target_include_directories(ipd PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/physics) endif() + set_target_properties(ipd PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod) -target_include_directories(ipd PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/mod) +target_include_directories(ipd PUBLIC $) diff --git a/ipd/makefile b/ipd/makefile index 88cc6cbd1..ed4a2749b 100644 --- a/ipd/makefile +++ b/ipd/makefile @@ -51,8 +51,6 @@ clean: MKDEPENDS = ../mkDepends.pl include ../conf/make.rules -include ./depend - # do not include 'depend' file if the target contains string 'clean' ifneq (clean,$(findstring clean,$(MAKECMDGOALS))) -include depend diff --git a/makefile b/makefile index 7ea34a736..4191b4777 100644 --- a/makefile +++ b/makefile @@ -12,7 +12,7 @@ ifeq ($(strip $(exist)),) endif -FFLAGS += -I$(FMS_DIR) -Igfsphysics -Iipd -Icpl -Iio -Iatmos_cubed_sphere -Iccpp/driver -I../stochastic_physics +FFLAGS += -I$(FMS_DIR) -Igfsphysics -Iipd -Icpl -Iio -Iatmos_cubed_sphere -Iccpp/driver -Istochastic_physics CPPDEFS += -DESMF_VERSION_MAJOR=$(ESMF_VERSION_MAJOR) # Flag to CCPP build for 32bit dynamics @@ -41,8 +41,9 @@ libs: $(MAKE) -C io $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) $(MAKE) -C atmos_cubed_sphere $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) $(MAKE) -C ../stochastic_physics $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) 32BIT=N # force gfs physics to 64bit + $(MAKE) -C stochastic_physics $(MAKE_OPTS) FMS_DIR=$(FMS_DIR) 32BIT=N # force gfs physics to 64bit -$(FV3_EXE): atmos_model.o coupler_main.o ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a ../stochastic_physics/libstochastic_physics.a cpl/libfv3cpl.a +$(FV3_EXE): atmos_model.o coupler_main.o ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a stochastic_physics/libstochastic_physics_wrapper.a ../stochastic_physics/libstochastic_physics.a cpl/libfv3cpl.a $(LD) -o $@ $^ $(NCEPLIBS) $(LDFLAGS) else @@ -87,7 +88,7 @@ endif ifneq (,$(findstring CCPP,$(CPPDEFS))) esmf_make_fragment: @rm -rf nems_dir; mkdir nems_dir - @cp $(FV3CAP_LIB) ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a cpl/libfv3cpl.a ../stochastic_physics/libstochastic_physics.a nems_dir + @cp $(FV3CAP_LIB) ccpp/driver/libccppdriver.a atmos_cubed_sphere/libfv3core.a io/libfv3io.a ipd/libipd.a gfsphysics/libgfsphys.a cpl/libfv3cpl.a stochastic_physics/libstochastic_physics_wrapper.a ../stochastic_physics/libstochastic_physics.a nems_dir @cp fv3gfs_cap_mod.mod nems_dir @echo "# ESMF self-describing build dependency makefile fragment" > fv3.mk @echo "# src location $(PWD)" >> fv3.mk @@ -97,7 +98,7 @@ esmf_make_fragment: #@echo "ESMF_DEP_INCPATH = $(PWD)/nems_dir" >> fv3.mk @echo "ESMF_DEP_INCPATH = $(PWD) $(addprefix $(PWD)/, nems_dir ccpp/driver atmos_cubed_sphere io gfsphysics cpl ipd ../stochastic_physics)" >> fv3.mk @echo "ESMF_DEP_CMPL_OBJS =" >> fv3.mk - @echo "ESMF_DEP_LINK_OBJS = $(addprefix $(PWD)/nems_dir/, libfv3cap.a libccppdriver.a libfv3core.a libfv3io.a libipd.a libgfsphys.a libfv3cpl.a libstochastic_physics.a) $(SIONLIB_LINK_FLAGS)" >> fv3.mk + @echo "ESMF_DEP_LINK_OBJS = $(addprefix $(PWD)/nems_dir/, libfv3cap.a libccppdriver.a libfv3core.a libfv3io.a libipd.a libgfsphys.a libfv3cpl.a libstochastic_physics_wrapper.a libstochastic_physics.a) $(SIONLIB_LINK_FLAGS)" >> fv3.mk @echo "ESMF_DEP_SHRD_PATH =" >> fv3.mk @echo "ESMF_DEP_SHRD_LIBS =" >> fv3.mk @echo @@ -143,6 +144,7 @@ clean: (cd ccpp/driver && make clean) (cd ipd && make clean) (cd ../stochastic_physics && make clean) + (cd stochastic_physics && make clean) (cd io && make clean) (cd atmos_cubed_sphere && make clean) (cd cpl && make clean) diff --git a/stochastic_physics/CMakeLists.txt b/stochastic_physics/CMakeLists.txt index a9ee27efa..c9131308b 100644 --- a/stochastic_physics/CMakeLists.txt +++ b/stochastic_physics/CMakeLists.txt @@ -1,61 +1,19 @@ - -if(32BIT) -remove_definitions(-DOVERLOAD_R4) -remove_definitions(-DOVERLOAD_R8) -message ("Force 64 bits in stochastic_physics") -if(CMAKE_Fortran_COMPILER_ID MATCHES "Intel") - if(REPRO) - string (REPLACE "-i4 -real-size 32" "-i4 -real-size 64" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") - else() - string (REPLACE "-i4 -real-size 32" "-i4 -real-size 64 -no-prec-div -no-prec-sqrt" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") - endif() -elseif(CMAKE_Fortran_COMPILER_ID MATCHES "GNU") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8") -endif() -endif() - add_library( - stochastic_physics + stochastic_physics_wrapper - ../../stochastic_physics/plumes.f90 - - ../../stochastic_physics/stochy_gg_def.f - ../../stochastic_physics/stochy_resol_def.f - ../../stochastic_physics/stochy_layout_lag.f - ../../stochastic_physics/four_to_grid_stochy.F - ../../stochastic_physics/glats_stochy.f - ../../stochastic_physics/sumfln_stochy.f - ../../stochastic_physics/gozrineo_stochy.f - ../../stochastic_physics/num_parthds_stochy.f - ../../stochastic_physics/get_ls_node_stochy.f - ../../stochastic_physics/get_lats_node_a_stochy.f - ../../stochastic_physics/setlats_a_stochy.f - ../../stochastic_physics/setlats_lag_stochy.f - ../../stochastic_physics/epslon_stochy.f - ../../stochastic_physics/getcon_lag_stochy.f - ../../stochastic_physics/pln2eo_stochy.f - ../../stochastic_physics/dozeuv_stochy.f - ../../stochastic_physics/dezouv_stochy.f - - ../../stochastic_physics/spectral_layout.F90 - ../../stochastic_physics/getcon_spectral.F90 - ../../stochastic_physics/stochy_namelist_def.F90 - ../../stochastic_physics/compns_stochy.F90 - ../../stochastic_physics/stochy_internal_state_mod.F90 - ../../stochastic_physics/stochastic_physics.F90 - ../../stochastic_physics/stochy_patterngenerator.F90 - ../../stochastic_physics/stochy_data_mod.F90 - ../../stochastic_physics/get_stochy_pattern.F90 - ../../stochastic_physics/initialize_spectral_mod.F90 - ../../stochastic_physics/cellular_automata_global.F90 - ../../stochastic_physics/cellular_automata_sgs.F90 - ../../stochastic_physics/update_ca.F90 + ./stochastic_physics_wrapper.F90 ) +target_include_directories(stochastic_physics_wrapper PRIVATE ${CMAKE_BINARY_DIR}/stochastic_physics) if(CCPP) - target_include_directories(stochastic_physics PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src) - target_include_directories(stochastic_physics PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/physics) + target_include_directories(stochastic_physics_wrapper PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/framework/src) + target_include_directories(stochastic_physics_wrapper PRIVATE ${CMAKE_BINARY_DIR}/FV3/ccpp/physics) endif() -target_link_libraries(stochastic_physics fms) -target_link_libraries(stochastic_physics gfsphysics) -target_link_libraries(stochastic_physics fv3core) +# DH* is there a better way to do this? +target_include_directories(stochastic_physics_wrapper PUBLIC $) +# *DH +target_link_libraries(stochastic_physics_wrapper fms) +target_link_libraries(stochastic_physics_wrapper stochastic_physics) +target_link_libraries(stochastic_physics_wrapper gfsphysics) +target_link_libraries(stochastic_physics_wrapper fv3dycore) + diff --git a/stochastic_physics/makefile b/stochastic_physics/makefile new file mode 100644 index 000000000..eb721c8c4 --- /dev/null +++ b/stochastic_physics/makefile @@ -0,0 +1,54 @@ +SHELL = /bin/sh + +inside_nems := $(wildcard ../../../conf/configure.nems) +ifneq ($(strip $(inside_nems)),) + include ../../../conf/configure.nems +else + exist_configure_fv3 := $(wildcard ../conf/configure.fv3) + ifneq ($(strip $(exist_configure_fv3)),) + include ../conf/configure.fv3 + else + $(error "../conf/configure.fv3 file is missing. Run ./configure") + endif + $(info ) + $(info Build standalone FV3 io ...) + $(info ) +endif +$(info $$ESMF_INC is [${ESMF_INC}]) + +LIBRARY = libstochastic_physics_wrapper.a + +FFLAGS += -I$(FMS_DIR) -I ../../stochastic_physics -I../ccpp/physics -I../atmos_cubed_sphere + +SRCS_f = + +SRCS_f90 = + +SRCS_F = + +SRCS_F90 = stochastic_physics_wrapper.F90 + +SRCS_c = + +DEPEND_FILES = $(SRCS_f) $(SRCS_f90) $(SRCS_F) $(SRCS_F90) + +OBJS_f = $(SRCS_f:.f=.o) +OBJS_f90 = $(SRCS_f90:.f90=.o) +OBJS_F = $(SRCS_F:.F=.o) +OBJS_F90 = $(SRCS_F90:.F90=.o) +OBJS_c = $(SRCS_c:.c=.o) + +OBJS = $(OBJS_f) $(OBJS_f90) $(OBJS_F) $(OBJS_F90) $(OBJS_c) + +all default: depend $(LIBRARY) + +$(LIBRARY): $(OBJS) + $(AR) $(ARFLAGS) $@ $? + +.PHONY: clean +clean: + @echo "Cleaning io ... " + @echo + $(RM) -f $(LIBRARY) *__genmod.f90 *.o *.mod *.i90 *.lst *.i + +include ../conf/make.rules diff --git a/stochastic_physics/stochastic_physics_wrapper.F90 b/stochastic_physics/stochastic_physics_wrapper.F90 new file mode 100644 index 000000000..2fb8cafd1 --- /dev/null +++ b/stochastic_physics/stochastic_physics_wrapper.F90 @@ -0,0 +1,249 @@ +module stochastic_physics_wrapper_mod + + use machine, only: kind_phys + + implicit none + + ! For stochastic physics pattern generation + real(kind=kind_phys), dimension(:,:), allocatable, save :: xlat + real(kind=kind_phys), dimension(:,:), allocatable, save :: xlon + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: sppt_wts + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: shum_wts + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: skebu_wts + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: skebv_wts + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: sfc_wts + + ! For cellular automata + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: ugrs + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: qgrs + real(kind=kind_phys), dimension(:,:), allocatable, save :: pgr + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: vvl + real(kind=kind_phys), dimension(:,:,:), allocatable, save :: prsl + real(kind=kind_phys), dimension(:,:), allocatable, save :: condition + real(kind=kind_phys), dimension(:,:), allocatable, save :: ca_deep_cpl, ca_turb_cpl, ca_shal_cpl + real(kind=kind_phys), dimension(:,:), allocatable, save :: ca_deep_diag,ca_turb_diag,ca_shal_diag + real(kind=kind_phys), dimension(:,:), allocatable, save :: ca1_cpl, ca2_cpl, ca3_cpl + real(kind=kind_phys), dimension(:,:), allocatable, save :: ca1_diag,ca2_diag,ca3_diag + + +!---------------- +! Public Entities +!---------------- +! functions + public stochastic_physics_wrapper + + contains + +!------------------------------- +! CCPP step +!------------------------------- + subroutine stochastic_physics_wrapper (GFS_Control, GFS_Data, Atm_block) + +#ifdef OPENMP + use omp_lib +#endif + + use GFS_typedefs, only: GFS_control_type, GFS_data_type + use mpp_mod, only: FATAL, mpp_error + use block_control_mod, only: block_control_type + use atmosphere_mod, only: Atm, mygrid + + use stochastic_physics, only: init_stochastic_physics, run_stochastic_physics + use stochastic_physics_sfc, only: run_stochastic_physics_sfc + use cellular_automata_global_mod, only: cellular_automata_global + use cellular_automata_sgs_mod, only: cellular_automata_sgs + + implicit none + + type(GFS_control_type), intent(inout) :: GFS_Control + type(GFS_data_type), intent(inout) :: GFS_Data(:) + type(block_control_type), intent(inout) :: Atm_block + + integer :: nthreads, nb + +#ifdef OPENMP + nthreads = omp_get_max_threads() +#else + nthreads = 1 +#endif + + ! Initialize + initalize_stochastic_physics: if (GFS_Control%kdt==0) then + + if (GFS_Control%do_sppt .OR. GFS_Control%do_shum .OR. GFS_Control%do_skeb .OR. GFS_Control%do_sfcperts) then + ! Initialize stochastic physics + call init_stochastic_physics(GFS_Control%levs, GFS_Control%blksz, GFS_Control%dtp, & + GFS_Control%input_nml_file, GFS_Control%fn_nml, GFS_Control%nlunit, GFS_Control%do_sppt, GFS_Control%do_shum, & + GFS_Control%do_skeb, GFS_Control%do_sfcperts, GFS_Control%use_zmtnblck, GFS_Control%skeb_npass, GFS_Control%nsfcpert, & + GFS_Control%pertz0, GFS_Control%pertzt, GFS_Control%pertshc, GFS_Control%pertlai, GFS_Control%pertalb, GFS_Control%pertvegf, & + GFS_Control%ak, GFS_Control%bk, nthreads, GFS_Control%master, GFS_Control%communicator) + end if + + ! Get land surface perturbations here (move to "else" block below if wanting to update each time-step) + if (GFS_Control%do_sfcperts) then + ! Copy blocked data into contiguous arrays; no need to copy sfc_wts in (intent out) + allocate(xlat(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(xlon(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(sfc_wts(1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + do nb=1,Atm_block%nblks + xlat(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Grid%xlat(:) + xlon(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Grid%xlon(:) + end do + call run_stochastic_physics_sfc(GFS_Control%blksz, xlat=xlat, xlon=xlon, sfc_wts=sfc_wts) + ! Copy contiguous data back; no need to copy xlat/xlon, these are intent(in) - just deallocate + do nb=1,Atm_block%nblks + GFS_Data(nb)%Coupling%sfc_wts(:,:) = sfc_wts(nb,1:GFS_Control%blksz(nb),:) + end do + deallocate(xlat) + deallocate(xlon) + deallocate(sfc_wts) + end if + + ! Consistency check for cellular automata + if(GFS_Control%do_ca)then + ! DH* The current implementation of cellular_automata assumes that all blocksizes are the + ! same - abort if this is not the case, otherwise proceed with Atm_block%blksz(1) below + if (.not. minval(Atm_block%blksz)==maxval(Atm_block%blksz)) then + call mpp_error(FATAL, 'Logic errror: cellular_automata not compatible with non-uniform blocksizes') + end if + ! *DH + endif + + else initalize_stochastic_physics + + if (GFS_Control%do_sppt .OR. GFS_Control%do_shum .OR. GFS_Control%do_skeb) then + ! Copy blocked data into contiguous arrays; no need to copy weights in (intent(out)) + allocate(xlat(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(xlon(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + do nb=1,Atm_block%nblks + xlat(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Grid%xlat(:) + xlon(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Grid%xlon(:) + end do + if (GFS_Control%do_sppt) then + allocate(sppt_wts(1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + end if + if (GFS_Control%do_shum) then + allocate(shum_wts(1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + end if + if (GFS_Control%do_skeb) then + allocate(skebu_wts(1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + allocate(skebv_wts(1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + end if + call run_stochastic_physics(GFS_Control%levs, GFS_Control%kdt, GFS_Control%phour, GFS_Control%blksz, xlat=xlat, xlon=xlon, & + sppt_wts=sppt_wts, shum_wts=shum_wts, skebu_wts=skebu_wts, skebv_wts=skebv_wts, nthreads=nthreads) + ! Copy contiguous data back; no need to copy xlat/xlon, these are intent(in) - just deallocate + deallocate(xlat) + deallocate(xlon) + if (GFS_Control%do_sppt) then + do nb=1,Atm_block%nblks + GFS_Data(nb)%Coupling%sppt_wts(:,:) = sppt_wts(nb,1:GFS_Control%blksz(nb),:) + end do + deallocate(sppt_wts) + end if + if (GFS_Control%do_shum) then + do nb=1,Atm_block%nblks + GFS_Data(nb)%Coupling%shum_wts(:,:) = shum_wts(nb,1:GFS_Control%blksz(nb),:) + end do + deallocate(shum_wts) + end if + if (GFS_Control%do_skeb) then + do nb=1,Atm_block%nblks + GFS_Data(nb)%Coupling%skebu_wts(:,:) = skebu_wts(nb,1:GFS_Control%blksz(nb),:) + GFS_Data(nb)%Coupling%skebv_wts(:,:) = skebv_wts(nb,1:GFS_Control%blksz(nb),:) + end do + deallocate(skebu_wts) + deallocate(skebv_wts) + end if + end if + + endif initalize_stochastic_physics + + ! Cellular automata code is identical for initialization (kstep=0) and time integration (kstep>0) + if(GFS_Control%do_ca)then + if(GFS_Control%ca_sgs)then + ! Allocate contiguous arrays; copy in as needed + allocate(ugrs (1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + allocate(qgrs (1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + allocate(pgr (1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(vvl (1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + allocate(prsl (1:Atm_block%nblks,maxval(GFS_Control%blksz),1:GFS_Control%levs)) + allocate(ca_deep_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(ca_turb_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(ca_shal_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(condition (1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(ca_deep_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(ca_turb_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + allocate(ca_shal_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz) )) + do nb=1,Atm_block%nblks + ugrs (nb,1:GFS_Control%blksz(nb),:) = GFS_Data(nb)%Statein%ugrs(:,:) + qgrs (nb,1:GFS_Control%blksz(nb),:) = GFS_Data(nb)%Statein%qgrs(:,:,1) + pgr (nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Statein%pgr(:) + vvl (nb,1:GFS_Control%blksz(nb),:) = GFS_Data(nb)%Statein%vvl(:,:) + prsl (nb,1:GFS_Control%blksz(nb),:) = GFS_Data(nb)%Statein%prsl(:,:) + condition (nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Coupling%condition(:) + ca_deep_cpl(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Coupling%ca_deep(:) + ca_turb_cpl(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Coupling%ca_turb(:) + ca_shal_cpl(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Coupling%ca_shal(:) + enddo + call cellular_automata_sgs(GFS_Control%kdt,ugrs,qgrs,pgr,vvl,prsl,condition,ca_deep_cpl,ca_turb_cpl,ca_shal_cpl, & + ca_deep_diag,ca_turb_diag,ca_shal_diag,Atm(mygrid)%domain_for_coupler,Atm_block%nblks, & + Atm_block%isc,Atm_block%iec,Atm_block%jsc,Atm_block%jec,Atm(mygrid)%npx,Atm(mygrid)%npy, GFS_Control%levs, & + GFS_Control%nca,GFS_Control%ncells,GFS_Control%nlives,GFS_Control%nfracseed, & + GFS_Control%nseed,GFS_Control%nthresh,GFS_Control%ca_global,GFS_Control%ca_sgs,GFS_Control%iseed_ca, & + GFS_Control%ca_smooth,GFS_Control%nspinup,Atm_block%blksz(1),GFS_Control%master,GFS_Control%communicator) + ! Copy contiguous data back as needed + do nb=1,Atm_block%nblks + GFS_Data(nb)%Intdiag%ca_deep(:) = ca_deep_diag(nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Intdiag%ca_turb(:) = ca_turb_diag(nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Intdiag%ca_shal(:) = ca_shal_diag(nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Coupling%ca_deep(:) = ca_deep_cpl (nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Coupling%ca_turb(:) = ca_turb_cpl (nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Coupling%ca_shal(:) = ca_shal_cpl (nb,1:GFS_Control%blksz(nb)) + enddo + deallocate(ugrs ) + deallocate(qgrs ) + deallocate(pgr ) + deallocate(vvl ) + deallocate(prsl ) + deallocate(condition ) + deallocate(ca_deep_cpl ) + deallocate(ca_turb_cpl ) + deallocate(ca_shal_cpl ) + deallocate(ca_deep_diag) + deallocate(ca_turb_diag) + deallocate(ca_shal_diag) + endif + if(GFS_Control%ca_global)then + ! Allocate contiguous arrays; no need to copy in (intent out) + allocate(ca1_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(ca2_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(ca3_cpl (1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(ca1_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(ca2_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + allocate(ca3_diag(1:Atm_block%nblks,maxval(GFS_Control%blksz))) + call cellular_automata_global(GFS_Control%kdt,ca1_cpl,ca2_cpl,ca3_cpl,ca1_diag,ca2_diag,ca3_diag,Atm(mygrid)%domain_for_coupler, & + Atm_block%nblks,Atm_block%isc,Atm_block%iec,Atm_block%jsc,Atm_block%jec,Atm(mygrid)%npx,Atm(mygrid)%npy,GFS_Control%levs, & + GFS_Control%nca_g,GFS_Control%ncells_g,GFS_Control%nlives_g,GFS_Control%nfracseed,GFS_Control%nseed_g,GFS_Control%nthresh, & + GFS_Control%ca_global,GFS_Control%ca_sgs,GFS_Control%iseed_ca,GFS_Control%ca_smooth,GFS_Control%nspinup,Atm_block%blksz(1), & + GFS_Control%nsmooth,GFS_Control%ca_amplitude,GFS_Control%master,GFS_Control%communicator) + ! Copy contiguous data back + do nb=1,Atm_block%nblks + GFS_Data(nb)%Coupling%ca1(:) = ca1_cpl (nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Coupling%ca2(:) = ca2_cpl (nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Coupling%ca3(:) = ca3_cpl (nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Intdiag%ca1(:) = ca1_diag(nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Intdiag%ca2(:) = ca2_diag(nb,1:GFS_Control%blksz(nb)) + GFS_Data(nb)%Intdiag%ca3(:) = ca3_diag(nb,1:GFS_Control%blksz(nb)) + enddo + deallocate(ca1_cpl ) + deallocate(ca2_cpl ) + deallocate(ca3_cpl ) + deallocate(ca1_diag) + deallocate(ca2_diag) + deallocate(ca3_diag) + endif + endif + + end subroutine stochastic_physics_wrapper + +end module stochastic_physics_wrapper_mod