From 74f3569bca40f619bb35ddce60d9185201ff628a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 5 Dec 2022 18:08:51 -0700 Subject: [PATCH 1/8] Add cmake option JEDI = no physics whatsoever --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 415a0068f..e17ef3fd0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,7 @@ option(DEBUG "Enable compiler definition -DDEBUG" OFF) option(MOVING_NEST "Enable compiler definition -DMOVING_NEST" OFF) option(MULTI_GASES "Enable compiler definition -DMULTI_GASES" OFF) option(USE_GFSL63 "Enable compiler definition -DUSE_GFSL63" OFF) +option(JEDI "Enable compiler definition -DJEDI" OFF) option(GFS_PHYS "Enable compiler definition -DGFS_PHYS" OFF) option(GFS_TYPES "Enable compiler definition -DGFS_TYPES" OFF) option(use_WRTCOMP "Enable compiler definition -Duse_WRTCOMP" OFF) @@ -137,6 +138,10 @@ if(USE_GFSL63) list(APPEND fv3_defs USE_GFSL63) endif() +if(JEDI) + list(APPEND fv3_defs JEDI) +endif() + if(GFS_PHYS) list(APPEND fv3_defs GFS_PHYS) endif() From bd5305c5b914bac5144c05a0c32d2d7bcbe9d295 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 5 Dec 2022 18:12:19 -0700 Subject: [PATCH 2/8] Add necessary ifdefs for cmake option JEDI --- CMakeLists.txt | 3 ++ model/fv_dynamics.F90 | 60 ++++++++++++++++++++++++++++++++++-- model/fv_fill.F90 | 5 +++ model/fv_mapz.F90 | 41 ++++++++++++++++++------ moving_nest/bounding_box.F90 | 2 +- 5 files changed, 97 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e17ef3fd0..b2bd03475 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,6 +161,9 @@ if(MOVING_NEST) endif() if(MULTI_GASES) + if(JEDI) + message(FATAL_ERROR "Turning on MULTI_GASES and JEDI at the same time doesn't make sense") + endif() list(APPEND fv3_defs MULTI_GASES) endif() diff --git a/model/fv_dynamics.F90 b/model/fv_dynamics.F90 index ed6836f15..6ccbbb75f 100644 --- a/model/fv_dynamics.F90 +++ b/model/fv_dynamics.F90 @@ -190,11 +190,13 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, parent_grid, domain, diss_est, inline_mp) use mpp_mod, only: FATAL, mpp_error +#ifndef JEDI use ccpp_static_api, only: ccpp_physics_timestep_init, & ccpp_physics_timestep_finalize use CCPP_data, only: ccpp_suite use CCPP_data, only: cdata => cdata_tile use CCPP_data, only: GFDL_interstitial +#endif use molecular_diffusion_mod, only: md_time, md_wait_sec, md_tadj_layers, & thermosphere_adjustment @@ -271,11 +273,17 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, ! Local Arrays real:: ws(bd%is:bd%ie,bd%js:bd%je) +#ifdef JEDI + real:: te_2d(bd%is:bd%ie,bd%js:bd%je) +#endif real:: teq(bd%is:bd%ie,bd%js:bd%je) real:: ps2(bd%isd:bd%ied,bd%jsd:bd%jed) real:: m_fac(bd%is:bd%ie,bd%js:bd%je) real:: pfull(npz) real, dimension(bd%is:bd%ie):: cvm +#ifdef JEDI + real, allocatable :: dp1(:,:,:), dtdt_m(:,:,:), cappa(:,:,:) +#endif #ifdef MULTI_GASES real, allocatable :: kapad(:,:,:) #endif @@ -290,6 +298,9 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, integer :: rainwat = -999, snowwat = -999, graupel = -999, hailwat = -999, cld_amt = -999 integer :: theta_d = -999 logical used, do_omega +#ifdef JEDI + logical :: last_step +#endif integer, parameter :: max_packs=13 type(group_halo_update_type), save :: i_pack(max_packs) integer :: is, ie, js, je @@ -299,11 +310,13 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, real :: time_total integer :: seconds, days +#ifndef JEDI ccpp_associate: associate( cappa => GFDL_interstitial%cappa, & dp1 => GFDL_interstitial%te0, & dtdt_m => GFDL_interstitial%dtdt, & last_step => GFDL_interstitial%last_step, & te_2d => GFDL_interstitial%te0_2d ) +#endif is = bd%is ie = bd%ie @@ -324,7 +337,17 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, nq = nq_tot - flagstruct%dnats nr = nq_tot - flagstruct%dnrts rdg = -rdgas * agrav +#ifdef JEDI + allocate ( dp1(isd:ied, jsd:jed, 1:npz) ) +#ifdef MOIST_CAPPA + allocate ( cappa(isd:ied,jsd:jed,npz) ) + call init_ijk_mem(isd,ied, jsd,jed, npz, cappa, 0.) +#else + allocate ( cappa(isd:isd,jsd:jsd,1) ) + cappa = 0. +#endif +#else ! Call CCPP timestep init call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), group_name="fast_physics", ierr=ierr) ! Reset all interstitial variables for CCPP version @@ -333,6 +356,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, if (flagstruct%do_sat_adj) then GFDL_interstitial%out_dt = (idiag%id_mdt > 0) end if +#endif #ifdef MULTI_GASES allocate ( kapad(isd:ied, jsd:jed, npz) ) @@ -424,12 +448,17 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, enddo if ( hydrostatic ) then +#ifdef JEDI +!$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,dp1,zvir,nwat,q,q_con,sphum,liq_wat, & +!$OMP rainwat,ice_wat,snowwat,graupel,hailwat) private(cvm,i,j,k) +#else #ifdef __GFORTRAN__ !$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,zvir,nwat,q,q_con,sphum,liq_wat, & #else !$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,dp1,zvir,nwat,q,q_con,sphum,liq_wat, & #endif !$OMP rainwat,ice_wat,snowwat,graupel,hailwat) private(cvm,i,j,k) +#endif do k=1,npz do j=js,je #ifdef USE_COND @@ -442,6 +471,12 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, enddo enddo else +#ifdef JEDI +!$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,dp1,zvir,q,q_con,sphum,liq_wat, & +!$OMP rainwat,ice_wat,snowwat,graupel,hailwat,pkz,flagstruct, & +!$OMP cappa,kappa,rdg,delp,pt,delz,nwat) & +!$OMP private(cvm) +#else #ifdef __GFORTRAN__ !$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,zvir,q,q_con,sphum,liq_wat, & #else @@ -457,6 +492,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, !$OMP cappa,kappa,rdg,delp,pt,delz,nwat) & #endif !$OMP private(cvm,i,j,k) +#endif do k=1,npz if ( flagstruct%moist_phys ) then do j=js,je @@ -583,7 +619,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, pt_initialized = .true. endif else -#ifdef __GFORTRAN__ +#if defined(__GFORTRAN__) && ! defined(JEDI) !$OMP parallel do default(none) shared(is,ie,js,je,npz,pt,pkz,q_con) #else !$OMP parallel do default(none) shared(is,ie,js,je,npz,pt,dp1,pkz,q_con) @@ -611,10 +647,15 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, mdt = bdt / real(k_split) if ( idiag%id_mdt > 0 .and. (.not. do_adiabatic_init) ) then +#ifdef JEDI + allocate ( dtdt_m(is:ie,js:je,npz) ) +!$OMP parallel do default(none) shared(is,ie,js,je,npz,dtdt_m) +#else #ifdef __GFORTRAN__ !$OMP parallel do default(none) shared(is,ie,js,je,npz) #else !$OMP parallel do default(none) shared(is,ie,js,je,npz,dtdt_m) +#endif #endif do k=1,npz do j=js,je @@ -645,7 +686,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, call start_group_halo_update(i_pack(8), u, v, domain, gridtype=DGRID_NE) #endif call timing_off('COMM_TOTAL') -#ifdef __GFORTRAN__ +#if defined(__GFORTRAN__) && ! defined(JEDI) !$OMP parallel do default(none) shared(isd,ied,jsd,jed,npz,delp) #else !$OMP parallel do default(none) shared(isd,ied,jsd,jed,npz,dp1,delp) @@ -845,11 +886,15 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, endif if ( idiag%id_mdt > 0 .and. (.not.do_adiabatic_init) ) then +#ifdef JEDI +!$OMP parallel do default(none) shared(is,ie,js,je,npz,dtdt_m,bdt) +#else ! Output temperature tendency due to inline moist physics: #ifdef __GFORTRAN__ !$OMP parallel do default(none) shared(is,ie,js,je,npz,bdt) #else !$OMP parallel do default(none) shared(is,ie,js,je,npz,dtdt_m,bdt) +#endif #endif do k=1,npz do j=js,je @@ -860,6 +905,9 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, enddo ! call prt_mxm('Fast DTDT (deg/Day)', dtdt_m, is, ie, js, je, 0, npz, 1., gridstruct%area_64, domain) used = send_data(idiag%id_mdt, dtdt_m, fv_time) +#ifdef JEDI + deallocate ( dtdt_m ) +#endif endif if( nwat==7 ) then @@ -980,7 +1028,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, endif if( (flagstruct%consv_am.or.idiag%id_amdt>0) .and. (.not.do_adiabatic_init) ) then -#ifdef __GFORTRAN__ +#if defined(__GFORTRAN__) && ! defined(JEDI) !$OMP parallel do default(none) shared(is,ie,js,je,teq,dt2,ps2,ps,idiag) #else !$OMP parallel do default(none) shared(is,ie,js,je,te_2d,teq,dt2,ps2,ps,idiag) @@ -1028,6 +1076,10 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, 911 call cubed_to_latlon(u, v, ua, va, gridstruct, & npx, npy, npz, 1, gridstruct%grid_type, domain, gridstruct%bounded_domain, flagstruct%c2l_ord, bd) +#ifndef JEDI + deallocate(dp1) + deallocate(cappa) +#endif #ifdef MULTI_GASES deallocate(kapad) #endif @@ -1062,10 +1114,12 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, endif endif +#ifndef JEDI ! Call CCPP timestep finalize call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), group_name="fast_physics", ierr=ierr) end associate ccpp_associate +#endif end subroutine fv_dynamics diff --git a/model/fv_fill.F90 b/model/fv_fill.F90 index 5579c7221..b39ae8eee 100644 --- a/model/fv_fill.F90 +++ b/model/fv_fill.F90 @@ -37,7 +37,12 @@ module fv_fill_mod ! use mpp_domains_mod, only: mpp_update_domains, domain2D +#ifdef GFS_TYPES use GFS_typedefs, only: kind_phys +#else + use platform_mod, only: kind_phys => r8_kind +#endif +! implicit none public fillz diff --git a/model/fv_mapz.F90 b/model/fv_mapz.F90 index 00914df76..9c42a9fb5 100644 --- a/model/fv_mapz.F90 +++ b/model/fv_mapz.F90 @@ -95,11 +95,13 @@ module fv_mapz_mod use fv_arrays_mod, only: fv_grid_type, fv_grid_bounds_type, R_GRID, inline_mp_type use fv_timing_mod, only: timing_on, timing_off use fv_mp_mod, only: is_master, mp_reduce_min, mp_reduce_max +#ifndef JEDI ! CCPP fast physics use ccpp_static_api, only: ccpp_physics_run use CCPP_data, only: ccpp_suite use CCPP_data, only: cdata => cdata_tile use CCPP_data, only: GFDL_interstitial +#endif #ifdef MULTI_GASES use multi_gases_mod, only: virq, virqd, vicpqd, vicvqd, num_gas #endif @@ -223,14 +225,18 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & real, dimension(is:ie+1,km+1):: pe0, pe3 real, dimension(is:ie):: gsize, gz, cvm, qv +#ifdef JEDI + logical :: fast_mp_consv +#endif real rcp, rg, rrg, bkh, dtmp, k1k integer:: i,j,k integer:: kdelz - integer:: nt, liq_wat, ice_wat, rainwat, snowwat, cld_amt, graupel, hailwat, ccn_cm3, iq, n, kmp, kp, k_next + integer:: nt, liq_wat, ice_wat, rainwat, snowwat, cld_amt, graupel, hailwat, ccn_cm3, iq, n, kp, k_next integer :: ierr - ccpp_associate: associate( fast_mp_consv => GFDL_interstitial%fast_mp_consv, & - kmp => GFDL_interstitial%kmp ) +#ifndef JEDI + ccpp_associate: associate( fast_mp_consv => GFDL_interstitial%fast_mp_consv ) +#endif k1k = rdgas/cv_air ! akap / (1.-akap) = rg/Cv=0.4 rg = rdgas @@ -665,31 +671,40 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & 1000 continue #ifdef __GFORTRAN__ -!$OMP parallel default(none) shared(is,ie,js,je,km,ptop,u,v,pe,ua,va,isd,ied,jsd,jed,kord_mt, & +!$OMP parallel default(none) shared(is,ie,js,je,km,ptop,u,v,pe,ua,va,isd,ied,jsd,jed,kord_mt, & !$OMP te_2d,te,delp,hydrostatic,hs,rg,pt,peln, adiabatic, & !$OMP cp,delz,nwat,rainwat,liq_wat,ice_wat,snowwat, & -!$OMP graupel,hailwat,q_con,r_vir,sphum,w,pk,pkz,last_step,consv, & +!$OMP graupel,hailwat,q_con,r_vir,sphum,w,pk,pkz,last_step,consv,& !$OMP do_adiabatic_init,zsum1,zsum0,te0_2d,domain, & !$OMP ng,gridstruct,E_Flux,pdt,dtmp,reproduce_sum,q, & !$OMP mdt,cld_amt,cappa,dtdt,out_dt,rrg,akap,do_sat_adj, & -!$OMP kord_tm,pe4, npx,npy,ccn_cm3,u_dt,v_dt, c2l_ord,bd,dp0,ps, & -!$OMP cdata,GFDL_interstitial) & +!$OMP kord_tm,pe4, npx,npy,ccn_cm3,u_dt,v_dt, c2l_ord,bd,dp0,ps & +#ifdef JEDI +!$OMP ) & +#else +!$OMP ,cdata,GFDL_interstitial) & !$OMP shared(ccpp_suite) & +#endif #ifdef MULTI_GASES !$OMP shared(num_gas) & #endif !$OMP private(q2,pe0,pe1,pe2,pe3,qv,cvm,gz,gsize,phis,kdelz,dp2,t0, ierr) #else -!$OMP parallel default(none) shared(is,ie,js,je,km,kmp,ptop,u,v,pe,ua,va,isd,ied,jsd,jed,kord_mt, & +!$OMP parallel default(none) shared(is,ie,js,je,km,ptop,u,v,pe,ua,va,isd,ied,jsd,jed,kord_mt, & !$OMP te_2d,te,delp,hydrostatic,hs,rg,pt,peln, adiabatic, & !$OMP cp,delz,nwat,rainwat,liq_wat,ice_wat,snowwat, & -!$OMP graupel,hailwat,q_con,r_vir,sphum,w,pk,pkz,last_step,consv, & +!$OMP graupel,hailwat,q_con,r_vir,sphum,w,pk,pkz,last_step,consv,& !$OMP do_adiabatic_init,zsum1,zsum0,te0_2d,domain, & !$OMP ng,gridstruct,E_Flux,pdt,dtmp,reproduce_sum,q, & !$OMP mdt,cld_amt,cappa,dtdt,out_dt,rrg,akap,do_sat_adj, & !$OMP fast_mp_consv,kord_tm, pe4,npx,npy, ccn_cm3, & -!$OMP u_dt,v_dt,c2l_ord,bd,dp0,ps,cdata,GFDL_interstitial) & +!$OMP u_dt,v_dt,c2l_ord,bd,dp0,ps +#ifdef JEDI +!$OMP ) & +#else +!$OMP ,cdata,GFDL_interstitial) & !$OMP shared(ccpp_suite) & +#endif #ifdef MULTI_GASES !$OMP shared(num_gas) & #endif @@ -837,6 +852,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & ! if ( (.not.do_adiabatic_init) .and. do_sat_adj ) then if ( do_sat_adj ) then +#ifndef JEDI call timing_on('sat_adj2') ! Call to CCPP fast_physics group if (cdata%initialized()) then @@ -849,6 +865,9 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & call mpp_error (FATAL, 'Lagrangian_to_Eulerian: can not call CCPP fast physics because CCPP not initialized') endif call timing_off('sat_adj2') +#else + call mpp_error (FATAL, 'Lagrangian_to_Eulerian: calling saturation adjustment for no-physics JEDI configuration makes no sense') +#endif endif ! do_sat_adj if ( last_step ) then @@ -924,7 +943,9 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & endif !$OMP end parallel +#ifndef JEDI end associate ccpp_associate +#endif end subroutine Lagrangian_to_Eulerian diff --git a/moving_nest/bounding_box.F90 b/moving_nest/bounding_box.F90 index cd93a8308..9fba6aa69 100644 --- a/moving_nest/bounding_box.F90 +++ b/moving_nest/bounding_box.F90 @@ -36,7 +36,7 @@ module bounding_box_mod #ifdef GFS_TYPES use GFS_typedefs, only : kind_phys #else - use IPD_typedefs, only : kind_phys => IPD_kind_phys + use platform_mod, only : kind_phys => r8_kind #endif ! Simple aggregation of the start and end indices of a 2D grid From 352e8eee0db21f46fb9fb121a2d516362195a2c3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 5 Dec 2022 20:19:00 -0700 Subject: [PATCH 3/8] Bug fix in model/fv_dynamics.F90 --- model/fv_dynamics.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/fv_dynamics.F90 b/model/fv_dynamics.F90 index 6ccbbb75f..5da57f15f 100644 --- a/model/fv_dynamics.F90 +++ b/model/fv_dynamics.F90 @@ -1076,7 +1076,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, 911 call cubed_to_latlon(u, v, ua, va, gridstruct, & npx, npy, npz, 1, gridstruct%grid_type, domain, gridstruct%bounded_domain, flagstruct%c2l_ord, bd) -#ifndef JEDI +#ifdef JEDI deallocate(dp1) deallocate(cappa) #endif From d8ec8469064e815f0910ca2c0646fbd2cfede17d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 5 Dec 2022 21:11:55 -0700 Subject: [PATCH 4/8] Bug fix in model/fv_mapz.F90 --- model/fv_mapz.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/fv_mapz.F90 b/model/fv_mapz.F90 index 9c42a9fb5..c2e70bcf0 100644 --- a/model/fv_mapz.F90 +++ b/model/fv_mapz.F90 @@ -698,7 +698,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & !$OMP ng,gridstruct,E_Flux,pdt,dtmp,reproduce_sum,q, & !$OMP mdt,cld_amt,cappa,dtdt,out_dt,rrg,akap,do_sat_adj, & !$OMP fast_mp_consv,kord_tm, pe4,npx,npy, ccn_cm3, & -!$OMP u_dt,v_dt,c2l_ord,bd,dp0,ps +!$OMP u_dt,v_dt,c2l_ord,bd,dp0,ps & #ifdef JEDI !$OMP ) & #else From dbce1269aacf0b0d049ab9902dcc68a6bf874ed1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 6 Dec 2022 08:49:41 -0700 Subject: [PATCH 5/8] Rename JEDI cmake option to NO_PHYS, and make it work without GFS_PHYS --- CMakeLists.txt | 13 ++++++++----- model/fv_dynamics.F90 | 32 ++++++++++++++++---------------- model/fv_mapz.F90 | 16 ++++++++-------- model/fv_sg.F90 | 16 ++++++++-------- tools/fv_diagnostics.F90 | 6 +++--- tools/test_cases.F90 | 4 ++-- 6 files changed, 45 insertions(+), 42 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b2bd03475..89478b0fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,7 @@ option(DEBUG "Enable compiler definition -DDEBUG" OFF) option(MOVING_NEST "Enable compiler definition -DMOVING_NEST" OFF) option(MULTI_GASES "Enable compiler definition -DMULTI_GASES" OFF) option(USE_GFSL63 "Enable compiler definition -DUSE_GFSL63" OFF) -option(JEDI "Enable compiler definition -DJEDI" OFF) +option(NO_PHYS "Enable compiler definition -DNO_PHYS" OFF) option(GFS_PHYS "Enable compiler definition -DGFS_PHYS" OFF) option(GFS_TYPES "Enable compiler definition -DGFS_TYPES" OFF) option(use_WRTCOMP "Enable compiler definition -Duse_WRTCOMP" OFF) @@ -138,11 +138,14 @@ if(USE_GFSL63) list(APPEND fv3_defs USE_GFSL63) endif() -if(JEDI) - list(APPEND fv3_defs JEDI) +if(NO_PHYS) + list(APPEND fv3_defs NO_PHYS) endif() if(GFS_PHYS) + if(NO_PHYS) + message(FATAL_ERROR "Turning on GFS_PHYS and NO_PHYS at the same time doesn't make sense") + endif() list(APPEND fv3_defs GFS_PHYS) endif() @@ -161,8 +164,8 @@ if(MOVING_NEST) endif() if(MULTI_GASES) - if(JEDI) - message(FATAL_ERROR "Turning on MULTI_GASES and JEDI at the same time doesn't make sense") + if(NO_PHYS) + message(FATAL_ERROR "Turning on MULTI_GASES and NO_PHYS at the same time doesn't make sense") endif() list(APPEND fv3_defs MULTI_GASES) endif() diff --git a/model/fv_dynamics.F90 b/model/fv_dynamics.F90 index 5da57f15f..3817af68c 100644 --- a/model/fv_dynamics.F90 +++ b/model/fv_dynamics.F90 @@ -190,7 +190,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, parent_grid, domain, diss_est, inline_mp) use mpp_mod, only: FATAL, mpp_error -#ifndef JEDI +#ifndef NO_PHYS use ccpp_static_api, only: ccpp_physics_timestep_init, & ccpp_physics_timestep_finalize use CCPP_data, only: ccpp_suite @@ -273,7 +273,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, ! Local Arrays real:: ws(bd%is:bd%ie,bd%js:bd%je) -#ifdef JEDI +#ifdef NO_PHYS real:: te_2d(bd%is:bd%ie,bd%js:bd%je) #endif real:: teq(bd%is:bd%ie,bd%js:bd%je) @@ -281,7 +281,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, real:: m_fac(bd%is:bd%ie,bd%js:bd%je) real:: pfull(npz) real, dimension(bd%is:bd%ie):: cvm -#ifdef JEDI +#ifdef NO_PHYS real, allocatable :: dp1(:,:,:), dtdt_m(:,:,:), cappa(:,:,:) #endif #ifdef MULTI_GASES @@ -298,7 +298,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, integer :: rainwat = -999, snowwat = -999, graupel = -999, hailwat = -999, cld_amt = -999 integer :: theta_d = -999 logical used, do_omega -#ifdef JEDI +#ifdef NO_PHYS logical :: last_step #endif integer, parameter :: max_packs=13 @@ -310,7 +310,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, real :: time_total integer :: seconds, days -#ifndef JEDI +#ifndef NO_PHYS ccpp_associate: associate( cappa => GFDL_interstitial%cappa, & dp1 => GFDL_interstitial%te0, & dtdt_m => GFDL_interstitial%dtdt, & @@ -337,7 +337,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, nq = nq_tot - flagstruct%dnats nr = nq_tot - flagstruct%dnrts rdg = -rdgas * agrav -#ifdef JEDI +#ifdef NO_PHYS allocate ( dp1(isd:ied, jsd:jed, 1:npz) ) #ifdef MOIST_CAPPA @@ -448,7 +448,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, enddo if ( hydrostatic ) then -#ifdef JEDI +#ifdef NO_PHYS !$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,dp1,zvir,nwat,q,q_con,sphum,liq_wat, & !$OMP rainwat,ice_wat,snowwat,graupel,hailwat) private(cvm,i,j,k) #else @@ -471,7 +471,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, enddo enddo else -#ifdef JEDI +#ifdef NO_PHYS !$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,dp1,zvir,q,q_con,sphum,liq_wat, & !$OMP rainwat,ice_wat,snowwat,graupel,hailwat,pkz,flagstruct, & !$OMP cappa,kappa,rdg,delp,pt,delz,nwat) & @@ -619,7 +619,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, pt_initialized = .true. endif else -#if defined(__GFORTRAN__) && ! defined(JEDI) +#if defined(__GFORTRAN__) && ! defined(NO_PHYS) !$OMP parallel do default(none) shared(is,ie,js,je,npz,pt,pkz,q_con) #else !$OMP parallel do default(none) shared(is,ie,js,je,npz,pt,dp1,pkz,q_con) @@ -647,7 +647,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, mdt = bdt / real(k_split) if ( idiag%id_mdt > 0 .and. (.not. do_adiabatic_init) ) then -#ifdef JEDI +#ifdef NO_PHYS allocate ( dtdt_m(is:ie,js:je,npz) ) !$OMP parallel do default(none) shared(is,ie,js,je,npz,dtdt_m) #else @@ -686,7 +686,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, call start_group_halo_update(i_pack(8), u, v, domain, gridtype=DGRID_NE) #endif call timing_off('COMM_TOTAL') -#if defined(__GFORTRAN__) && ! defined(JEDI) +#if defined(__GFORTRAN__) && ! defined(NO_PHYS) !$OMP parallel do default(none) shared(isd,ied,jsd,jed,npz,delp) #else !$OMP parallel do default(none) shared(isd,ied,jsd,jed,npz,dp1,delp) @@ -886,7 +886,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, endif if ( idiag%id_mdt > 0 .and. (.not.do_adiabatic_init) ) then -#ifdef JEDI +#ifdef NO_PHYS !$OMP parallel do default(none) shared(is,ie,js,je,npz,dtdt_m,bdt) #else ! Output temperature tendency due to inline moist physics: @@ -905,7 +905,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, enddo ! call prt_mxm('Fast DTDT (deg/Day)', dtdt_m, is, ie, js, je, 0, npz, 1., gridstruct%area_64, domain) used = send_data(idiag%id_mdt, dtdt_m, fv_time) -#ifdef JEDI +#ifdef NO_PHYS deallocate ( dtdt_m ) #endif endif @@ -1028,7 +1028,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, endif if( (flagstruct%consv_am.or.idiag%id_amdt>0) .and. (.not.do_adiabatic_init) ) then -#if defined(__GFORTRAN__) && ! defined(JEDI) +#if defined(__GFORTRAN__) && ! defined(NO_PHYS) !$OMP parallel do default(none) shared(is,ie,js,je,teq,dt2,ps2,ps,idiag) #else !$OMP parallel do default(none) shared(is,ie,js,je,te_2d,teq,dt2,ps2,ps,idiag) @@ -1076,7 +1076,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, 911 call cubed_to_latlon(u, v, ua, va, gridstruct, & npx, npy, npz, 1, gridstruct%grid_type, domain, gridstruct%bounded_domain, flagstruct%c2l_ord, bd) -#ifdef JEDI +#ifdef NO_PHYS deallocate(dp1) deallocate(cappa) #endif @@ -1114,7 +1114,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, endif endif -#ifndef JEDI +#ifndef NO_PHYS ! Call CCPP timestep finalize call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), group_name="fast_physics", ierr=ierr) diff --git a/model/fv_mapz.F90 b/model/fv_mapz.F90 index c2e70bcf0..c3dd80661 100644 --- a/model/fv_mapz.F90 +++ b/model/fv_mapz.F90 @@ -95,7 +95,7 @@ module fv_mapz_mod use fv_arrays_mod, only: fv_grid_type, fv_grid_bounds_type, R_GRID, inline_mp_type use fv_timing_mod, only: timing_on, timing_off use fv_mp_mod, only: is_master, mp_reduce_min, mp_reduce_max -#ifndef JEDI +#ifndef NO_PHYS ! CCPP fast physics use ccpp_static_api, only: ccpp_physics_run use CCPP_data, only: ccpp_suite @@ -225,7 +225,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & real, dimension(is:ie+1,km+1):: pe0, pe3 real, dimension(is:ie):: gsize, gz, cvm, qv -#ifdef JEDI +#ifdef NO_PHYS logical :: fast_mp_consv #endif real rcp, rg, rrg, bkh, dtmp, k1k @@ -234,7 +234,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & integer:: nt, liq_wat, ice_wat, rainwat, snowwat, cld_amt, graupel, hailwat, ccn_cm3, iq, n, kp, k_next integer :: ierr -#ifndef JEDI +#ifndef NO_PHYS ccpp_associate: associate( fast_mp_consv => GFDL_interstitial%fast_mp_consv ) #endif @@ -679,7 +679,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & !$OMP ng,gridstruct,E_Flux,pdt,dtmp,reproduce_sum,q, & !$OMP mdt,cld_amt,cappa,dtdt,out_dt,rrg,akap,do_sat_adj, & !$OMP kord_tm,pe4, npx,npy,ccn_cm3,u_dt,v_dt, c2l_ord,bd,dp0,ps & -#ifdef JEDI +#ifdef NO_PHYS !$OMP ) & #else !$OMP ,cdata,GFDL_interstitial) & @@ -699,7 +699,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & !$OMP mdt,cld_amt,cappa,dtdt,out_dt,rrg,akap,do_sat_adj, & !$OMP fast_mp_consv,kord_tm, pe4,npx,npy, ccn_cm3, & !$OMP u_dt,v_dt,c2l_ord,bd,dp0,ps & -#ifdef JEDI +#ifdef NO_PHYS !$OMP ) & #else !$OMP ,cdata,GFDL_interstitial) & @@ -852,7 +852,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & ! if ( (.not.do_adiabatic_init) .and. do_sat_adj ) then if ( do_sat_adj ) then -#ifndef JEDI +#ifndef NO_PHYS call timing_on('sat_adj2') ! Call to CCPP fast_physics group if (cdata%initialized()) then @@ -866,7 +866,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & endif call timing_off('sat_adj2') #else - call mpp_error (FATAL, 'Lagrangian_to_Eulerian: calling saturation adjustment for no-physics JEDI configuration makes no sense') + call mpp_error (FATAL, 'Lagrangian_to_Eulerian: calling saturation adjustment for no-physics configuration makes no sense') #endif endif ! do_sat_adj @@ -943,7 +943,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, & endif !$OMP end parallel -#ifndef JEDI +#ifndef NO_PHYS end associate ccpp_associate #endif diff --git a/model/fv_sg.F90 b/model/fv_sg.F90 index e60693203..056117c50 100644 --- a/model/fv_sg.F90 +++ b/model/fv_sg.F90 @@ -58,7 +58,7 @@ module fv_sg_mod use constants_mod, only: rdgas, rvgas, cp_air, cp_vapor, hlv, hlf, kappa, grav use tracer_manager_mod, only: get_tracer_index use field_manager_mod, only: MODEL_ATMOS -#ifndef GFS_PHYS +#if ! defined(GFS_PHYS) && ! defined(NO_PHYS) use gfdl_cloud_microphys_mod, only: wqs1, wqs2, wqsat2_moist #endif use fv_mp_mod, only: mp_reduce_min, is_master @@ -104,7 +104,7 @@ module fv_sg_mod contains -#ifdef GFS_PHYS +#if defined(GFS_PHYS) || defined(NO_PHYS) !>@brief The subroutine 'fv_subgrid_z' performs dry convective adjustment mixing. !>@details Two different versions of this subroutine are implemented: !!-one for the GFS physics @@ -685,7 +685,7 @@ subroutine fv_subgrid_z( isd, ied, jsd, jed, is, ie, js, je, km, nq, dt, & u_dt(i,j,k) = rdt*(u0(i,k) - ua(i,j,k)) v_dt(i,j,k) = rdt*(v0(i,k) - va(i,j,k)) ta(i,j,k) = t0(i,k) ! *** temperature updated *** -#ifdef GFS_PHYS +#if defined(GFS_PHYS) || defined (NO_PHYS) ua(i,j,k) = u0(i,k) va(i,j,k) = v0(i,k) #endif @@ -1266,7 +1266,7 @@ subroutine fv_subgrid_z( isd, ied, jsd, jed, is, ie, js, je, km, nq, dt, & !---------------------- ! Saturation adjustment !---------------------- -#ifndef GFS_PHYS +#if ! defined(GFS_PHYS) && ! defined(NO_PHYS) if ( nwat == 6 ) then do k=1, kbot if ( hydrostatic ) then @@ -1341,7 +1341,7 @@ subroutine fv_subgrid_z( isd, ied, jsd, jed, is, ie, js, je, km, nq, dt, & u_dt(i,j,k) = rdt*(u0(i,k) - ua(i,j,k)) v_dt(i,j,k) = rdt*(v0(i,k) - va(i,j,k)) ta(i,j,k) = t0(i,k) ! *** temperature updated *** -#ifdef GFS_PHYS +#if defined(GFS_PHYS) || defined(NO_PHYS) ua(i,j,k) = u0(i,k) va(i,j,k) = v0(i,k) #endif @@ -1765,7 +1765,7 @@ subroutine neg_adj3(is, ie, js, je, ng, kbot, hydrostatic, peln, delz, pt, dp, !****************************************** ! Fast moist physics: Saturation adjustment !****************************************** -#ifndef GFS_PHYS +#if ! defined(GFS_PHYS) && ! defined(NO_PHYS) if ( sat_adj ) then do j=js, je @@ -2141,7 +2141,7 @@ subroutine neg_adj4(is, ie, js, je, ng, kbot, hydrostatic, & !****************************************** ! Fast moist physics: Saturation adjustment !****************************************** -#ifndef GFS_PHYS +#if ! defined(GFS_PHYS) && ! defined(NO_PHYS) if ( sat_adj ) then do j=js, je @@ -2473,7 +2473,7 @@ subroutine neg_adj2(is, ie, js, je, ng, kbot, hydrostatic, & !****************************************** ! Fast moist physics: Saturation adjustment !****************************************** -#ifndef GFS_PHYS +#if ! defined(GFS_PHYS) && ! defined(NO_PHYS) if ( sat_adj ) then do j=js, je diff --git a/tools/fv_diagnostics.F90 b/tools/fv_diagnostics.F90 index 205c4ae8a..9fc870e1c 100644 --- a/tools/fv_diagnostics.F90 +++ b/tools/fv_diagnostics.F90 @@ -137,7 +137,7 @@ module fv_diagnostics_mod use fv_arrays_mod, only: max_step -#ifndef GFS_PHYS +#if ! defined(GFS_PHYS) && ! defined(NO_PHYS) use gfdl_mp_mod, only: wqs1, qsmith_init, c_liq #endif @@ -1506,7 +1506,7 @@ subroutine fv_diag_init(Atm, axes, Time, npx, npy, npz, p_ref) module_is_initialized=.true. istep = 0 -#ifndef GFS_PHYS +#if ! defined(GFS_PHYS) && ! defined(NO_PHYS) if(id_theta_e >0 ) call qsmith_init #endif @@ -3245,7 +3245,7 @@ subroutine fv_diag(Atm, zvir, Time, print_freq) endif -#ifdef GFS_PHYS +#if defined(GFS_PHYS) || defined(NO_PHYS) if(id_delp > 0 .or. id_cape > 0 .or. id_cin > 0 .or. ((.not. Atm(n)%flagstruct%hydrostatic) .and. id_pfnh > 0)) then do k=1,npz do j=jsc,jec diff --git a/tools/test_cases.F90 b/tools/test_cases.F90 index 6ef0b7534..3e1538eef 100644 --- a/tools/test_cases.F90 +++ b/tools/test_cases.F90 @@ -5652,7 +5652,7 @@ end subroutine superK_u subroutine SuperCell_Sounding(km, ps, pk1, tp, qp) -#ifndef GFS_PHYS +#if ! defined(GFS_PHYS) && ! defined(NO_PHYS) use gfdl_cloud_microphys_mod, only: wqsat_moist, qsmith_init, qs_blend #endif ! Morris Weisman & J. Klemp 2002 sounding @@ -5676,7 +5676,7 @@ subroutine SuperCell_Sounding(km, ps, pk1, tp, qp) real:: dz0, zvir, fac_z, pk0, temp1, p2 integer:: k, n, kk -#ifdef GFS_PHYS +#if defined(GFS_PHYS) || defined(NO_PHYS) call mpp_error(FATAL, 'SuperCell sounding cannot perform with GFS Physics.') From 20e2b5a102bc0e7b351be4cd29cfd917a038b146 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 16 Dec 2022 11:48:48 -0700 Subject: [PATCH 6/8] Add missing dependencies on netCDF - may not be needed with fms 2022.02+ --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 89478b0fe..4f1066eb2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,6 +223,8 @@ if(OPENMP) target_link_libraries(fv3 PUBLIC OpenMP::OpenMP_Fortran) endif() +target_link_libraries(fv3 PUBLIC NetCDF::NetCDF_C NetCDF::NetCDF_Fortran) + # Install compiled Fortran module files install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include DESTINATION ${CMAKE_INSTALL_PREFIX}) From 9794ed668995b8a09d8477cefcf4181ee566814d Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 5 Jan 2023 17:01:23 -0500 Subject: [PATCH 7/8] publicize fv3 source flag --- tools/external_ic.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/external_ic.F90 b/tools/external_ic.F90 index 6be85a45f..26ef14694 100644 --- a/tools/external_ic.F90 +++ b/tools/external_ic.F90 @@ -196,7 +196,7 @@ module external_ic_mod real, parameter:: zvir = rvgas/rdgas - 1. real(kind=R_GRID), parameter :: cnst_0p20=0.20d0 real, parameter :: deg2rad = pi/180. - logical :: data_source_fv3gfs + logical, public :: data_source_fv3gfs ! version number of this module ! Include variable "version" to be written to log file. From 55ee93da8925121d5116ebdc07ae898624eaa6ec Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 6 Jan 2023 17:17:26 -0500 Subject: [PATCH 8/8] allow for custom paths in the gridspec files --- tools/fv_grid_tools.F90 | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/fv_grid_tools.F90 b/tools/fv_grid_tools.F90 index 680d74fbf..45810af35 100644 --- a/tools/fv_grid_tools.F90 +++ b/tools/fv_grid_tools.F90 @@ -184,7 +184,7 @@ subroutine read_grid(Atm, grid_file, ndims, nregions, ng) real, allocatable, dimension(:,:) :: tmpx, tmpy real(kind=R_GRID), pointer, dimension(:,:,:) :: grid character(len=128) :: units = "" - character(len=256) :: atm_mosaic, atm_hgrid, grid_form + character(len=256) :: atm_mosaic, atm_hgrid, grid_form, gridfiles_path character(len=1024) :: attvalue integer :: ntiles, i, j, stdunit integer :: isc2, iec2, jsc2, jec2 @@ -222,11 +222,23 @@ subroutine read_grid(Atm, grid_file, ndims, nregions, ng) else atm_mosaic = trim(grid_file) endif + + ! Option to have a path that is /gridfiles_path/INPUT/CXXX_grid.tileY.nc instead of just INPUT/CXXX_grid.tileY.nc + if(variable_exists(Grid_input, 'gridfiles_path') ) then + call read_data(Grid_input, "gridfiles_path", gridfiles_path) + gridfiles_path = trim(gridfiles_path)//"/" ! Just incase it doesn't end in a slash + else + gridfiles_path = '' + endif + call close_file(Grid_input) endif call get_mosaic_tile_grid(atm_hgrid, atm_mosaic, Atm%domain) + ! Add optional path to the atm_hgrid path + atm_hgrid = trim(gridfiles_path)//trim(atm_hgrid) + grid_form = "none" if (open_file(Grid_input, atm_hgrid, "read")) then call get_global_attribute(Grid_input, "history", attvalue)