Skip to content

Build dev/emc branch with no physics #231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
climbfuji wants to merge 9 commits into from
Closed

Build dev/emc branch with no physics #231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
74f3569
Add cmake option JEDI = no physics whatsoever
climbfuji Dec 6, 2022
bd5305c
Add necessary ifdefs for cmake option JEDI
climbfuji Dec 6, 2022
352e8ee
Bug fix in model/fv_dynamics.F90
climbfuji Dec 6, 2022
d8ec846
Bug fix in model/fv_mapz.F90
climbfuji Dec 6, 2022
dbce126
Rename JEDI cmake option to NO_PHYS, and make it work without GFS_PHYS
climbfuji Dec 6, 2022
20e2b5a
Add missing dependencies on netCDF - may not be needed with fms 2022.02+
climbfuji Dec 16, 2022
9794ed6
publicize fv3 source flag
danholdaway Jan 5, 2023
55ee93d
allow for custom paths in the gridspec files
danholdaway Jan 6, 2023
fcbb20d
Merge branch 'dev/emc' of https://github.com/noaa-gfdl/gfdl_atmos_cub…
climbfuji Jan 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ option(DEBUG "Enable compiler definition -DDEBUG" OFF)
option(MOVING_NEST "Enable compiler definition -DMOVING_NEST" OFF)
option(MULTI_GASES "Enable compiler definition -DMULTI_GASES" OFF)
option(USE_GFSL63 "Enable compiler definition -DUSE_GFSL63" OFF)
option(NO_PHYS "Enable compiler definition -DNO_PHYS" OFF)
option(GFS_PHYS "Enable compiler definition -DGFS_PHYS" OFF)
option(GFS_TYPES "Enable compiler definition -DGFS_TYPES" OFF)
option(use_WRTCOMP "Enable compiler definition -Duse_WRTCOMP" OFF)
Expand Down Expand Up @@ -127,7 +128,14 @@ if(USE_GFSL63)
list(APPEND fv3_defs USE_GFSL63)
endif()

if(NO_PHYS)
list(APPEND fv3_defs NO_PHYS)
endif()

if(GFS_PHYS)
if(NO_PHYS)
message(FATAL_ERROR "Turning on GFS_PHYS and NO_PHYS at the same time doesn't make sense")
endif()
list(APPEND fv3_defs GFS_PHYS)
endif()

Expand All @@ -146,6 +154,9 @@ if(MOVING_NEST)
endif()

if(MULTI_GASES)
if(NO_PHYS)
message(FATAL_ERROR "Turning on MULTI_GASES and NO_PHYS at the same time doesn't make sense")
endif()
list(APPEND fv3_defs MULTI_GASES)
endif()

Expand Down Expand Up @@ -202,6 +213,8 @@ if(OPENMP)
target_link_libraries(fv3 PUBLIC OpenMP::OpenMP_Fortran)
endif()

target_link_libraries(fv3 PUBLIC NetCDF::NetCDF_C NetCDF::NetCDF_Fortran)

# Install compiled Fortran module files
install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include DESTINATION ${CMAKE_INSTALL_PREFIX})

Expand Down
60 changes: 57 additions & 3 deletions model/fv_dynamics.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,11 +194,13 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
parent_grid, domain, diss_est, inline_mp)

use mpp_mod, only: FATAL, mpp_error
#ifndef NO_PHYS
use ccpp_static_api, only: ccpp_physics_timestep_init, &
ccpp_physics_timestep_finalize
use CCPP_data, only: ccpp_suite
use CCPP_data, only: cdata => cdata_tile
use CCPP_data, only: GFDL_interstitial
#endif

use molecular_diffusion_mod, only: md_time, md_wait_sec, md_tadj_layers, &
thermosphere_adjustment
Expand Down Expand Up @@ -275,11 +277,17 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,

! Local Arrays
real:: ws(bd%is:bd%ie,bd%js:bd%je)
#ifdef NO_PHYS
real:: te_2d(bd%is:bd%ie,bd%js:bd%je)
#endif
real:: teq(bd%is:bd%ie,bd%js:bd%je)
real:: ps2(bd%isd:bd%ied,bd%jsd:bd%jed)
real:: m_fac(bd%is:bd%ie,bd%js:bd%je)
real:: pfull(npz)
real, dimension(bd%is:bd%ie):: cvm
#ifdef NO_PHYS
real, allocatable :: dp1(:,:,:), dtdt_m(:,:,:), cappa(:,:,:)
#endif
#ifdef MULTI_GASES
real, allocatable :: kapad(:,:,:)
#endif
Expand All @@ -294,6 +302,9 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
integer :: rainwat = -999, snowwat = -999, graupel = -999, hailwat = -999, cld_amt = -999
integer :: theta_d = -999
logical used, do_omega
#ifdef NO_PHYS
logical :: last_step
#endif
integer, parameter :: max_packs=13
type(group_halo_update_type), save :: i_pack(max_packs)
integer :: is, ie, js, je
Expand All @@ -303,11 +314,13 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
real :: time_total
integer :: seconds, days

#ifndef NO_PHYS
ccpp_associate: associate( cappa => GFDL_interstitial%cappa, &
dp1 => GFDL_interstitial%te0, &
dtdt_m => GFDL_interstitial%dtdt, &
last_step => GFDL_interstitial%last_step, &
te_2d => GFDL_interstitial%te0_2d )
#endif

is = bd%is
ie = bd%ie
Expand All @@ -328,7 +341,17 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
nq = nq_tot - flagstruct%dnats
nr = nq_tot - flagstruct%dnrts
rdg = -rdgas * agrav
#ifdef NO_PHYS
allocate ( dp1(isd:ied, jsd:jed, 1:npz) )

#ifdef MOIST_CAPPA
allocate ( cappa(isd:ied,jsd:jed,npz) )
call init_ijk_mem(isd,ied, jsd,jed, npz, cappa, 0.)
#else
allocate ( cappa(isd:isd,jsd:jsd,1) )
cappa = 0.
#endif
#else
! Call CCPP timestep init
call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), group_name="fast_physics", ierr=ierr)
! Reset all interstitial variables for CCPP version
Expand All @@ -337,6 +360,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
if (flagstruct%do_sat_adj) then
GFDL_interstitial%out_dt = (idiag%id_mdt > 0)
end if
#endif

#ifdef MULTI_GASES
allocate ( kapad(isd:ied, jsd:jed, npz) )
Expand Down Expand Up @@ -428,12 +452,17 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
enddo

if ( hydrostatic ) then
#ifdef NO_PHYS
!$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,dp1,zvir,nwat,q,q_con,sphum,liq_wat, &
!$OMP rainwat,ice_wat,snowwat,graupel,hailwat) private(cvm,i,j,k)
#else
#ifdef __GFORTRAN__
!$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,zvir,nwat,q,q_con,sphum,liq_wat, &
#else
!$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,dp1,zvir,nwat,q,q_con,sphum,liq_wat, &
#endif
!$OMP rainwat,ice_wat,snowwat,graupel,hailwat) private(cvm,i,j,k)
#endif
do k=1,npz
do j=js,je
#ifdef USE_COND
Expand All @@ -446,6 +475,12 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
enddo
enddo
else
#ifdef NO_PHYS
!$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,dp1,zvir,q,q_con,sphum,liq_wat, &
!$OMP rainwat,ice_wat,snowwat,graupel,hailwat,pkz,flagstruct, &
!$OMP cappa,kappa,rdg,delp,pt,delz,nwat) &
!$OMP private(cvm)
#else
#ifdef __GFORTRAN__
!$OMP parallel do default(none) shared(is,ie,js,je,isd,ied,jsd,jed,npz,zvir,q,q_con,sphum,liq_wat, &
#else
Expand All @@ -461,6 +496,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
!$OMP cappa,kappa,rdg,delp,pt,delz,nwat) &
#endif
!$OMP private(cvm,i,j,k)
#endif
do k=1,npz
if ( flagstruct%moist_phys ) then
do j=js,je
Expand Down Expand Up @@ -587,7 +623,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
pt_initialized = .true.
endif
else
#ifdef __GFORTRAN__
#if defined(__GFORTRAN__) && ! defined(NO_PHYS)
!$OMP parallel do default(none) shared(is,ie,js,je,npz,pt,pkz,q_con)
#else
!$OMP parallel do default(none) shared(is,ie,js,je,npz,pt,dp1,pkz,q_con)
Expand Down Expand Up @@ -615,10 +651,15 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
mdt = bdt / real(k_split)

if ( idiag%id_mdt > 0 .and. (.not. do_adiabatic_init) ) then
#ifdef NO_PHYS
allocate ( dtdt_m(is:ie,js:je,npz) )
!$OMP parallel do default(none) shared(is,ie,js,je,npz,dtdt_m)
#else
#ifdef __GFORTRAN__
!$OMP parallel do default(none) shared(is,ie,js,je,npz)
#else
!$OMP parallel do default(none) shared(is,ie,js,je,npz,dtdt_m)
#endif
#endif
do k=1,npz
do j=js,je
Expand Down Expand Up @@ -649,7 +690,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
call start_group_halo_update(i_pack(8), u, v, domain, gridtype=DGRID_NE)
#endif
call timing_off('COMM_TOTAL')
#ifdef __GFORTRAN__
#if defined(__GFORTRAN__) && ! defined(NO_PHYS)
!$OMP parallel do default(none) shared(isd,ied,jsd,jed,npz,delp)
#else
!$OMP parallel do default(none) shared(isd,ied,jsd,jed,npz,dp1,delp)
Expand Down Expand Up @@ -849,11 +890,15 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
endif

if ( idiag%id_mdt > 0 .and. (.not.do_adiabatic_init) ) then
#ifdef NO_PHYS
!$OMP parallel do default(none) shared(is,ie,js,je,npz,dtdt_m,bdt)
#else
! Output temperature tendency due to inline moist physics:
#ifdef __GFORTRAN__
!$OMP parallel do default(none) shared(is,ie,js,je,npz,bdt)
#else
!$OMP parallel do default(none) shared(is,ie,js,je,npz,dtdt_m,bdt)
#endif
#endif
do k=1,npz
do j=js,je
Expand All @@ -864,6 +909,9 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
enddo
! call prt_mxm('Fast DTDT (deg/Day)', dtdt_m, is, ie, js, je, 0, npz, 1., gridstruct%area_64, domain)
used = send_data(idiag%id_mdt, dtdt_m, fv_time)
#ifdef NO_PHYS
deallocate ( dtdt_m )
#endif
endif

if( nwat==7 ) then
Expand Down Expand Up @@ -984,7 +1032,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
endif

if( (flagstruct%consv_am.or.idiag%id_amdt>0) .and. (.not.do_adiabatic_init) ) then
#ifdef __GFORTRAN__
#if defined(__GFORTRAN__) && ! defined(NO_PHYS)
!$OMP parallel do default(none) shared(is,ie,js,je,teq,dt2,ps2,ps,idiag)
#else
!$OMP parallel do default(none) shared(is,ie,js,je,te_2d,teq,dt2,ps2,ps,idiag)
Expand Down Expand Up @@ -1032,6 +1080,10 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
911 call cubed_to_latlon(u, v, ua, va, gridstruct, &
npx, npy, npz, 1, gridstruct%grid_type, domain, gridstruct%bounded_domain, flagstruct%c2l_ord, bd)

#ifdef NO_PHYS
deallocate(dp1)
deallocate(cappa)
#endif
#ifdef MULTI_GASES
deallocate(kapad)
#endif
Expand Down Expand Up @@ -1066,10 +1118,12 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill,
endif
endif

#ifndef NO_PHYS
! Call CCPP timestep finalize
call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), group_name="fast_physics", ierr=ierr)

end associate ccpp_associate
#endif

end subroutine fv_dynamics

Expand Down
5 changes: 5 additions & 0 deletions model/fv_fill.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,12 @@ module fv_fill_mod
! </table>

use mpp_domains_mod, only: mpp_update_domains, domain2D
#ifdef GFS_TYPES
use GFS_typedefs, only: kind_phys
#else
use platform_mod, only: kind_phys => r8_kind
#endif
!

implicit none
public fillz
Expand Down
41 changes: 31 additions & 10 deletions model/fv_mapz.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,11 +95,13 @@ module fv_mapz_mod
use fv_arrays_mod, only: fv_grid_type, fv_grid_bounds_type, R_GRID, inline_mp_type
use fv_timing_mod, only: timing_on, timing_off
use fv_mp_mod, only: is_master, mp_reduce_min, mp_reduce_max
#ifndef NO_PHYS
! CCPP fast physics
use ccpp_static_api, only: ccpp_physics_run
use CCPP_data, only: ccpp_suite
use CCPP_data, only: cdata => cdata_tile
use CCPP_data, only: GFDL_interstitial
#endif
#ifdef MULTI_GASES
use multi_gases_mod, only: virq, virqd, vicpqd, vicvqd, num_gas
#endif
Expand Down Expand Up @@ -223,14 +225,18 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, &
real, dimension(is:ie+1,km+1):: pe0, pe3
real, dimension(is:ie):: gsize, gz, cvm, qv

#ifdef NO_PHYS
logical :: fast_mp_consv
#endif
real rcp, rg, rrg, bkh, dtmp, k1k
integer:: i,j,k
integer:: kdelz
integer:: nt, liq_wat, ice_wat, rainwat, snowwat, cld_amt, graupel, hailwat, ccn_cm3, iq, n, kmp, kp, k_next
integer:: nt, liq_wat, ice_wat, rainwat, snowwat, cld_amt, graupel, hailwat, ccn_cm3, iq, n, kp, k_next
integer :: ierr

ccpp_associate: associate( fast_mp_consv => GFDL_interstitial%fast_mp_consv, &
kmp => GFDL_interstitial%kmp )
#ifndef NO_PHYS
ccpp_associate: associate( fast_mp_consv => GFDL_interstitial%fast_mp_consv )
#endif

k1k = rdgas/cv_air ! akap / (1.-akap) = rg/Cv=0.4
rg = rdgas
Expand Down Expand Up @@ -665,31 +671,40 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, &
1000 continue

#ifdef __GFORTRAN__
!$OMP parallel default(none) shared(is,ie,js,je,km,ptop,u,v,pe,ua,va,isd,ied,jsd,jed,kord_mt, &
!$OMP parallel default(none) shared(is,ie,js,je,km,ptop,u,v,pe,ua,va,isd,ied,jsd,jed,kord_mt, &
!$OMP te_2d,te,delp,hydrostatic,hs,rg,pt,peln, adiabatic, &
!$OMP cp,delz,nwat,rainwat,liq_wat,ice_wat,snowwat, &
!$OMP graupel,hailwat,q_con,r_vir,sphum,w,pk,pkz,last_step,consv, &
!$OMP graupel,hailwat,q_con,r_vir,sphum,w,pk,pkz,last_step,consv,&
!$OMP do_adiabatic_init,zsum1,zsum0,te0_2d,domain, &
!$OMP ng,gridstruct,E_Flux,pdt,dtmp,reproduce_sum,q, &
!$OMP mdt,cld_amt,cappa,dtdt,out_dt,rrg,akap,do_sat_adj, &
!$OMP kord_tm,pe4, npx,npy,ccn_cm3,u_dt,v_dt, c2l_ord,bd,dp0,ps, &
!$OMP cdata,GFDL_interstitial) &
!$OMP kord_tm,pe4, npx,npy,ccn_cm3,u_dt,v_dt, c2l_ord,bd,dp0,ps &
#ifdef NO_PHYS
!$OMP ) &
#else
!$OMP ,cdata,GFDL_interstitial) &
!$OMP shared(ccpp_suite) &
#endif
#ifdef MULTI_GASES
!$OMP shared(num_gas) &
#endif
!$OMP private(q2,pe0,pe1,pe2,pe3,qv,cvm,gz,gsize,phis,kdelz,dp2,t0, ierr)
#else
!$OMP parallel default(none) shared(is,ie,js,je,km,kmp,ptop,u,v,pe,ua,va,isd,ied,jsd,jed,kord_mt, &
!$OMP parallel default(none) shared(is,ie,js,je,km,ptop,u,v,pe,ua,va,isd,ied,jsd,jed,kord_mt, &
!$OMP te_2d,te,delp,hydrostatic,hs,rg,pt,peln, adiabatic, &
!$OMP cp,delz,nwat,rainwat,liq_wat,ice_wat,snowwat, &
!$OMP graupel,hailwat,q_con,r_vir,sphum,w,pk,pkz,last_step,consv, &
!$OMP graupel,hailwat,q_con,r_vir,sphum,w,pk,pkz,last_step,consv,&
!$OMP do_adiabatic_init,zsum1,zsum0,te0_2d,domain, &
!$OMP ng,gridstruct,E_Flux,pdt,dtmp,reproduce_sum,q, &
!$OMP mdt,cld_amt,cappa,dtdt,out_dt,rrg,akap,do_sat_adj, &
!$OMP fast_mp_consv,kord_tm, pe4,npx,npy, ccn_cm3, &
!$OMP u_dt,v_dt,c2l_ord,bd,dp0,ps,cdata,GFDL_interstitial) &
!$OMP u_dt,v_dt,c2l_ord,bd,dp0,ps &
#ifdef NO_PHYS
!$OMP ) &
#else
!$OMP ,cdata,GFDL_interstitial) &
!$OMP shared(ccpp_suite) &
#endif
#ifdef MULTI_GASES
!$OMP shared(num_gas) &
#endif
Expand Down Expand Up @@ -837,6 +852,7 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, &
! if ( (.not.do_adiabatic_init) .and. do_sat_adj ) then

if ( do_sat_adj ) then
#ifndef NO_PHYS
call timing_on('sat_adj2')
! Call to CCPP fast_physics group
if (cdata%initialized()) then
Expand All @@ -849,6 +865,9 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, &
call mpp_error (FATAL, 'Lagrangian_to_Eulerian: can not call CCPP fast physics because CCPP not initialized')
endif
call timing_off('sat_adj2')
#else
call mpp_error (FATAL, 'Lagrangian_to_Eulerian: calling saturation adjustment for no-physics configuration makes no sense')
#endif
endif ! do_sat_adj

if ( last_step ) then
Expand Down Expand Up @@ -924,7 +943,9 @@ subroutine Lagrangian_to_Eulerian(last_step, consv, ps, pe, delp, pkz, pk, &
endif
!$OMP end parallel

#ifndef NO_PHYS
end associate ccpp_associate
#endif

end subroutine Lagrangian_to_Eulerian

Expand Down
Loading