Skip to content

Add 2d domain decomposition to the inline post #533

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jkbk2004 merged 9 commits into from
Jun 13, 2022
Merged

Add 2d domain decomposition to the inline post #533

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e251943
20220505 Jesse Meng create branch upp_unify_inline_2d from
May 5, 2022
264a1d5
20220511 Jesse Meng Merge remote-tracking branch 'upstream/develop' i…
May 11, 2022
5444aa6
20220525 Jesse Meng update upp
May 25, 2022
fb217ce
20220603 Jesse Meng Merge remote-tracking branch 'upstream/develop' i…
Jun 3, 2022
0a1360a
20220606 Jesse Meng comment out debug lines.
Jun 6, 2022
fcfcca4
20220606 Jesse Meng Merge remote-tracking branch 'upstream/develop' i…
Jun 6, 2022
4f2e656
20220608 Jesse Meng modify .gitmodules where upp now point to NOAA-EM…
Jun 8, 2022
ad5c4e5
20220609 Jesse Meng Merge remote-tracking branch 'upstream/develop' i…
Jun 9, 2022
3538fe3
20220610 Jesse Meng update upp url
Jun 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions io/module_wrt_grid_comp.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock,
!*** INITIALIZE THE WRITE GRIDDED COMPONENT.
!-----------------------------------------------------------------------
!
use ctlblk_mod, only: numx
type(esmf_GridComp) :: wrt_comp
type(ESMF_State) :: imp_state_write, exp_state_write
type(esmf_Clock) :: clock
Expand Down Expand Up @@ -379,6 +380,8 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock,
itasks = 1
jtasks = ntasks
endif
numx = itasks
if (lprnt) print *,'jtasks=',jtasks,' itasks=',itasks,' numx=',numx

if (trim(output_grid(n)) == 'gaussian_grid' .or. trim(output_grid(n)) == 'global_latlon') then
call ESMF_ConfigGetAttribute(config=cf_output_grid, value=imo(n), label ='imo:',rc=rc)
Expand Down
111 changes: 59 additions & 52 deletions io/post_fv3.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,15 @@ subroutine post_run_fv3(wrt_int_state,mypei,mpicomp,lead_write, &
! and HAFS.
! 6)read 3D cloud fraction from cld_amt for GFDL MP,
! and from cldfra for other MPs.
! Jun 2022 J. Meng 2D decomposition
!
!-----------------------------------------------------------------------
!*** run post on write grid comp
!-----------------------------------------------------------------------
!
use ctlblk_mod, only : komax,ifhr,ifmin,modelname,datapd,fld_info, &
npset,grib,gocart_on,icount_calmict, jsta, &
jend,im, nsoil, filenameflat
jend,ista,iend, im, nsoil, filenameflat,numx
use gridspec_mod, only : maptype, gridtype,latstart,latlast, &
lonstart,lonlast
use grib2_module, only : gribit2,num_pset,nrecout,first_grbtbl
Expand All @@ -69,6 +70,8 @@ subroutine post_run_fv3(wrt_int_state,mypei,mpicomp,lead_write, &
!
integer n,nwtpg,ieof,lcntrl,ierr,i,j,k,jts,jte,mynsoil
integer,allocatable :: jstagrp(:),jendgrp(:)
integer its,ite
integer,allocatable :: istagrp(:),iendgrp(:)
integer,save :: kpo,kth,kpv
logical,save :: log_postalct=.false.
real,dimension(komax),save :: po, th, pv
Expand All @@ -93,6 +96,8 @@ subroutine post_run_fv3(wrt_int_state,mypei,mpicomp,lead_write, &
nwtpg = wrt_int_state%petcount
jts = wrt_int_state%lat_start !<-- Starting J of this write task's subsection
jte = wrt_int_state%lat_end !<-- Ending J of this write task's subsection
its = wrt_int_state%lon_start !<-- Starting I of this write task's subsection
ite = wrt_int_state%lon_end !<-- Ending I of this write task's subsection
maptype = wrt_int_state%post_maptype
nbdl = wrt_int_state%FBCount

Expand All @@ -107,12 +112,16 @@ subroutine post_run_fv3(wrt_int_state,mypei,mpicomp,lead_write, &
if (.not.log_postalct) then
!
allocate(jstagrp(nwtpg),jendgrp(nwtpg))
allocate(istagrp(nwtpg),iendgrp(nwtpg))
!
do n=0,nwtpg-1
jstagrp(n+1) = wrt_int_state%lat_start_wrtgrp(n+1)
jendgrp(n+1) = wrt_int_state%lat_end_wrtgrp (n+1)
istagrp(n+1) = wrt_int_state%lon_start_wrtgrp(n+1)
iendgrp(n+1) = wrt_int_state%lon_end_wrtgrp (n+1)
enddo
if(mype==0) print *,'in post_run,jstagrp=',jstagrp,'jendgrp=',jendgrp
if(mype==0) print *,'in post_run,istagrp=',istagrp,'iendgrp=',iendgrp

!-----------------------------------------------------------------------
!*** read namelist for pv,th,po
Expand All @@ -131,7 +140,7 @@ subroutine post_run_fv3(wrt_int_state,mypei,mpicomp,lead_write, &
'jstagrp=',jstagrp,'jendgrp=',jendgrp
call post_alctvars(wrt_int_state%im,wrt_int_state%jm, &
wrt_int_state%lm,mype,wrttasks_per_group,lead_write, &
mpicomp,jts,jte,jstagrp,jendgrp)
mpicomp,jts,jte,jstagrp,jendgrp,its,ite,istagrp,iendgrp)
!
!-----------------------------------------------------------------------
!*** read namelist for pv,th,po
Expand Down Expand Up @@ -200,11 +209,11 @@ subroutine post_run_fv3(wrt_int_state,mypei,mpicomp,lead_write, &
npset = npset + 1
call set_outflds(kth,th,kpv,pv)
if(allocated(datapd))deallocate(datapd)
allocate(datapd(wrt_int_state%im,jte-jts+1,nrecout+100))
!$omp parallel do default(none),private(i,j,k),shared(nrecout,jend,jsta,im,datapd)
allocate(datapd(ite-its+1,jte-jts+1,nrecout+100))
!$omp parallel do default(none),private(i,j,k),shared(nrecout,jend,jsta,im,datapd,ista,iend)
do k=1,nrecout+100
do j=1,jend+1-jsta
do i=1,im
do i=1,iend+1-ista
datapd(i,j,k) = 0.
enddo
enddo
Expand Down Expand Up @@ -511,6 +520,7 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
use soil, only: sldpth, sh2o, smc, stc
use masks, only: lmv, lmh, htm, vtm, gdlat, gdlon, dx, dy, hbm2, sm, sice
use ctlblk_mod, only: im, jm, lm, lp1, jsta, jend, jsta_2l, jend_2u, jsta_m,jend_m, &
ista, iend, ista_2l, iend_2u, ista_m,iend_m, &
lsm, pt, imp_physics, spval, mpi_comm_comp, gdsdegr, &
tprec, tclod, trdlw, trdsw, tsrfc, tmaxmin, theat, &
ardlw, ardsw, asrfc, avrain, avcnvc, iSF_SURFACE_PHYSICS,&
Expand Down Expand Up @@ -546,7 +556,7 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
!
integer i, ip1, j, l, k, n, iret, ibdl, rc, kstart, kend
integer i1,i2,j1,j2,k1,k2
integer ista,iend,fieldDimCount,gridDimCount,ncount_field
integer fieldDimCount,gridDimCount,ncount_field
integer jdate(8)
logical foundland, foundice, found, mvispresent
integer totalLBound3d(3), totalUBound3d(3)
Expand Down Expand Up @@ -594,18 +604,19 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
!
!$omp parallel do default(shared),private(i,j)
do j=jsta,jend
do i=1,im
do i=ista,iend
gdlat(i,j) = wrt_int_state%latPtr(i,j)
gdlon(i,j) = wrt_int_state%lonPtr(i,j)
enddo
enddo

call exch(gdlat(1,jsta_2l))
call exch(gdlat)
call exch(gdlon)

!$omp parallel do default(none),private(i,j,ip1), &
!$omp& shared(jsta,jend_m,im,dx,gdlat,gdlon,dy)
!$omp& shared(jsta,jend_m,im,dx,gdlat,gdlon,dy,ista,iend_m)
do j = jsta, jend_m
do i = 1, im-1
do i = ista, iend_m
ip1 = i + 1
!if (ip1 > im) ip1 = ip1 - im
dx(i,j) = erad*cos(gdlat(i,j)*dtr)*(gdlon(ip1,j)-gdlon(i,j))*dtr
Expand All @@ -619,9 +630,9 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
bk5(i) = wrt_int_state%bk(i)
enddo

!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,f,gdlat)
!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,f,gdlat,ista,iend)
do j=jsta,jend
do i=1,im
do i=ista,iend
f(I,J) = 1.454441e-4*sin(gdlat(i,j)*dtr) ! 2*omeg*sin(phi)
end do
end do
Expand All @@ -631,9 +642,9 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
! GFS does not have surface specific humidity
! inst sensible heat flux
! inst latent heat flux
!$omp parallel do default(none),private(i,j),shared(jsta,jend,im,spval,qs,twbs,qwbs,ths)
!$omp parallel do default(none),private(i,j),shared(jsta,jend,im,spval,qs,twbs,qwbs,ths,ista,iend)
do j=jsta,jend
do i=1,im
do i=ista,iend
qs(i,j) = SPVAL
twbs(i,j) = SPVAL
qwbs(i,j) = SPVAL
Expand All @@ -650,10 +661,10 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
! 10 m theta
! 10 m humidity
! snow free albedo
!$omp parallel do default(none), private(i,j), shared(jsta,jend,im,spval), &
!$omp parallel do default(none), private(i,j), shared(jsta,jend,im,spval,ista,iend), &
!$omp& shared(cldefi,lspa,th10,q10,albase)
do j=jsta,jend
do i=1,im
do i=ista,iend
cldefi(i,j) = SPVAL
lspa(i,j) = SPVAL
th10(i,j) = SPVAL
Expand All @@ -663,9 +674,9 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
enddo

! GFS does not have convective precip
!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,cprate)
!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,cprate,ista,iend)
do j=jsta,jend
do i=1,im
do i=ista,iend
cprate(i,j) = 0.
enddo
enddo
Expand All @@ -675,10 +686,10 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
! inst cloud fraction for high, middle, and low cloud,
! cfrach
! inst ground heat flux, grnflx
!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,spval), &
!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,spval,ista,iend), &
!$omp& shared(czen,czmean,radot,cfrach,cfracl,cfracm,grnflx)
do j=jsta,jend
do i=1,im
do i=ista,iend
czen(i,j) = SPVAL
czmean(i,j) = SPVAL
cfrach(i,j) = SPVAL
Expand All @@ -705,10 +716,10 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
! inst outgoing sfc shortwave, rswout
! snow phase change heat flux, snopcx
! GFS does not use total momentum flux,sfcuvx
!$omp parallel do default(none),private(i,j),shared(jsta,jend,im,spval), &
!$omp parallel do default(none),private(i,j),shared(jsta,jend,im,spval,ista,iend), &
!$omp& shared(acfrcv,ncfrcv,acfrst,ncfrst,bgroff,rlwtoa,rswin,rswinc,rswout,snopcx,sfcuvx)
do j=jsta,jend
do i=1,im
do i=ista,iend
acfrcv(i,j) = spval
ncfrcv(i,j) = 1.0
acfrst(i,j) = spval
Expand All @@ -731,10 +742,10 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
! temperature tendency due to latent heating from convection
! temperature tendency due to latent heating from grid scale
do l=1,lm
!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,spval,l), &
!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,spval,l,ista_2l,iend_2u), &
!$omp& shared(rlwtt,rswtt,tcucn,tcucns,train)
do j=jsta_2l,jend_2u
do i=1,im
do i=ista_2l,iend_2u
rlwtt(i,j,l) = spval
rswtt(i,j,l) = spval
tcucn(i,j,l) = spval
Expand All @@ -761,10 +772,10 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
! v at roughness length, vz0
! shelter rh max, maxrhshltr
! shelter rh min, minrhshltr
!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,spval), &
!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,spval,ista_2l,iend_2u), &
!$omp& shared(smstav,sfcevp,acsnow,acsnom,qz0,uz0,vz0,maxrhshltr,minrhshltr)
do j=jsta_2l,jend_2u
do i=1,im
do i=ista_2l,iend_2u
smstav(i,j) = spval
sfcevp(i,j) = spval
acsnow(i,j) = spval
Expand All @@ -778,20 +789,20 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
! GFS does not have mixing length,el_pbl
! exchange coefficient, exch_h
do l=1,lm
!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,l,spval,el_pbl,exch_h)
!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,l,spval,el_pbl,exch_h,ista_2l,iend_2u)
do j=jsta_2l,jend_2u
do i=1,im
do i=ista_2l,iend_2u
el_pbl(i,j,l) = spval
exch_h(i,j,l) = spval
enddo
enddo
enddo

! GFS does not have deep convective cloud top and bottom fields
!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,spval), &
!$omp parallel do default(none),private(i,j),shared(jsta_2l,jend_2u,im,spval,ista_2l,iend_2u), &
!$omp& shared(htopd,hbotd,htops,hbots,cuppt)
do j=jsta_2l,jend_2u
do i=1,im
do i=ista_2l,iend_2u
htopd(i,j) = SPVAL
hbotd(i,j) = SPVAL
htops(i,j) = SPVAL
Expand Down Expand Up @@ -852,8 +863,6 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
line=__LINE__, file=__FILE__)) return ! bail out
! print *,'in post_lam, get land field value,fillvalue=',fillvalue

ista = lbound(arrayr42d,1)
iend = ubound(arrayr42d,1)
!$omp parallel do default(none),private(i,j),shared(jsta,jend,ista,iend,spval,arrayr42d,sm,fillValue)
do j=jsta, jend
do i=ista, iend
Expand Down Expand Up @@ -886,8 +895,6 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
line=__LINE__, file=__FILE__)) return ! bail out
! if(mype==0) print *,'in post_lam, get icec field value,fillvalue=',fillvalue

ista = lbound(arrayr42d,1)
iend = ubound(arrayr42d,1)
!$omp parallel do default(none) private(i,j) shared(jsta,jend,ista,iend,spval,sice,arrayr42d,sm,fillValue)
do j=jsta, jend
do i=ista, iend
Expand Down Expand Up @@ -3029,9 +3036,9 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
enddo file_loop_all

! recompute full layer of zint
!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,lp1,spval,zint,fis)
!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,lp1,spval,zint,fis,ista,iend)
do j=jsta,jend
do i=1,im
do i=ista,iend
if (fis(i,j) /= spval) then
zint(i,j,lp1) = fis(i,j)
fis(i,j) = fis(i,j) * grav
Expand All @@ -3043,9 +3050,9 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
enddo

do l=lm,1,-1
!$omp parallel do default(none) private(i,j) shared(l,jsta,jend,im,omga,wh,dpres,zint,spval)
!$omp parallel do default(none) private(i,j) shared(l,jsta,jend,im,omga,wh,dpres,zint,spval,ista,iend)
do j=jsta,jend
do i=1,im
do i=ista,iend
if(wh(i,j,l) /= spval) then
omga(i,j,l) = (-1.) * wh(i,j,l) * dpres(i,j,l)/zint(i,j,l)
zint(i,j,l) = zint(i,j,l) + zint(i,j,l+1)
Expand All @@ -3060,17 +3067,17 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
! 'lm=',maxval(omga(ista:iend,jsta:jend,lm)),minval(omga(ista:iend,jsta:jend,lm))

! compute pint from top down
!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,ak5,pint)
!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,ak5,pint,ista,iend)
do j=jsta,jend
do i=1,im
do i=ista,iend
pint(i,j,1) = ak5(1)
end do
end do

do l=2,lp1
!$omp parallel do default(none) private(i,j) shared(l,jsta,jend,im,pint,dpres,spval)
!$omp parallel do default(none) private(i,j) shared(l,jsta,jend,im,pint,dpres,spval,ista,iend)
do j=jsta,jend
do i=1,im
do i=ista,iend
if(dpres(i,j,l-1) /= spval) then
pint(i,j,l) = pint(i,j,l-1) + dpres(i,j,l-1)
else
Expand All @@ -3082,9 +3089,9 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &

!compute pmid from averaged two layer pint
do l=lm,1,-1
!$omp parallel do default(none) private(i,j) shared(l,jsta,jend,im,pmid,pint,spval)
!$omp parallel do default(none) private(i,j) shared(l,jsta,jend,im,pmid,pint,spval,ista,iend)
do j=jsta,jend
do i=1,im
do i=ista,iend
if(pint(i,j,l+1) /= spval) then
pmid(i,j,l) = 0.5*(pint(i,j,l)+pint(i,j,l+1))
else
Expand All @@ -3094,9 +3101,9 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &
enddo
enddo

!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,spval,pt,pd,pint)
!$omp parallel do default(none) private(i,j) shared(jsta,jend,im,spval,pt,pd,pint,ista,iend)
do j=jsta,jend
do i=1,im
do i=ista,iend
pd(i,j) = spval
pint(i,j,1) = pt
end do
Expand All @@ -3105,9 +3112,9 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &

! compute alpint
do l=lp1,1,-1
!$omp parallel do default(none) private(i,j) shared(l,jsta,jend,im,alpint,pint,spval)
!$omp parallel do default(none) private(i,j) shared(l,jsta,jend,im,alpint,pint,spval,ista,iend)
do j=jsta,jend
do i=1,im
do i=ista,iend
if(pint(i,j,l) /= spval) then
alpint(i,j,l) = log(pint(i,j,l))
else
Expand All @@ -3119,9 +3126,9 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &

! compute zmid
do l=lm,1,-1
!$omp parallel do default(none) private(i,j) shared(l,jsta,jend,im,zmid,zint,pmid,alpint,spval)
!$omp parallel do default(none) private(i,j) shared(l,jsta,jend,im,zmid,zint,pmid,alpint,spval,ista,iend)
do j=jsta,jend
do i=1,im
do i=ista,iend
if( zint(i,j,l+1)/=spval .and. zint(i,j,l)/=spval .and. pmid(i,j,l) /= spval) then
zmid(i,j,l)=zint(i,j,l+1)+(zint(i,j,l)-zint(i,j,l+1))* &
(log(pmid(i,j,l))-alpint(i,j,l+1))/ &
Expand Down Expand Up @@ -3199,7 +3206,7 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &

!htop
do j=jsta,jend
do i=1,im
do i=ista,iend
htop(i,j) = spval
if(ptop(i,j) < spval)then
do l=1,lm
Expand All @@ -3214,7 +3221,7 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &

! hbot
do j=jsta,jend
do i=1,im
do i=ista,iend
hbot(i,j) = spval
if(pbot(i,j) < spval)then
do l=lm,1,-1
Expand Down
Loading