Skip to content

inline post updates for P8 and ESMF -managed threading #569

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BrianCurtis-NOAA merged 4 commits into from
Aug 9, 2022
Merged

inline post updates for P8 and ESMF -managed threading #569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0f106b9
Gerhard: Initialize wrttasks_per_group member in a way that automatic…
junwang-noaa Aug 1, 2022
6dfed4e
fix the fields on pv and sigma levels
junwang-noaa Aug 4, 2022
d07298a
update upp to fix pressure level TCDC
junwang-noaa Aug 5, 2022
4291a73
Merge remote-tracking branch 'upstream/develop' into esmfthrds
junwang-noaa Aug 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 20 additions & 8 deletions fv3_cap.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,9 @@ subroutine InitializeAdvertise(gcomp, rc)

character(len=*),parameter :: subname='(fv3_cap:InitializeAdvertise)'
real(kind=8) :: MPI_Wtime, timeis, timerhs

integer :: wrttasks_per_group_from_parent, wrtLocalPet

!
!------------------------------------------------------------------------
!
Expand Down Expand Up @@ -281,7 +284,7 @@ subroutine InitializeAdvertise(gcomp, rc)
label ='write_groups:',rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
!
call ESMF_ConfigGetAttribute(config=CF,value=wrttasks_per_group, &
call ESMF_ConfigGetAttribute(config=CF,value=wrttasks_per_group_from_parent, &
label ='write_tasks_per_group:',rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

Expand All @@ -290,7 +293,7 @@ subroutine InitializeAdvertise(gcomp, rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

if(mype == 0) print *,'af nems config,quilting=',quilting,' write_groups=', &
write_groups,wrttasks_per_group,' isrcTermProcessing=', isrcTermProcessing
write_groups,wrttasks_per_group_from_parent,' isrcTermProcessing=', isrcTermProcessing
!
call ESMF_ConfigGetAttribute(config=CF,value=num_files, &
label ='num_files:',rc=rc)
Expand Down Expand Up @@ -338,7 +341,7 @@ subroutine InitializeAdvertise(gcomp, rc)
! create fcst grid component

if( quilting ) then
num_pes_fcst = petcount - write_groups * wrttasks_per_group
num_pes_fcst = petcount - write_groups * wrttasks_per_group_from_parent
else
num_pes_fcst = petcount
endif
Expand Down Expand Up @@ -419,9 +422,9 @@ subroutine InitializeAdvertise(gcomp, rc)
allocate(wrtFB(FBCount,write_groups), routehandle(FBCount,write_groups))
allocate(srcGrid(FBCount,write_groups), dstGrid(FBCount,write_groups), gridRedistRH(FBCount,write_groups))
allocate(lead_wrttask(write_groups), last_wrttask(write_groups))
allocate(petList(wrttasks_per_group))
allocate(originPetList(num_pes_fcst+wrttasks_per_group))
allocate(targetPetList(num_pes_fcst+wrttasks_per_group))
allocate(petList(wrttasks_per_group_from_parent))
allocate(originPetList(num_pes_fcst+wrttasks_per_group_from_parent))
allocate(targetPetList(num_pes_fcst+wrttasks_per_group_from_parent))
if(mype == 0) print *,'af allco wrtComp,write_groups=',write_groups

! pull out the item names and item types from fcstState
Expand Down Expand Up @@ -457,10 +460,10 @@ subroutine InitializeAdvertise(gcomp, rc)

! prepare petList for wrtComp(i)
lead_wrttask(i) = k
do j=1, wrttasks_per_group
do j=1, wrttasks_per_group_from_parent
petList(j) = k + j-1
enddo
k = k + wrttasks_per_group
k = k + wrttasks_per_group_from_parent
last_wrttask(i) = k - 1
! if(mype==0)print *,'af wrtComp(i)=',i,'k=',k

Expand All @@ -487,6 +490,15 @@ subroutine InitializeAdvertise(gcomp, rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return

! get the actual number of PETs executing wrtComp, considering threading
call ESMF_GridCompGet(gridcomp=wrtComp(i),localPet=wrtLocalPet,rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
if (wrtLocalPet/=-1) then
! This PET does execute inside of wrtComp(i)
call ESMF_GridCompGet(gridcomp=wrtComp(i),petCount=wrttasks_per_group,rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
endif

! add configuration file
call ESMF_GridCompSet(gridcomp=wrtComp(i),config=CF,rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
Expand Down
38 changes: 19 additions & 19 deletions io/post_fv3.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ subroutine post_run_fv3(wrt_int_state,mypei,mpicomp,lead_write, &
integer,allocatable :: istagrp(:),iendgrp(:)
integer,save :: kpo,kth,kpv
logical,save :: log_postalct=.false.
real,dimension(komax),save :: po, th, pv
real(4),dimension(komax),save :: po, th, pv
logical :: Log_runpost
character(255) :: post_fname*255

Expand Down Expand Up @@ -2996,29 +2996,29 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, &

! model level cloud fraction
if(imp_physics == 11) then !GFDL MP
if(trim(fieldname)=='cld_amt') then
Comment thread
junwang-noaa marked this conversation as resolved.
!$omp parallel do default(none) private(i,j,l) shared(lm,jsta,jend,ista,iend,cfr,arrayr43d,fillvalue,spval)
do l=1,lm
do j=jsta,jend
do i=ista, iend
cfr(i,j,l) = arrayr43d(i,j,l)
if(abs(arrayr43d(i,j,l)-fillvalue)<small) cfr(i,j,l) = spval
if(trim(fieldname)=='cld_amt') then
!$omp parallel do default(none) private(i,j,l) shared(lm,jsta,jend,ista,iend,cfr,arrayr43d,fillvalue,spval)
do l=1,lm
do j=jsta,jend
do i=ista, iend
cfr(i,j,l) = arrayr43d(i,j,l)
if(abs(arrayr43d(i,j,l)-fillvalue)<small) cfr(i,j,l) = spval
enddo
enddo
enddo
enddo
endif
endif
else !Other MP
if(trim(fieldname)=='cldfra') then
!$omp parallel do default(none) private(i,j,l) shared(lm,jsta,jend,ista,iend,cfr,arrayr43d,fillvalue,spval)
do l=1,lm
do j=jsta,jend
do i=ista, iend
cfr(i,j,l) = arrayr43d(i,j,l)
if(abs(arrayr43d(i,j,l)-fillvalue)<small) cfr(i,j,l) = spval
if(trim(fieldname)=='cldfra') then
!$omp parallel do default(none) private(i,j,l) shared(lm,jsta,jend,ista,iend,cfr,arrayr43d,fillvalue,spval)
do l=1,lm
do j=jsta,jend
do i=ista, iend
cfr(i,j,l) = arrayr43d(i,j,l)
if(abs(arrayr43d(i,j,l)-fillvalue)<small) cfr(i,j,l) = spval
enddo
enddo
enddo
enddo
endif
endif
endif

!3d fields
Expand Down
2 changes: 1 addition & 1 deletion io/post_nems_routines.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ subroutine read_postnmlt(kpo,kth,kpv,po,th,pv,post_namelist)
!---
character (len=*), intent(in) :: post_namelist
integer,intent(out) :: kpo,kth,kpv
real,dimension(komax),intent(out) :: po,th,pv
real(4),dimension(komax),intent(out) :: po,th,pv
integer :: nlunit
real :: untcnvt
logical :: popascal
Expand Down
2 changes: 1 addition & 1 deletion upp
Open in desktop
Submodule upp updated 3 files
+2 −1 sorc/ncep_post.fd/DEALLOCATE.f
+31 −5 sorc/ncep_post.fd/INITPOST_NETCDF.f
+6 −4 sorc/ncep_post.fd/MDL2P.f