From 0f106b9b5a0663d026f3dbd819542d43fd458c35 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Mon, 1 Aug 2022 20:07:35 +0000 Subject: [PATCH 1/3] Gerhard: Initialize wrttasks_per_group member in a way that automatically considers ESMF-managed threading when active. --- fv3_cap.F90 | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 0c72a2cd1..3f09cbf3b 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -203,6 +203,9 @@ subroutine InitializeAdvertise(gcomp, rc) character(len=*),parameter :: subname='(fv3_cap:InitializeAdvertise)' real(kind=8) :: MPI_Wtime, timeis, timerhs + + integer :: wrttasks_per_group_from_parent, wrtLocalPet + ! !------------------------------------------------------------------------ ! @@ -281,7 +284,7 @@ subroutine InitializeAdvertise(gcomp, rc) label ='write_groups:',rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! - call ESMF_ConfigGetAttribute(config=CF,value=wrttasks_per_group, & + call ESMF_ConfigGetAttribute(config=CF,value=wrttasks_per_group_from_parent, & label ='write_tasks_per_group:',rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -290,7 +293,7 @@ subroutine InitializeAdvertise(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if(mype == 0) print *,'af nems config,quilting=',quilting,' write_groups=', & - write_groups,wrttasks_per_group,' isrcTermProcessing=', isrcTermProcessing + write_groups,wrttasks_per_group_from_parent,' isrcTermProcessing=', isrcTermProcessing ! call ESMF_ConfigGetAttribute(config=CF,value=num_files, & label ='num_files:',rc=rc) @@ -338,7 +341,7 @@ subroutine InitializeAdvertise(gcomp, rc) ! create fcst grid component if( quilting ) then - num_pes_fcst = petcount - write_groups * wrttasks_per_group + num_pes_fcst = petcount - write_groups * wrttasks_per_group_from_parent else num_pes_fcst = petcount endif @@ -419,9 +422,9 @@ subroutine InitializeAdvertise(gcomp, rc) allocate(wrtFB(FBCount,write_groups), routehandle(FBCount,write_groups)) allocate(srcGrid(FBCount,write_groups), dstGrid(FBCount,write_groups), gridRedistRH(FBCount,write_groups)) allocate(lead_wrttask(write_groups), last_wrttask(write_groups)) - allocate(petList(wrttasks_per_group)) - allocate(originPetList(num_pes_fcst+wrttasks_per_group)) - allocate(targetPetList(num_pes_fcst+wrttasks_per_group)) + allocate(petList(wrttasks_per_group_from_parent)) + allocate(originPetList(num_pes_fcst+wrttasks_per_group_from_parent)) + allocate(targetPetList(num_pes_fcst+wrttasks_per_group_from_parent)) if(mype == 0) print *,'af allco wrtComp,write_groups=',write_groups ! pull out the item names and item types from fcstState @@ -457,10 +460,10 @@ subroutine InitializeAdvertise(gcomp, rc) ! prepare petList for wrtComp(i) lead_wrttask(i) = k - do j=1, wrttasks_per_group + do j=1, wrttasks_per_group_from_parent petList(j) = k + j-1 enddo - k = k + wrttasks_per_group + k = k + wrttasks_per_group_from_parent last_wrttask(i) = k - 1 ! if(mype==0)print *,'af wrtComp(i)=',i,'k=',k @@ -487,6 +490,15 @@ subroutine InitializeAdvertise(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return +! get the actual number of PETs executing wrtComp, considering threading + call ESMF_GridCompGet(gridcomp=wrtComp(i),localPet=wrtLocalPet,rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + if (wrtLocalPet/=-1) then + ! This PET does execute inside of wrtComp(i) + call ESMF_GridCompGet(gridcomp=wrtComp(i),petCount=wrttasks_per_group,rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + endif + ! add configuration file call ESMF_GridCompSet(gridcomp=wrtComp(i),config=CF,rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return From 6dfed4eb88123127780e7c4f07f8b5ce38e6cdd2 Mon Sep 17 00:00:00 2001 From: "Jun.Wang" Date: Thu, 4 Aug 2022 20:48:20 +0000 Subject: [PATCH 2/3] fix the fields on pv and sigma levels --- io/post_fv3.F90 | 38 +++++++++++++++++++------------------- io/post_nems_routines.F90 | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/io/post_fv3.F90 b/io/post_fv3.F90 index 4619fbec3..634e55910 100644 --- a/io/post_fv3.F90 +++ b/io/post_fv3.F90 @@ -74,7 +74,7 @@ subroutine post_run_fv3(wrt_int_state,mypei,mpicomp,lead_write, & integer,allocatable :: istagrp(:),iendgrp(:) integer,save :: kpo,kth,kpv logical,save :: log_postalct=.false. - real,dimension(komax),save :: po, th, pv + real(4),dimension(komax),save :: po, th, pv logical :: Log_runpost character(255) :: post_fname*255 @@ -2996,29 +2996,29 @@ subroutine set_postvars_fv3(wrt_int_state,mpicomp,setvar_atmfile, & ! model level cloud fraction if(imp_physics == 11) then !GFDL MP - if(trim(fieldname)=='cld_amt') then - !$omp parallel do default(none) private(i,j,l) shared(lm,jsta,jend,ista,iend,cfr,arrayr43d,fillvalue,spval) - do l=1,lm - do j=jsta,jend - do i=ista, iend - cfr(i,j,l) = arrayr43d(i,j,l) - if(abs(arrayr43d(i,j,l)-fillvalue) Date: Fri, 5 Aug 2022 01:05:10 +0000 Subject: [PATCH 3/3] update upp to fix pressure level TCDC --- upp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upp b/upp index 876792943..e22724738 160000 --- a/upp +++ b/upp @@ -1 +1 @@ -Subproject commit 87679294301b56b9d1fe645b3eafa2a666f93870 +Subproject commit e22724738fd104327fee7c3c7ffc805ccabd619f