Skip to content

Use ESMF Redist instead of Remap for history bundles with 'cubed_sphere_grid' output grid #929

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jkbk2004 merged 5 commits into from
Apr 7, 2025
Merged

Use ESMF Redist instead of Remap for history bundles with 'cubed_sphere_grid' output grid #929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c3711fe
Use ESMF Redist instad of Remap for history bundles with 'cubed_sphe…
DusanJovic-NOAA Feb 7, 2025
8427ad1
Merge remote-tracking branch 'origin/develop' into cubed_sphere_redist
DusanJovic-NOAA Mar 4, 2025
5309dc1
ESMF logging and tracing updates around RouteHandle setup (#8)
theurich Mar 4, 2025
751f1f4
Merge remote-tracking branch 'origin/develop' into cubed_sphere_redist
DusanJovic-NOAA Mar 19, 2025
3002d6d
Merge remote-tracking branch 'origin/develop' into cubed_sphere_redist
DusanJovic-NOAA Apr 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 54 additions & 12 deletions fv3_cap.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ subroutine InitializeAdvertise(gcomp, rc)
integer :: sloc
type(ESMF_StaggerLoc) :: staggerloc
character(len=20) :: cvalue
character(ESMF_MAXSTR) :: output_grid
!
!------------------------------------------------------------------------
!
Expand Down Expand Up @@ -757,10 +758,13 @@ subroutine InitializeAdvertise(gcomp, rc)
if(mype == 0) print *,'af get wrtfb=',"output_"//trim(fcstItemNameList(j)),' rc=',rc
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

call ESMF_AttributeGet(wrtFB(j,i), convention="NetCDF", purpose="FV3-nooutput", &
name="output_grid", value=output_grid, isPresent=isPresent, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

fieldbundle_uses_redist = .false.
! if (fcstItemNameList(j)(1:8) == "restart_" .or. fcstItemNameList(j)(1:18) == "cubed_sphere_grid_") then
if (fcstItemNameList(j)(1:8) == "restart_") then
! restart output forecast bundles, no need to set regridmethod
if (trim(output_grid) == "restart_grid" .or. trim(output_grid) == "cubed_sphere_grid") then
! restart output forecast bundles, or history cubed_sphere (native) grid; no need to set regridmethod
! Redist will be used instead of Regrid
fieldbundle_uses_redist = .true.
else
Expand All @@ -784,8 +788,7 @@ subroutine InitializeAdvertise(gcomp, rc)
endif
endif

call ESMF_LogWrite('bf FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc)
write(msgString,"(A,I2.2,',',I2.2,A)") "calling into wrtFB(",j,i, ") FieldBundleRegridStore()...."
write(msgString,"(A,I2.2,',',I2.2,A)") "RH creation for wrtFB(",j,i, ") ...."//trim(fcstItemNameList(j))
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)

if (i==1) then
Expand All @@ -795,11 +798,25 @@ subroutine InitializeAdvertise(gcomp, rc)

if (rh_file_exist .and. use_saved_routehandles) then
if(mype==0) print *,'in fv3cap init, routehandle file ',trim(rh_filename), ' exists'

write(msgString,*) "Calling into ESMF_RouteHandleCreate(from file)...", trim(rh_filename)
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)

call ESMF_TraceRegionEnter("ESMF_RouteHandleCreate(from file)", rc=rc)
routehandle(j,1) = ESMF_RouteHandleCreate(fileName=trim(rh_filename), rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
call ESMF_TraceRegionExit("ESMF_RouteHandleCreate(from file)", rc=rc)

write(msgString,*) "... returned from ESMF_RouteHandleCreate(from file)."
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)

else
! this is a Store() for the first wrtComp -> must do the Store()
if (fieldbundle_uses_redist) then

write(msgString,*) "Calling into FieldBundleRedistStore..."
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)

call ESMF_TraceRegionEnter("ESMF_FieldBundleRedistStore()", rc=rc)
call ESMF_FieldBundleRedistStore(fcstFB(j), wrtFB(j,1), &
routehandle=routehandle(j,1), &
Expand All @@ -810,9 +827,15 @@ subroutine InitializeAdvertise(gcomp, rc)
! call ESMF_Finalize(endflag=ESMF_END_ABORT)
endif
call ESMF_TraceRegionExit("ESMF_FieldBundleRedistStore()", rc=rc)
call ESMF_LogWrite('af FieldBundleRedistStore', ESMF_LOGMSG_INFO, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

write(msgString,*) "... returned from FieldBundleRedistStore."
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)

else

write(msgString,*) "Calling into FieldBundleRegridStore..."
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)

call ESMF_TraceRegionEnter("ESMF_FieldBundleRegridStore()", rc=rc)
call ESMF_FieldBundleRegridStore(fcstFB(j), wrtFB(j,1), &
regridMethod=regridmethod, routehandle=routehandle(j,1), &
Expand All @@ -823,14 +846,26 @@ subroutine InitializeAdvertise(gcomp, rc)
call ESMF_Finalize(endflag=ESMF_END_ABORT)
endif
call ESMF_TraceRegionExit("ESMF_FieldBundleRegridStore()", rc=rc)
call ESMF_LogWrite('af FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

write(msgString,*) "... returned from FieldBundleRegridStore."
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)

endif

if (use_saved_routehandles) then

write(msgString,*) "Calling into ESMF_RouteHandleWrite...", trim(rh_filename)
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)

call ESMF_TraceRegionEnter("ESMF_RouteHandleWrite()", rc=rc)
call ESMF_RouteHandleWrite(routehandle(j,1), fileName=trim(rh_filename), rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
call ESMF_TraceRegionExit("ESMF_RouteHandleWrite()", rc=rc)
if(mype==0) print *,'in fv3cap init, saved routehandle file ',trim(rh_filename)

write(msgString,*) "... returned from ESMF_RouteHandleWrite."
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)

endif

endif
Expand All @@ -841,15 +876,22 @@ subroutine InitializeAdvertise(gcomp, rc)
else
targetPetList(1:num_pes_fcst) = fcstPetList(:)
targetPetList(num_pes_fcst+1:) = petList(:)
call ESMF_TraceRegionEnter("ESMF_RouteHandleCreate() in lieu of ESMF_FieldBundleRegridStore()", rc=rc)

write(msgString,*) "Calling into ESMF_RouteHandleCreate(from RH)..."
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)

call ESMF_TraceRegionEnter("ESMF_RouteHandleCreate(from RH) in lieu of ESMF_FieldBundleRegridStore()", rc=rc)
routehandle(j,i) = ESMF_RouteHandleCreate(routehandle(j,1), &
originPetList=originPetList, &
targetPetList=targetPetList, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
call ESMF_TraceRegionExit("ESMF_RouteHandleCreate() in lieu of ESMF_FieldBundleRegridStore()", rc=rc)
call ESMF_TraceRegionExit("ESMF_RouteHandleCreate(from RH) in lieu of ESMF_FieldBundleRegridStore()", rc=rc)

write(msgString,*) "... returned from ESMF_RouteHandleCreate(from RH)."
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)

endif
write(msgString,"(A,I2.2,',',I2.2,A)") "... returned from wrtFB(",j,i, ") FieldBundleRegridStore()."
write(msgString,"(A,I2.2,',',I2.2,A)") "... returned from RH creation for wrtFB(",j,i, ")."
call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc)
endif
enddo ! j=1, FBcount
Expand Down
3 changes: 3 additions & 0 deletions io/module_wrt_grid_comp.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -990,6 +990,9 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock,
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
actualWrtGrid = ESMF_GridCreate(fcstGrid, newAcceptorDG, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

call ESMF_AttributeSet(fieldbundle, convention="NetCDF", purpose="FV3-nooutput", name="output_grid", value="restart_grid", rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
else
actualWrtGrid = wrtGrid(grid_id)
call ESMF_AttributeSet(fieldbundle, convention="NetCDF", purpose="FV3-nooutput", name="output_grid", value=output_grid(grid_id), rc=rc)
Expand Down