From c3711feea8906b0a9e71d16bb7bc9231dcbd9330 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Fri, 7 Feb 2025 13:06:20 -0600 Subject: [PATCH 1/2] Use ESMF Redist instad of Remap for history bundles with 'cubed_sphere_grid' output grid. --- fv3_cap.F90 | 10 +++++++--- io/module_wrt_grid_comp.F90 | 3 +++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 4d3f999d2..5b8bcc407 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -220,6 +220,7 @@ subroutine InitializeAdvertise(gcomp, rc) integer :: sloc type(ESMF_StaggerLoc) :: staggerloc character(len=20) :: cvalue + character(ESMF_MAXSTR) :: output_grid ! !------------------------------------------------------------------------ ! @@ -757,10 +758,13 @@ subroutine InitializeAdvertise(gcomp, rc) if(mype == 0) print *,'af get wrtfb=',"output_"//trim(fcstItemNameList(j)),' rc=',rc if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_AttributeGet(wrtFB(j,i), convention="NetCDF", purpose="FV3-nooutput", & + name="output_grid", value=output_grid, isPresent=isPresent, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + fieldbundle_uses_redist = .false. - ! if (fcstItemNameList(j)(1:8) == "restart_" .or. fcstItemNameList(j)(1:18) == "cubed_sphere_grid_") then - if (fcstItemNameList(j)(1:8) == "restart_") then - ! restart output forecast bundles, no need to set regridmethod + if (trim(output_grid) == "restart_grid" .or. trim(output_grid) == "cubed_sphere_grid") then + ! restart output forecast bundles, or history cubed_sphere (native) grid; no need to set regridmethod ! Redist will be used instead of Regrid fieldbundle_uses_redist = .true. else diff --git a/io/module_wrt_grid_comp.F90 b/io/module_wrt_grid_comp.F90 index af468c57e..97516a420 100644 --- a/io/module_wrt_grid_comp.F90 +++ b/io/module_wrt_grid_comp.F90 @@ -990,6 +990,9 @@ subroutine wrt_initialize_p1(wrt_comp, imp_state_write, exp_state_write, clock, if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return actualWrtGrid = ESMF_GridCreate(fcstGrid, newAcceptorDG, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeSet(fieldbundle, convention="NetCDF", purpose="FV3-nooutput", name="output_grid", value="restart_grid", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return else actualWrtGrid = wrtGrid(grid_id) call ESMF_AttributeSet(fieldbundle, convention="NetCDF", purpose="FV3-nooutput", name="output_grid", value=output_grid(grid_id), rc=rc) From 5309dc1d61cedd74865bc9d4640c838e3bad28b5 Mon Sep 17 00:00:00 2001 From: Gerhard Theurich Date: Tue, 4 Mar 2025 08:43:11 -0800 Subject: [PATCH 2/2] ESMF logging and tracing updates around RouteHandle setup (#8) * Debugging log around RH creation. * Log the fcstItemNameList(j) when looping over RH creation... which contains info on regrid method. * Cleanup to be generally usable code. --- fv3_cap.F90 | 56 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 5b8bcc407..747228c5d 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -788,8 +788,7 @@ subroutine InitializeAdvertise(gcomp, rc) endif endif - call ESMF_LogWrite('bf FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc) - write(msgString,"(A,I2.2,',',I2.2,A)") "calling into wrtFB(",j,i, ") FieldBundleRegridStore()...." + write(msgString,"(A,I2.2,',',I2.2,A)") "RH creation for wrtFB(",j,i, ") ...."//trim(fcstItemNameList(j)) call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) if (i==1) then @@ -799,11 +798,25 @@ subroutine InitializeAdvertise(gcomp, rc) if (rh_file_exist .and. use_saved_routehandles) then if(mype==0) print *,'in fv3cap init, routehandle file ',trim(rh_filename), ' exists' + + write(msgString,*) "Calling into ESMF_RouteHandleCreate(from file)...", trim(rh_filename) + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + + call ESMF_TraceRegionEnter("ESMF_RouteHandleCreate(from file)", rc=rc) routehandle(j,1) = ESMF_RouteHandleCreate(fileName=trim(rh_filename), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_TraceRegionExit("ESMF_RouteHandleCreate(from file)", rc=rc) + + write(msgString,*) "... returned from ESMF_RouteHandleCreate(from file)." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + else ! this is a Store() for the first wrtComp -> must do the Store() if (fieldbundle_uses_redist) then + + write(msgString,*) "Calling into FieldBundleRedistStore..." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_TraceRegionEnter("ESMF_FieldBundleRedistStore()", rc=rc) call ESMF_FieldBundleRedistStore(fcstFB(j), wrtFB(j,1), & routehandle=routehandle(j,1), & @@ -814,9 +827,15 @@ subroutine InitializeAdvertise(gcomp, rc) ! call ESMF_Finalize(endflag=ESMF_END_ABORT) endif call ESMF_TraceRegionExit("ESMF_FieldBundleRedistStore()", rc=rc) - call ESMF_LogWrite('af FieldBundleRedistStore', ESMF_LOGMSG_INFO, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + write(msgString,*) "... returned from FieldBundleRedistStore." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + else + + write(msgString,*) "Calling into FieldBundleRegridStore..." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + call ESMF_TraceRegionEnter("ESMF_FieldBundleRegridStore()", rc=rc) call ESMF_FieldBundleRegridStore(fcstFB(j), wrtFB(j,1), & regridMethod=regridmethod, routehandle=routehandle(j,1), & @@ -827,14 +846,26 @@ subroutine InitializeAdvertise(gcomp, rc) call ESMF_Finalize(endflag=ESMF_END_ABORT) endif call ESMF_TraceRegionExit("ESMF_FieldBundleRegridStore()", rc=rc) - call ESMF_LogWrite('af FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + write(msgString,*) "... returned from FieldBundleRegridStore." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + endif if (use_saved_routehandles) then + + write(msgString,*) "Calling into ESMF_RouteHandleWrite...", trim(rh_filename) + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + + call ESMF_TraceRegionEnter("ESMF_RouteHandleWrite()", rc=rc) call ESMF_RouteHandleWrite(routehandle(j,1), fileName=trim(rh_filename), rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_TraceRegionExit("ESMF_RouteHandleWrite()", rc=rc) if(mype==0) print *,'in fv3cap init, saved routehandle file ',trim(rh_filename) + + write(msgString,*) "... returned from ESMF_RouteHandleWrite." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + endif endif @@ -845,15 +876,22 @@ subroutine InitializeAdvertise(gcomp, rc) else targetPetList(1:num_pes_fcst) = fcstPetList(:) targetPetList(num_pes_fcst+1:) = petList(:) - call ESMF_TraceRegionEnter("ESMF_RouteHandleCreate() in lieu of ESMF_FieldBundleRegridStore()", rc=rc) + + write(msgString,*) "Calling into ESMF_RouteHandleCreate(from RH)..." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) + + call ESMF_TraceRegionEnter("ESMF_RouteHandleCreate(from RH) in lieu of ESMF_FieldBundleRegridStore()", rc=rc) routehandle(j,i) = ESMF_RouteHandleCreate(routehandle(j,1), & originPetList=originPetList, & targetPetList=targetPetList, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_TraceRegionExit("ESMF_RouteHandleCreate() in lieu of ESMF_FieldBundleRegridStore()", rc=rc) + call ESMF_TraceRegionExit("ESMF_RouteHandleCreate(from RH) in lieu of ESMF_FieldBundleRegridStore()", rc=rc) + + write(msgString,*) "... returned from ESMF_RouteHandleCreate(from RH)." + call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) endif - write(msgString,"(A,I2.2,',',I2.2,A)") "... returned from wrtFB(",j,i, ") FieldBundleRegridStore()." + write(msgString,"(A,I2.2,',',I2.2,A)") "... returned from RH creation for wrtFB(",j,i, ")." call ESMF_LogWrite(msgString, ESMF_LOGMSG_INFO, rc=rc) endif enddo ! j=1, FBcount