Skip to content

Write restart files using the write grid component #629

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BrianCurtis-NOAA merged 26 commits into from
Mar 28, 2023
Merged

Write restart files using the write grid component #629

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
ecb99cb
Update .gitmodules
DusanJovic-NOAA Jan 27, 2023
7465502
Started to add support for writing restart file on the write grid comp
DusanJovic-NOAA Feb 2, 2023
b1a20ed
Write restart module, add dyn, add checksum in physics
DusanJovic-NOAA Feb 10, 2023
dbcdb66
Add fv_core support to fv_ufs_restart_io.F90,compute checksums
DusanJovic-NOAA Feb 15, 2023
4f42048
Clean up dimension attributes.
DusanJovic-NOAA Feb 15, 2023
6acb567
More updates for quilting restart
DusanJovic-NOAA Feb 21, 2023
bac4adb
Create write grid for restart files by rebalancing distgrid of the fc…
DusanJovic-NOAA Feb 21, 2023
7af914a
Fix syncing fcstState restart field attributes from fcst to wrt bundles
DusanJovic-NOAA Feb 22, 2023
ba798c0
Move checksum computation to write grid component
DusanJovic-NOAA Feb 23, 2023
689b426
Merge remote-tracking branch 'origin/develop' into quilting_restart
DusanJovic-NOAA Feb 24, 2023
6fc2b83
Fix write grid comp for cases when iau_offset is not 0
DusanJovic-NOAA Feb 25, 2023
21e5c87
Merge remote-tracking branch 'origin/develop' into quilting_restart
DusanJovic-NOAA Feb 27, 2023
3b2070d
clean up after merge, fv3atmStartTime, fv3atmStopTime are not needed …
DusanJovic-NOAA Feb 27, 2023
7525fa5
Remove NF90_CLASSIC_MODEL in order to support 64bit integer attributes
DusanJovic-NOAA Mar 2, 2023
ae0fd1f
Remove unused variables
DusanJovic-NOAA Mar 2, 2023
9cdb9a2
Update atmos_cubed_sphere
DusanJovic-NOAA Mar 7, 2023
6da4f65
Merge remote-tracking branch 'origin/develop' into quilting_restart
DusanJovic-NOAA Mar 7, 2023
a06295a
Update atmos_cubed_sphere
DusanJovic-NOAA Mar 13, 2023
9915c38
Merge remote-tracking branch 'origin/develop' into quilting_restart
DusanJovic-NOAA Mar 13, 2023
176dde5
Code cleanup
DusanJovic-NOAA Mar 15, 2023
f7853d9
Print error messages only on mype==0
DusanJovic-NOAA Mar 15, 2023
f8b03c2
Merge remote-tracking branch 'origin/develop' into quilting_restart
DusanJovic-NOAA Mar 15, 2023
27c0a9e
Fix compilation errors
DusanJovic-NOAA Mar 15, 2023
7ecbc81
Update atmos_cubed_sphere
DusanJovic-NOAA Mar 27, 2023
ff39ec5
Merge remote-tracking branch 'origin/develop' into quilting_restart
DusanJovic-NOAA Mar 27, 2023
b826957
Update atmos_cubed_sphere
DusanJovic-NOAA Mar 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,9 @@ add_library(fv3atm
cpl/module_cplfields.F90
cpl/module_cap_cpl.F90
io/FV3GFS_io.F90
io/FV3GFS_restart_io.F90
io/module_write_netcdf.F90
io/module_write_restart_netcdf.F90
io/module_fv3_io_def.F90
io/module_write_internal_state.F90
io/module_wrt_grid_comp.F90
Expand Down
2 changes: 1 addition & 1 deletion atmos_cubed_sphere
Open in desktop
Submodule atmos_cubed_sphere updated 2 files
+1 −0 CMakeLists.txt
+521 −0 driver/fvGFS/fv_ufs_restart_io.F90
40 changes: 22 additions & 18 deletions atmos_model.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ module atmos_model_mod
use fms_mod, only: check_nml_error
use diag_manager_mod, only: diag_send_complete_instant
use time_manager_mod, only: time_type, get_time, get_date, &
operator(+), operator(-),real_to_time_type
operator(+), operator(-), real_to_time_type
use field_manager_mod, only: MODEL_ATMOS
use tracer_manager_mod, only: get_number_tracers, get_tracer_names, &
get_tracer_index, NO_TRACER
Expand Down Expand Up @@ -94,9 +94,15 @@ module atmos_model_mod
FV3GFS_GFS_checksum, &
FV3GFS_diag_register, FV3GFS_diag_output, &
DIAG_SIZE
use FV3GFS_restart_io_mod, only: FV3GFS_restart_register, &
fv_phy_restart_output, &
fv_sfc_restart_output
use fv_ufs_restart_io_mod, only: fv_dyn_restart_register, &
fv_dyn_restart_output
use fv_iau_mod, only: iau_external_data_type,getiauforcing,iau_initialize
use module_fv3_config, only: output_1st_tstep_rst, first_kdt, nsout, &
output_fh, fcst_mpi_comm, fcst_ntasks
output_fh, fcst_mpi_comm, fcst_ntasks, &
quilting_restart
use module_block_data, only: block_atmos_copy, block_data_copy, &
block_data_copy_or_fill, &
block_data_combine_fractions
Expand Down Expand Up @@ -177,7 +183,6 @@ module atmos_model_mod
logical :: debug = .false.
!logical :: debug = .true.
logical :: sync = .false.
logical :: restart_endfcst = .false.
real :: avg_max_length=3600.
logical :: ignore_rst_cksum = .false.
namelist /atmos_model_nml/ blocksize, chksum_debug, dycore_only, debug, sync, ccpp_suite, avg_max_length, &
Expand Down Expand Up @@ -735,6 +740,10 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
call GFS_restart_populate (GFS_restart_var, GFS_control, GFS_data%Statein, GFS_data%Stateout, GFS_data%Sfcprop, &
GFS_data%Coupling, GFS_data%Grid, GFS_data%Tbd, GFS_data%Cldprop, GFS_data%Radtend, &
GFS_data%IntDiag, Init_parm, GFS_Diag)
if (quilting_restart) then
call fv_dyn_restart_register (Atm(mygrid))
call FV3GFS_restart_register (GFS_data%Sfcprop, GFS_restart_var, Atm_block, GFS_control)
endif
call FV3GFS_restart_read (GFS_data, GFS_restart_var, Atm_block, GFS_control, Atmos%domain_for_read, &
Atm(mygrid)%flagstruct%warm_start, ignore_rst_cksum)
if(GFS_control%do_ca .and. Atm(mygrid)%flagstruct%warm_start)then
Expand Down Expand Up @@ -1055,21 +1064,10 @@ subroutine atmos_model_end (Atmos)
endif
#endif

call atmosphere_end (Atmos % Time, Atmos%grid, restart_endfcst)
call atmosphere_end (Atmos % Time, Atmos%grid, .false.)

if(restart_endfcst) then
call FV3GFS_restart_write (GFS_data, GFS_restart_var, Atm_block, &
GFS_control, Atmos%domain)
! call write_stoch_restart_atm('RESTART/atm_stoch.res.nc')
endif
if (GFS_Control%do_sppt .or. GFS_Control%do_shum .or. GFS_Control%do_skeb .or. &
GFS_Control%lndp_type > 0 .or. GFS_Control%do_ca .or. GFS_Control%do_spp) then
if(restart_endfcst) then
call write_stoch_restart_atm('RESTART/atm_stoch.res.nc')
if (GFS_control%do_ca)then
call write_ca_restart()
endif
endif
call stochastic_physics_wrapper_end(GFS_control)
endif

Expand Down Expand Up @@ -1099,9 +1097,15 @@ subroutine atmos_model_restart(Atmos, timestamp)
type (atmos_data_type), intent(inout) :: Atmos
character(len=*), intent(in) :: timestamp

call atmosphere_restart(timestamp)
call FV3GFS_restart_write (GFS_data, GFS_restart_var, Atm_block, &
GFS_control, Atmos%domain, timestamp)
if (quilting_restart) then
call fv_sfc_restart_output(GFS_Data%Sfcprop, Atm_block, GFS_control)
call fv_phy_restart_output(GFS_restart_var, Atm_block)
call fv_dyn_restart_output(Atm(mygrid), timestamp)
else
call atmosphere_restart(timestamp)
call FV3GFS_restart_write (GFS_data, GFS_restart_var, Atm_block, &
GFS_control, Atmos%domain, timestamp)
endif
if(GFS_control%do_ca)then
call write_ca_restart(timestamp)
endif
Expand Down
131 changes: 89 additions & 42 deletions fv3_cap.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ module fv3gfs_cap_mod
label_Finalize, &
NUOPC_ModelGet
!
use module_fv3_config, only: quilting, output_fh, &
use module_fv3_config, only: quilting, quilting_restart, output_fh, &
nfhout, nfhout_hf, nsout, dt_atmos, &
calendar, cpl_grid_id, &
cplprint_flag,output_1st_tstep_rst, &
Expand Down Expand Up @@ -58,6 +58,7 @@ module fv3gfs_cap_mod
type(ESMF_GridComp) :: fcstComp
type(ESMF_State) :: fcstState
type(ESMF_FieldBundle), allocatable :: fcstFB(:)
integer,dimension(:), allocatable :: fcstPetList
integer, save :: FBCount

type(ESMF_GridComp), allocatable :: wrtComp(:)
Expand All @@ -73,6 +74,7 @@ module fv3gfs_cap_mod

integer :: mype = -1
integer :: dbug = 0
integer :: frestart(999) = -1

!-----------------------------------------------------------------------

Expand Down Expand Up @@ -187,8 +189,8 @@ subroutine InitializeAdvertise(gcomp, rc)
real :: nfhmax
real :: output_startfh, outputfh, outputfh2(2)
logical :: loutput_fh, lfreq
character(ESMF_MAXSTR) :: name
integer,dimension(:), allocatable :: petList, fcstPetList, originPetList, targetPetList
character(ESMF_MAXSTR) :: gc_name, fb_name
integer,dimension(:), allocatable :: petList, originPetList, targetPetList
character(len=esmf_maxstr),allocatable :: fcstItemNameList(:)
type(ESMF_StateItem_Flag), allocatable :: fcstItemTypeList(:)
character(20) :: cwrtcomp
Expand All @@ -205,22 +207,29 @@ subroutine InitializeAdvertise(gcomp, rc)
character(len=*),parameter :: subname='(fv3_cap:InitializeAdvertise)'
real(kind=8) :: MPI_Wtime, timeis, timerhs

integer :: wrttasks_per_group_from_parent, wrtLocalPet
integer :: wrttasks_per_group_from_parent, wrtLocalPet, num_threads
character(len=64) :: rh_filename
logical :: use_saved_routehandles, rh_file_exist
logical :: fieldbundle_is_restart = .false.

!
!------------------------------------------------------------------------
!
rc = ESMF_SUCCESS
timeis = MPI_Wtime()

call ESMF_GridCompGet(gcomp,name=name,vm=vm,rc=rc)
call ESMF_GridCompGet(gcomp, name=gc_name, vm=vm,rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

call ESMF_VMGet(vm, petCount=petcount, localpet=mype, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

! num_threads is needed to compute actual wrttasks_per_group_from_parent
call ESMF_InfoGetFromHost(gcomp, info=info, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
call ESMF_InfoGet(info, key="/NUOPC/Hint/PePerPet/MaxCount", value=num_threads, default=1, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

! query for importState and exportState
call NUOPC_ModelGet(gcomp, driverClock=clock, importState=importState, exportState=exportState, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
Expand Down Expand Up @@ -269,6 +278,12 @@ subroutine InitializeAdvertise(gcomp, rc)
label ='quilting:',rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

call ESMF_ConfigGetAttribute(config=CF,value=quilting_restart, &
default=.false., label ='quilting_restart:',rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

if (.not.quilting) quilting_restart = .false.

call ESMF_ConfigGetAttribute(config=CF,value=output_1st_tstep_rst, &
default=.false., label ='output_1st_tstep_rst:',rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
Expand Down Expand Up @@ -351,6 +366,7 @@ subroutine InitializeAdvertise(gcomp, rc)
! create fcst grid component

if( quilting ) then
wrttasks_per_group_from_parent = wrttasks_per_group_from_parent * num_threads
Comment thread
junwang-noaa marked this conversation as resolved.
num_pes_fcst = petcount - write_groups * wrttasks_per_group_from_parent
else
num_pes_fcst = petcount
Expand Down Expand Up @@ -396,7 +412,7 @@ subroutine InitializeAdvertise(gcomp, rc)
! determine number elements in fcstState
call ESMF_StateGet(fcstState, itemCount=FBCount, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
if(mype == 0) print *,'af fcstCom FBCount= ',FBcount
if(mype == 0) print *,'fv3_cap: field bundles in fcstComp export state, FBCount= ',FBcount
!
! set start time for output
output_startfh = 0.
Expand Down Expand Up @@ -461,6 +477,9 @@ subroutine InitializeAdvertise(gcomp, rc)
endif
call ESMF_AttributeGet(fcstFB(i), convention="NetCDF", purpose="FV3", name="grid_id", value=grid_id, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
call ESMF_AttributeGet(fcstFB(i), convention="NetCDF", purpose="FV3-nooutput", name="frestart", valueList=frestart, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

is_moving_fb(i) = is_moving(grid_id)
enddo
!
Expand Down Expand Up @@ -540,8 +559,7 @@ subroutine InitializeAdvertise(gcomp, rc)

if(mype==0) print *,'af wrtState reconcile, FBcount=',FBcount

call ESMF_AttributeCopy(fcstState, wrtState(i), &
attcopy=ESMF_ATTCOPY_REFERENCE, rc=rc)
call ESMF_AttributeCopy(fcstState, wrtState(i), attcopy=ESMF_ATTCOPY_REFERENCE, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

! deal with GridTransfer if needed
Expand Down Expand Up @@ -596,8 +614,8 @@ subroutine InitializeAdvertise(gcomp, rc)
! loop over all FieldBundle in the states, for moving nests handle GridTransfer
do j=1, FBcount
if (is_moving_fb(j)) then
! access the fcst (provider) Grid
call ESMF_FieldBundleGet(fcstFB(j), grid=providerGrid, rc=rc)
! access the fcst (provider) Grid and fieldbundle name
call ESMF_FieldBundleGet(fcstFB(j), grid=providerGrid, name=fb_name, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
! access the mirror FieldBundle on the wrtComp
call ESMF_StateGet(wrtState(i), itemName="mirror_"//trim(fcstItemNameList(j)), fieldbundle=mirrorFB, rc=rc)
Expand Down Expand Up @@ -683,22 +701,30 @@ subroutine InitializeAdvertise(gcomp, rc)
if(mype == 0) print *,'af get wrtfb=',"output_"//trim(fcstItemNameList(j)),' rc=',rc
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

! determine regridmethod
if (index(fcstItemNameList(j),"_bilinear") >0 ) then
regridmethod = ESMF_REGRIDMETHOD_BILINEAR
else if (index(fcstItemNameList(j),"_patch") >0) then
regridmethod = ESMF_REGRIDMETHOD_PATCH
else if (index(fcstItemNameList(j),"_nearest_stod") >0) then
regridmethod = ESMF_REGRIDMETHOD_NEAREST_STOD
else if (index(fcstItemNameList(j),"_nearest_dtos") >0) then
regridmethod = ESMF_REGRIDMETHOD_NEAREST_DTOS
else if (index(fcstItemNameList(j),"_conserve") >0) then
regridmethod = ESMF_REGRIDMETHOD_CONSERVE
fieldbundle_is_restart = .false.
if (fcstItemNameList(j)(1:8) == "restart_") then
! restart output forecast bundles, no need to set regridmethod
! Redist will be used instead of Regrid
fieldbundle_is_restart = .true.
else
call ESMF_LogSetError(ESMF_RC_ARG_BAD, &
msg="Unable to determine regrid method.", &
line=__LINE__, file=__FILE__, rcToReturn=rc)
return
! history output forecast bundles
! determine regridmethod
if (index(fcstItemNameList(j),"_bilinear") >0 ) then
regridmethod = ESMF_REGRIDMETHOD_BILINEAR
else if (index(fcstItemNameList(j),"_patch") >0) then
regridmethod = ESMF_REGRIDMETHOD_PATCH
else if (index(fcstItemNameList(j),"_nearest_stod") >0) then
regridmethod = ESMF_REGRIDMETHOD_NEAREST_STOD
else if (index(fcstItemNameList(j),"_nearest_dtos") >0) then
regridmethod = ESMF_REGRIDMETHOD_NEAREST_DTOS
else if (index(fcstItemNameList(j),"_conserve") >0) then
regridmethod = ESMF_REGRIDMETHOD_CONSERVE
else
call ESMF_LogSetError(ESMF_RC_ARG_BAD, &
msg="Unable to determine regrid method.", &
line=__LINE__, file=__FILE__, rcToReturn=rc)
return
endif
endif

call ESMF_LogWrite('bf FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc)
Expand All @@ -716,21 +742,33 @@ subroutine InitializeAdvertise(gcomp, rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
else
! this is a Store() for the first wrtComp -> must do the Store()
call ESMF_TraceRegionEnter("ESMF_FieldBundleRegridStore()", rc=rc)
call ESMF_FieldBundleRegridStore(fcstFB(j), wrtFB(j,1), &
regridMethod=regridmethod, routehandle=routehandle(j,1), &
unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, &
srcTermProcessing=isrcTermProcessing, rc=rc)

! if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
if (rc /= ESMF_SUCCESS) then
write(0,*)'fv3_cap.F90:InitializeAdvertise error in ESMF_FieldBundleRegridStore'
call ESMF_LogWrite('fv3_cap.F90:InitializeAdvertise error in ESMF_FieldBundleRegridStore', ESMF_LOGMSG_ERROR, rc=rc)
call ESMF_Finalize(endflag=ESMF_END_ABORT)
if (fieldbundle_is_restart) then
call ESMF_TraceRegionEnter("ESMF_FieldBundleRedistStore()", rc=rc)
call ESMF_FieldBundleRedistStore(fcstFB(j), wrtFB(j,1), &
routehandle=routehandle(j,1), &
rc=rc)
if (rc /= ESMF_SUCCESS) then
call ESMF_LogWrite('fv3_cap.F90:InitializeAdvertise error in ESMF_FieldBundleRedistStore', ESMF_LOGMSG_ERROR, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
! call ESMF_Finalize(endflag=ESMF_END_ABORT)
endif
call ESMF_TraceRegionExit("ESMF_FieldBundleRedistStore()", rc=rc)
call ESMF_LogWrite('af FieldBundleRedistStore', ESMF_LOGMSG_INFO, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
else
call ESMF_TraceRegionEnter("ESMF_FieldBundleRegridStore()", rc=rc)
call ESMF_FieldBundleRegridStore(fcstFB(j), wrtFB(j,1), &
regridMethod=regridmethod, routehandle=routehandle(j,1), &
unmappedaction=ESMF_UNMAPPEDACTION_IGNORE, &
srcTermProcessing=isrcTermProcessing, rc=rc)
if (rc /= ESMF_SUCCESS) then
call ESMF_LogWrite('fv3_cap.F90:InitializeAdvertise error in ESMF_FieldBundleRegridStore', ESMF_LOGMSG_ERROR, rc=rc)
call ESMF_Finalize(endflag=ESMF_END_ABORT)
endif
call ESMF_TraceRegionExit("ESMF_FieldBundleRegridStore()", rc=rc)
call ESMF_LogWrite('af FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
endif
call ESMF_TraceRegionExit("ESMF_FieldBundleRegridStore()", rc=rc)
call ESMF_LogWrite('af FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

if (use_saved_routehandles) then
call ESMF_RouteHandleWrite(routehandle(j,1), fileName=trim(rh_filename), rc=rc)
Expand Down Expand Up @@ -916,7 +954,16 @@ subroutine InitializeAdvertise(gcomp, rc)
endif ! end loutput_fh
endif
if(mype==0) print *,'output_fh=',output_fh(1:size(output_fh)),'lflname_fulltime=',lflname_fulltime
!

if ( quilting ) then
do i=1, write_groups
call ESMF_InfoGetFromHost(wrtState(i), info=info, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
call ESMF_InfoSet(info, key="output_fh", values=output_fh, rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
enddo
endif

! --- advertise Fields in importState and exportState -------------------

! call fcst Initialize (advertise phase)
Expand Down Expand Up @@ -1085,8 +1132,8 @@ subroutine ModelAdvance_phase2(gcomp, rc)
na = nint(time_elapsed/timeStep)
call ESMF_TimeIntervalGet(time_elapsed, s=nfseconds, rc=rc)

output: if (ANY(nint(output_fh(:)*3600.0) == nfseconds)) then
!
output: if (ANY(nint(output_fh(:)*3600.0) == nfseconds) .or. ANY(frestart(:) == nfseconds)) then

if (mype == 0 .or. mype == lead_wrttask(1)) print *,' aft fcst run output time=',nfseconds, &
'FBcount=',FBcount,'na=',na

Expand Down
Loading