Skip to content

[production/RRFS.v1] Hailcast restart reproducibility fix #91

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MatthewPyle-NOAA merged 2 commits into from
Jan 20, 2026
Merged

[production/RRFS.v1] Hailcast restart reproducibility fix #91

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
405ca88
Add new 'fv_diag.res' restart file, that currently contains two hailc…
DusanJovic-NOAA Jun 12, 2025
f7f935f
Merge pull request #21 from DusanJovic-NOAA/rrfs_v1_conus13km_tests
JiliDong-NOAA Oct 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 71 additions & 7 deletions driver/fvGFS/fv_ufs_restart_io.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
module fv_ufs_restart_io_mod

use esmf
use mpp_mod, only: mpp_pe, mpp_root_pe, mpp_chksum, mpp_npes, mpp_get_current_pelist
use fv_arrays_mod, only: fv_atmos_type
use tracer_manager_mod, only: get_tracer_names
use field_manager_mod, only: MODEL_ATMOS
use atmosphere_mod, only: atmosphere_resolution
use fv_io_mod, only: fv_io_write_BCs
use mpp_mod, only: mpp_pe, mpp_root_pe, mpp_chksum, mpp_npes, &
mpp_get_current_pelist, mpp_error, FATAL
use fv_arrays_mod, only: fv_atmos_type
use tracer_manager_mod, only: get_tracer_names
use field_manager_mod, only: MODEL_ATMOS
use atmosphere_mod, only: atmosphere_resolution
use fv_io_mod, only: fv_io_write_BCs
use module_diag_hailcast, only: do_hailcast, hailcast_wdur, hailcast_wup_mask

implicit none

Expand All @@ -16,6 +18,7 @@ module fv_ufs_restart_io_mod
public fv_core_restart_bundle_setup
public fv_srf_wnd_restart_bundle_setup
public fv_tracer_restart_bundle_setup
public fv_diag_restart_bundle_setup

private

Expand All @@ -36,7 +39,12 @@ module fv_ufs_restart_io_mod
real, allocatable, target, dimension(:,:,:,:) :: tracers_var3
character(len=32), allocatable, dimension(:) :: tracers_var3_names

type(ESMF_FieldBundle) :: core_bundle, srf_wnd_bundle, tracer_bundle
! fv_diag.res
integer :: nvar2d_diag = 0
real, allocatable, target, dimension(:,:,:) :: diag_var2
character(len=32), allocatable, dimension(:) :: diag_var2_names

type(ESMF_FieldBundle) :: core_bundle, srf_wnd_bundle, tracer_bundle, diag_bundle

contains

Expand Down Expand Up @@ -153,6 +161,15 @@ subroutine fv_dyn_restart_register (Atm)
tracers_var3_names(nt) = tracer_name
enddo

if (do_hailcast) then
! diag
nvar2d_diag = 2
allocate (diag_var2(nx,ny,nvar2d_diag), diag_var2_names(nvar2d_diag))
diag_var2 = 0.0
diag_var2_names(1) = 'hailcast_wdur'
diag_var2_names(2) = 'hailcast_wup_mask'
endif

end subroutine fv_dyn_restart_register

subroutine fv_dyn_restart_output(Atm, timestamp)
Expand Down Expand Up @@ -222,6 +239,12 @@ subroutine fv_dyn_restart_output(Atm, timestamp)
tracers_var3(:,:,:,nt) = Atm%qdiag(isc:iec,jsc:jec,:,nt)
enddo

! ---- fv_diag.res
if (do_hailcast) then
diag_var2(:,:,1) = hailcast_wdur(isc:iec,jsc:jec)
diag_var2(:,:,2) = hailcast_wup_mask(isc:iec,jsc:jec)
end if

! Instead of creating yet another esmf bundle just to write Atm%ak and Atm%bk, write them here synchronously
call write_ak_bk(Atm, timestamp)

Expand Down Expand Up @@ -352,6 +375,47 @@ subroutine fv_tracer_restart_bundle_setup(bundle, grid, rc)

end subroutine fv_tracer_restart_bundle_setup

subroutine fv_diag_restart_bundle_setup(bundle, grid, rc)
!
!-------------------------------------------------------------
!*** set esmf bundle for fv_diag restart fields
!------------------------------------------------------------
!
use esmf

implicit none

type(ESMF_FieldBundle),intent(inout) :: bundle
type(ESMF_Grid),intent(inout) :: grid
integer,intent(out) :: rc

!*** local variables
integer i, j, k, n
character(128) :: bdl_name
type(ESMF_Field) :: field
character(128) :: outputfile
integer :: num
real,dimension(:,:),pointer :: temp_r2d
real,dimension(:,:,:),pointer :: temp_r3d

if (.not. do_hailcast) then
call mpp_error(FATAL, 'fv_diag_restart_bundle_setup called with do_hailcast == .false.')
end if

diag_bundle = bundle

call ESMF_FieldBundleGet(bundle, name=bdl_name,rc=rc)
if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return

outputfile = trim(bdl_name)

do num = 1,nvar2d_diag
temp_r2d => diag_var2(:,:,num)
call create_2d_field_and_add_to_bundle(temp_r2d, trim(diag_var2_names(num)), trim(outputfile), grid, bundle)
enddo

end subroutine fv_diag_restart_bundle_setup

subroutine create_2d_field_and_add_to_bundle(temp_r2d, field_name, outputfile, grid, bundle)

use esmf
Expand Down
3 changes: 2 additions & 1 deletion model/fv_arrays.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1354,14 +1354,15 @@ module fv_arrays_mod
!!!!!!!!!!!!!!
type(FmsNetcdfFile_t) :: Fv_restart
type(FmsNetcdfDomainFile_t) :: SST_restart, Fv_restart_tile, &
Rsf_restart, Mg_restart, Lnd_restart, Tra_restart
Rsf_restart, Mg_restart, Lnd_restart, Tra_restart, Diag_restart
logical :: Fv_restart_is_open=.false.
logical :: SST_restart_is_open=.false.
logical :: Fv_restart_tile_is_open=.false.
logical :: Rsf_restart_is_open=.false.
logical :: Mg_restart_is_open=.false.
logical :: Lnd_restart_is_open=.false.
logical :: Tra_restart_is_open=.false.
logical :: Diag_restart_is_open=.false.
type(fv_nest_type) :: neststruct

!Hold on to coarse-grid global grid, so we don't have to waste processor time getting it again when starting to do grid nesting
Expand Down
60 changes: 59 additions & 1 deletion tools/fv_io.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,9 @@ module fv_io_mod
write_restart_bc, close_file, register_field, write_data, &
get_global_io_domain_indices, register_variable_attribute, &
variable_exists, read_data, set_filename_appendix, get_dimension_size
use fms_mod, only: check_nml_error
use mpp_mod, only: mpp_error, FATAL, NOTE, WARNING, mpp_root_pe, &
mpp_sync, mpp_pe, mpp_declare_pelist, mpp_get_current_pelist, &
mpp_sync, mpp_pe, mpp_declare_pelist, mpp_get_current_pelist, input_nml_file, &
#ifdef ENABLE_PARALLELRESTART
mpp_npes, MPP_COMM_NULL
#else
Expand All @@ -118,6 +119,7 @@ module fv_io_mod

use fv_mp_mod, only: mp_gather, is_master
use fv_treat_da_inc_mod, only: read_da_inc
use module_diag_hailcast, only: do_hailcast, hailcast_wdur, hailcast_wup_mask

implicit none
private
Expand Down Expand Up @@ -425,6 +427,14 @@ subroutine fv_io_register_restart(Atm)
call register_restart_field(Atm%Tra_restart, tracer_name, Atm%qdiag(:,:,:,nt), &
dim_names_4d, chunksizes=chunksizes_4d, is_optional=.true.)
enddo

! fname = 'fv_diag.res'//trim(stile_name)//'.nc
elseif (Atm%Diag_restart_is_open) then
call fv_io_register_axis(Atm%Diag_restart, numx=numx, numy=numy, xpos=xpos, ypos=ypos)
call get_dim_chunksizes(Atm%Diag_restart, dim_names_3d2, chunksizes_3d2)

call register_restart_field(Atm%Diag_restart, 'hailcast_wdur', hailcast_wdur, dim_names_3d2, chunksizes=chunksizes_3d2)
call register_restart_field(Atm%Diag_restart, 'hailcast_wup_mask', hailcast_wup_mask, dim_names_3d2, chunksizes=chunksizes_3d2)
endif
end subroutine fv_io_register_restart
! </SUBROUTINE> NAME="fv_io_register_restart"
Expand All @@ -434,12 +444,16 @@ end subroutine fv_io_register_restart
subroutine fv_io_read_restart(fv_domain,Atm)
type(domain2d), intent(inout) :: fv_domain
type(fv_atmos_type), intent(inout) :: Atm(:)
namelist /fv_diagnostics_nml/ do_hailcast

character(len=64) :: tracer_name
integer :: isc, iec, jsc, jec, n, nt, nk, ntracers
integer :: ntileMe
integer :: ks, ntiles
real :: ptop
integer :: isdo, iedo, jsdo, jedo
integer :: ios, ierr
integer :: unit

character(len=128) :: tracer_longname, tracer_units
character(len=120) :: fname
Expand All @@ -452,6 +466,13 @@ subroutine fv_io_read_restart(fv_domain,Atm)
allocate(pes(mpp_npes()))
call mpp_get_current_pelist(pes)

!namelist file for hailcast

! Read Main namelist
read (input_nml_file,fv_diagnostics_nml,iostat=ios)
ierr = check_nml_error(ios,'fv_diagnostics_nml')


suffix = ''
fname = 'INPUT/fv_core.res.nc'
Atm(1)%Fv_restart_is_open = open_file(Atm(1)%Fv_restart,fname,"read", is_restart=.true., pelist=pes)
Expand Down Expand Up @@ -564,6 +585,28 @@ subroutine fv_io_read_restart(fv_domain,Atm)
endif
endif

if ( do_hailcast ) then
!--- restore data for fv_diag - if it exists
fname = 'INPUT/fv_diag.res'//trim(suffix)//'.nc'
Atm(1)%Diag_restart_is_open = open_file(Atm(1)%Diag_restart, fname, "read", fv_domain, is_restart=.true.)
if (Atm(1)%Diag_restart_is_open) then
if (.not.allocated(hailcast_wdur)) then
isdo = Atm(1)%bd%isd; iedo = Atm(1)%bd%ied
jsdo = Atm(1)%bd%jsd; jedo = Atm(1)%bd%jed
allocate(hailcast_wdur(isdo:iedo,jsdo:jedo))
allocate(hailcast_wup_mask(isdo:iedo,jsdo:jedo))
hailcast_wdur = 0
hailcast_wup_mask = 0
endif
call fv_io_register_restart(Atm(1))
call read_restart(Atm(1)%Diag_restart, ignore_checksum=Atm(1)%flagstruct%ignore_rst_cksum)
call close_file(Atm(1)%Diag_restart)
Atm(1)%Diag_restart_is_open = .false.
else
call mpp_error(NOTE,'==> Warning from fv_read_restart: Expected file '//trim(fname)//' does not exist')
endif
endif

deallocate(pes)
return

Expand Down Expand Up @@ -944,6 +987,21 @@ subroutine fv_io_write_restart(Atm, timestamp)
Atm%Tra_restart_is_open = .false.
endif

if ( do_hailcast ) then
if (present(timestamp)) then
fname = 'RESTART/'//trim(timestamp)//'.fv_diag.res'//trim(suffix)//'.nc'
else
fname = 'RESTART/fv_diag.res'//trim(suffix)//'.nc'
endif
Atm%Diag_restart_is_open = open_file(Atm%Diag_restart, fname, "overwrite", fv_domain, is_restart=.true.)
if (Atm%Diag_restart_is_open) then
call fv_io_register_restart(Atm)
call write_restart(Atm%Diag_restart)
call close_file(Atm%Diag_restart)
Atm%Diag_restart_is_open = .false.
endif
endif

end subroutine fv_io_write_restart

subroutine register_bcs_2d(Atm, BCfile_ne, BCfile_sw, fname_ne, fname_sw, &
Expand Down