Skip to content

Parallelize surface increment regridding #1109

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BrianCurtis-NOAA merged 8 commits into from
Nov 4, 2025
Merged

Parallelize surface increment regridding #1109

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2df8472
Update
DavidNew-NOAA Oct 14, 2025
f091f4e
Bugfix
DavidNew-NOAA Oct 23, 2025
d543abc
Add nmem_ens to unit test namelist
DavidNew-NOAA Oct 31, 2025
bbe8a25
Fix regression test
DavidNew-NOAA Nov 3, 2025
908bbaa
Fix unit test
DavidNew-NOAA Nov 3, 2025
616c096
Add documentation for function subroutine inputs
DavidNew-NOAA Nov 4, 2025
7d16d0b
More missing documentation
DavidNew-NOAA Nov 4, 2025
a7d2cb9
More missing documentation
DavidNew-NOAA Nov 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions parm/regrid_sfc/regrid.nml_tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,26 @@
missing_value=0.,
time_list=@[time_list],
add_time_dim=@[add_time_dim],
nmem_ens=@[NMEM_REGRID],
/
&input
gridtype="gau_inc",
ires=@[ires],
jres=@[jres],
fname=@[in_fname],
dir="./",
dir=@[in_dir],
fname_coord="gaussian_scrip.nc",
dir_coord="./",
fname_mask=@[fname_mask_in],
dir_mask=@[dir_mask_in]
dir_mask=@[dir_mask_in],
/
&output
gridtype="fv3_rst",
ires=@[ireso],
jres=@[jreso],
fname=@[out_fname],
dir="./",
fname_mask="vegetation_type"
dir_mask="./"
dir_coord="./"
dir=@[out_dir],
fname_mask="vegetation_type",
dir_mask="./",
dir_coord="./",
/
1 change: 1 addition & 0 deletions reg_tests/regrid_sfc/gauss2fv3incr.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ cat << EOF > regrid.nml
time_list=6,
add_time_dim=.true.,
extrap_levs=2,
nmem_ens=1,
/
&input
gridtype="gau_inc",
Expand Down
44 changes: 23 additions & 21 deletions sorc/regrid_sfc.fd/grids_IO.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,17 +43,18 @@ module grids_IO
!> Create ESMF grid objects, with mask if requested
!! @param[in] localpet local pet
!! @param[in] npets total number of pets
!! @param[in] imem_ens ensemble member index
!! @param[in] grid_setup data structure with grid details
!! @param[out] mod_grid output esmf_grid structure
!! @param[in] timestamp timestep of input file

subroutine setup_grid(localpet, npets, grid_setup, mod_grid, timestamp )
subroutine setup_grid(localpet, npets, imem_ens, grid_setup, mod_grid, timestamp )

implicit none

! INTENT IN
type(grid_setup_type), intent(in) :: grid_setup
integer, intent(in) :: localpet, npets
integer, intent(in) :: localpet, npets, imem_ens
integer, intent(in), optional :: timestamp

! INTENT OUT
Expand All @@ -74,9 +75,9 @@ subroutine setup_grid(localpet, npets, grid_setup, mod_grid, timestamp )

select case (grid_setup%descriptor)
case ('fv3_rst')
call create_grid_fv3(grid_setup%ires, trim(grid_setup%dir_coord), npets, localpet ,mod_grid)
call create_grid_fv3(grid_setup%ires, trim(grid_setup%dir_coord), npets, localpet, imem_ens, mod_grid)
case ('gau_inc')
call create_grid_gauss(grid_setup, npets, localpet, mod_grid)
call create_grid_gauss(grid_setup, npets, localpet, imem_ens, mod_grid)
case default
call error_handler("unknown grid_setup%descriptor in setup_grid", 1)
end select
Expand Down Expand Up @@ -251,6 +252,7 @@ end subroutine read_into_fields

!> write variables from ESMF Fields into netcdf restart-like file
!! @param[in] localpet local pet
!! @param[in] imem_ens ensemble member index
!! @param[in] i_dim longitudinal dimension
!! @param[in] j_dim latitudinal dimension
!! @param[in] fname_out file name to write to
Expand All @@ -261,13 +263,13 @@ end subroutine read_into_fields
!! @param[in] fields fields to read variables into
!! @param[in] add_time_dim specify whether output file has time dimension

subroutine write_from_fields(localpet, i_dim, j_dim , fname_out, dir_out, &
subroutine write_from_fields(localpet, imem_ens, i_dim, j_dim , fname_out, dir_out, &
n_vars, n_tims, variable_list, fields, add_time_dim)

implicit none

! INTENT IN
integer, intent(in) :: localpet, i_dim, j_dim, n_vars, n_tims
integer, intent(in) :: localpet, imem_ens, i_dim, j_dim, n_vars, n_tims
character(*), intent(in) :: fname_out
character(*), intent(in) :: dir_out
character(15), dimension(n_vars), intent(in) :: variable_list
Expand All @@ -278,21 +280,18 @@ subroutine write_from_fields(localpet, i_dim, j_dim , fname_out, dir_out, &
integer :: tt, id_var, ncid, ierr, &
id_x, id_y, id_t, v, t
character(len=1) :: tchar
character(len=3) :: memchar
character(len=500) :: fname
real(esmf_kind_r8), allocatable :: array2D(:,:)
real(esmf_kind_r8), allocatable :: array_out(:,:,:,:)

do v = 1, n_vars
if (localpet == 0) print *, 'Writing ', trim(variable_list(v)), ' into field'
if (localpet == 0) print *, 'Writing ', trim(variable_list(v)), ' into field for ensemble member', imem_ens
enddo

if (localpet==0) then
allocate(array_out(n_vars, i_dim, j_dim, n_tims))
allocate(array2D(i_dim, j_dim))
else
allocate(array_out(0,0,0,0))
allocate(array2D(0,0))
end if
! All processes need properly sized arrays for FieldGather to work
allocate(array_out(n_vars, i_dim, j_dim, n_tims))
allocate(array2D(i_dim, j_dim))

do tt = 1, n_tiles

Expand All @@ -311,7 +310,8 @@ subroutine write_from_fields(localpet, i_dim, j_dim , fname_out, dir_out, &

! open file, set dimensions
write(tchar,'(i1)') tt
fname = dir_out//"/"//fname_out//".tile"//tchar//".nc"
write(memchar,'(I3)') imem_ens
fname = dir_out//"/"//fname_out//".mem"//trim(adjustl(memchar))//".tile"//tchar//".nc"

ierr = nf90_create(trim(fname), NF90_NETCDF4, ncid)
call netcdf_err(ierr, 'creating file='//trim(fname) )
Expand Down Expand Up @@ -369,13 +369,14 @@ end subroutine write_from_fields
!! @param[in] dir_fix orog fix directory
!! @param[in] localpet local pet
!! @param[in] npets total number of pets
!! @param[in] imem_ens ensemble member index
!! @param[out] fv3_grid output ESMF grid


subroutine create_grid_fv3(res_atm, dir_fix, npets, localpet, fv3_grid)
subroutine create_grid_fv3(res_atm, dir_fix, npets, localpet, imem_ens, fv3_grid)

! INTENT IN
integer, intent(in) :: npets, localpet
integer, intent(in) :: npets, localpet, imem_ens
integer, intent(in) :: res_atm
character(*), intent(in) :: dir_fix

Expand All @@ -388,7 +389,7 @@ subroutine create_grid_fv3(res_atm, dir_fix, npets, localpet, fv3_grid)
character(len=5) :: rchar
character(len=200) :: fname

if (localpet == 0) print*," creating fv3 grid for ", res_atm
if (localpet == 0) print*," creating fv3 grid for ", res_atm, " for ensemble member ", imem_ens

! pet distribution
extra = npets / n_tiles
Expand Down Expand Up @@ -418,13 +419,14 @@ end subroutine create_grid_fv3
!! @param[in] grid_setup data structure with grid details
!! @param[in] npets total number of pets
!! @param[in] localpet local pet
!! @param[in] imem_ens ensemble member index
!! @param[out] gauss_grid output ESMF grid
Comment thread
BrianCurtis-NOAA marked this conversation as resolved.

subroutine create_grid_gauss(grid_setup, npets, localpet, gauss_grid)
subroutine create_grid_gauss(grid_setup, npets, localpet, imem_ens, gauss_grid)

! INTENT IN
type(grid_setup_type), intent(in) :: grid_setup
integer, intent(in) :: npets, localpet
integer, intent(in) :: npets, localpet, imem_ens

! INTENT OUT
type(esmf_grid) :: gauss_grid
Expand All @@ -434,7 +436,7 @@ subroutine create_grid_gauss(grid_setup, npets, localpet, gauss_grid)

fname = trim(grid_setup%dir_coord)//trim(grid_setup%fname_coord)

if (localpet == 0) print*," creating gauss grid for ", trim(fname)
if (localpet == 0) print*," creating gauss grid for ", trim(fname), " for ensemble member ", imem_ens

fac = npets / n_tiles
gauss_grid = ESMF_GridCreate(filename=trim(fname), &
Expand Down
12 changes: 7 additions & 5 deletions sorc/regrid_sfc.fd/readin_setup.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,25 @@
!!
!! @param[in] unt file unit
!! @param[in] namel options: input or output
!! @param[in] nmem_ens number of ensemble members
!! @param[in] imem_ens ensemble member index
!! @param[out] grid_setup data structure with grid details
Comment thread
BrianCurtis-NOAA marked this conversation as resolved.
subroutine readin_setup(unt,namel,grid_setup)
subroutine readin_setup(unt,namel,nmem_ens,imem_ens,grid_setup)

use grids_IO, only : grid_setup_type
use utilities, only : error_handler

implicit none

! INPUTS
integer, intent(in) :: unt
integer, intent(in) :: unt, nmem_ens, imem_ens
character(*), intent(in) :: namel
! OUTPUTS
type(grid_setup_type), intent(out) :: grid_setup

character(len=7) :: gridtype
character(len=100) :: fname, fname_mask, fname_coord
character(len=100) :: dir, dir_mask, dir_coord
character(len=100) :: dir(nmem_ens), dir_mask, dir_coord
character(len=4) :: default_str="NULL"
integer :: ires, jres
integer :: ierr
Expand Down Expand Up @@ -71,7 +73,7 @@ subroutine readin_setup(unt,namel,grid_setup)

grid_setup%descriptor = gridtype

grid_setup%dir = dir
grid_setup%dir = dir(imem_ens)
grid_setup%fname = fname
grid_setup%mask_from_input = .false.

Expand All @@ -83,7 +85,7 @@ subroutine readin_setup(unt,namel,grid_setup)
grid_setup%mask_variable(1) = "vegetation_type" ! if getting from fix file
case ("gau_inc") ! gsi-output incr files only, use calculated mask
if (trim(fname_mask) == default_str) then ! if not specified, use input file
grid_setup%dir_mask = dir
grid_setup%dir_mask = dir(imem_ens)
grid_setup%fname_mask = fname
grid_setup%mask_from_input = .true.
else
Expand Down
Loading
Loading