From d0855879bf2a50dba169a4cbbfdbd98d5ce8ba0e Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Fri, 10 Feb 2023 02:28:08 +0000 Subject: [PATCH 1/7] Add driver/fvGFS/fv_ufs_restart_io.F90 to support writing restart files on wrt grid comp --- CMakeLists.txt | 1 + driver/fvGFS/fv_ufs_restart_io.F90 | 305 +++++++++++++++++++++++++++++ 2 files changed, 306 insertions(+) create mode 100644 driver/fvGFS/fv_ufs_restart_io.F90 diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d5189794..83f8ac5f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,7 @@ list(APPEND tools_srcs_extra list(APPEND driver_srcs driver/fvGFS/DYCORE_typedefs.F90 driver/fvGFS/fv_nggps_diag.F90 + driver/fvGFS/fv_ufs_restart_io.F90 driver/fvGFS/atmosphere.F90) list(APPEND fv3_srcs ${model_srcs} diff --git a/driver/fvGFS/fv_ufs_restart_io.F90 b/driver/fvGFS/fv_ufs_restart_io.F90 new file mode 100644 index 000000000..fa6ef248f --- /dev/null +++ b/driver/fvGFS/fv_ufs_restart_io.F90 @@ -0,0 +1,305 @@ +module fv_ufs_restart_io_mod + + use mpp_mod, only: mpp_pe, mpp_root_pe, mpp_chksum, mpp_pe + use fv_arrays_mod, only: fv_atmos_type + use tracer_manager_mod, only: tr_get_tracer_names=>get_tracer_names, & + get_tracer_names, get_number_tracers, & + set_tracer_profile, & + get_tracer_index + use field_manager_mod, only: MODEL_ATMOS + + implicit none + + public fv_ufs_restart_register + + public fv_srf_wnd_restart_output + public fv_srf_wnd_restart_bundle_setup + + public fv_tracer_restart_output + public fv_tracer_restart_bundle_setup + + private + + ! fv_core.res + + ! fv_srf_wnd.res + integer :: nvar2d_srf_wnd + real, allocatable, target, dimension(:,:,:) :: srf_wnd_var2 + character(32), allocatable,dimension(:) :: srf_wnd_var2_names + + ! fv_tracers + integer :: nvar3d_tracers + integer :: ntprog, ntdiag, tracers_zsize + real, allocatable, target, dimension(:,:,:,:) :: tracers_var3 + character(32), allocatable,dimension(:) :: tracers_var3_names + + contains + + subroutine fv_ufs_restart_register (Atm) + + ! this subroutine must allocate all data buffers and set the variable names + ! for restart bundles + + implicit none + + type(fv_atmos_type), intent(inout) :: Atm + + integer :: isc, iec, jsc, jec, nx, ny + integer :: num, nt + character(len=64) :: tracer_name + + isc = Atm%bd%isc + iec = Atm%bd%iec + jsc = Atm%bd%jsc + jec = Atm%bd%jec + nx = (iec - isc + 1) + ny = (jec - jsc + 1) + + if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_ufs_restart_register ' + + ! srf_wnd + nvar2d_srf_wnd = 2 + allocate (srf_wnd_var2(nx,ny,nvar2d_srf_wnd), srf_wnd_var2_names(nvar2d_srf_wnd)) + srf_wnd_var2_names(1) = 'u_srf' + srf_wnd_var2_names(2) = 'v_srf' + + ! tracers + ntprog = size(Atm%q,4) + ntdiag = size(Atm%qdiag,4) + nvar3d_tracers = ntprog+ntdiag + tracers_zsize = size(Atm%q,3) + allocate (tracers_var3(nx,ny,tracers_zsize,nvar3d_tracers), tracers_var3_names(nvar3d_tracers)) + + do nt = 1, ntprog + call get_tracer_names(MODEL_ATMOS, nt, tracer_name) + tracers_var3_names(nt) = tracer_name + enddo + do nt = ntprog+1, nvar3d_tracers + call get_tracer_names(MODEL_ATMOS, nt, tracer_name) + tracers_var3_names(nt) = tracer_name + enddo + + end subroutine fv_ufs_restart_register + + subroutine fv_srf_wnd_restart_output(Atm) + + implicit none + + type(fv_atmos_type), intent(inout) :: Atm + + if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_srf_wnd_restart_output ' + + srf_wnd_var2(:,:,1) = Atm%u_srf + srf_wnd_var2(:,:,2) = Atm%v_srf + + end subroutine fv_srf_wnd_restart_output + + subroutine fv_tracer_restart_output(Atm) + + implicit none + + type(fv_atmos_type), intent(inout) :: Atm + + integer :: nt + integer :: isc, iec, jsc, jec + + if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_tracer_restart_output ' + + isc = Atm%bd%isc + iec = Atm%bd%iec + jsc = Atm%bd%jsc + jec = Atm%bd%jec + + do nt = 1, ntprog + tracers_var3(:,:,:,nt) = Atm%q(isc:iec,jsc:jec,:,nt) + enddo + do nt = ntprog+1, nvar3d_tracers + tracers_var3(:,:,:,nt) = Atm%qdiag(isc:iec,jsc:jec,:,nt) + enddo + + end subroutine fv_tracer_restart_output + + subroutine fv_srf_wnd_restart_bundle_setup(bundle, grid, rc) +! +!------------------------------------------------------------- +!*** set esmf bundle for phys restart fields +!------------------------------------------------------------ +! + use esmf + + implicit none + + type(ESMF_FieldBundle),intent(inout) :: bundle + type(ESMF_Grid),intent(inout) :: grid + integer,intent(out) :: rc + +!*** local variables + integer i, j, k, n + character(128) :: bdl_name + type(ESMF_Field) :: field + character(128) :: outputfile + integer :: num + real,dimension(:,:),pointer :: temp_r2d + real,dimension(:,:,:),pointer :: temp_r3d + + if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_srf_wnd_restart_bundle_setup ' + + call ESMF_FieldBundleGet(bundle, name=bdl_name,rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + outputfile = trim(bdl_name) + + do num = 1,nvar2d_srf_wnd + temp_r2d => srf_wnd_var2(:,:,num) + call create_2d_field_and_add_to_bundle(temp_r2d, trim(srf_wnd_var2_names(num)), trim(outputfile), grid, bundle) + enddo + + end subroutine fv_srf_wnd_restart_bundle_setup + + subroutine fv_tracer_restart_bundle_setup(bundle, grid, rc) +! +!------------------------------------------------------------- +!*** set esmf bundle for phys restart fields +!------------------------------------------------------------ +! + use esmf + + implicit none + + type(ESMF_FieldBundle),intent(inout) :: bundle + type(ESMF_Grid),intent(inout) :: grid + integer,intent(out) :: rc + +!*** local variables + integer i, j, k, n + character(128) :: bdl_name + type(ESMF_Field) :: field + character(128) :: outputfile + integer :: num + real,dimension(:,:),pointer :: temp_r2d + real,dimension(:,:,:),pointer :: temp_r3d + + if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_tracer_restart_bundle_setup ' + + call ESMF_FieldBundleGet(bundle, name=bdl_name,rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + outputfile = trim(bdl_name) + + do num = 1,nvar3d_tracers + temp_r3d => tracers_var3(:,:,:,num) + call create_3d_field_and_add_to_bundle(temp_r3d, trim(tracers_var3_names(num)), "zaxis_1", tracers_zsize, trim(outputfile), grid, bundle) + enddo + + end subroutine fv_tracer_restart_bundle_setup + + subroutine create_2d_field_and_add_to_bundle(temp_r2d, field_name, outputfile, grid, bundle) + + use esmf + + implicit none + + real, dimension(:,:), pointer, intent(in) :: temp_r2d + character(len=*), intent(in) :: field_name + character(len=*), intent(in) :: outputfile + type(ESMF_Grid), intent(in) :: grid + type(ESMF_FieldBundle), intent(inout) :: bundle + + type(ESMF_Field) :: field + + integer :: rc, i + + field = ESMF_FieldCreate(grid, temp_r2d, datacopyflag=ESMF_DATACOPY_REFERENCE, & + name=trim(field_name), indexFlag=ESMF_INDEX_DELOCAL, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU,line=__LINE__, file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_AttributeAdd(field, convention="NetCDF", purpose="FV3", attrList=(/"output_file"/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_AttributeSet(field, convention="NetCDF", purpose="FV3", name='output_file', value=trim(outputfile), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_FieldBundleAdd(bundle, (/field/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) call ESMF_Finalize(endflag=ESMF_END_ABORT) + + end subroutine create_2d_field_and_add_to_bundle + + subroutine create_3d_field_and_add_to_bundle(temp_r3d, field_name, axis_name, num_levels, outputfile, grid, bundle) + + use esmf + + implicit none + + real, dimension(:,:,:), pointer, intent(in) :: temp_r3d + character(len=*), intent(in) :: field_name + character(len=*), intent(in) :: axis_name + integer, intent(in) :: num_levels + character(len=*), intent(in) :: outputfile + type(ESMF_Grid), intent(in) :: grid + type(ESMF_FieldBundle), intent(inout) :: bundle + + type(ESMF_Field) :: field + + integer :: rc, i + + ! if(mpp_pe() == mpp_root_pe()) write(0,*)'in add 3d field to bundle ', trim(field_name) + field = ESMF_FieldCreate(grid, temp_r3d, datacopyflag=ESMF_DATACOPY_REFERENCE, & + name=trim(field_name), indexFlag=ESMF_INDEX_DELOCAL, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU,line=__LINE__, file=__FILE__)) & + call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_AttributeAdd(field, convention="NetCDF", purpose="FV3", attrList=(/"output_file"/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call ESMF_AttributeSet(field, convention="NetCDF", purpose="FV3", name='output_file', value=trim(outputfile), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) call ESMF_Finalize(endflag=ESMF_END_ABORT) + + call add_zaxis_to_field(field, axis_name, num_levels) + + call ESMF_FieldBundleAdd(bundle, (/field/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) call ESMF_Finalize(endflag=ESMF_END_ABORT) + + end subroutine create_3d_field_and_add_to_bundle + + subroutine add_zaxis_to_field(field, axis_name, num_levels) + + use esmf + + implicit none + + type(ESMF_Field), intent(inout) :: field + character(len=*), intent(in) :: axis_name + integer, intent(in) :: num_levels + + real(8), allocatable, dimension(:) :: buffer + integer :: rc, i + + call ESMF_AttributeAdd(field, convention="NetCDF", purpose="FV3", & + attrList=(/"ESMF:ungridded_dim_labels"/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_AttributeSet(field, convention="NetCDF", purpose="FV3", & + name="ESMF:ungridded_dim_labels", valueList=(/trim(axis_name)/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeAdd(field, convention="NetCDF", purpose="FV3-dim", & + attrList=(/trim(axis_name),trim(axis_name)//":cartesian_axis"/), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + allocate( buffer(num_levels) ) + do i=1, num_levels + buffer(i)=i + end do + call ESMF_AttributeSet(field, convention="NetCDF", purpose="FV3-dim", & + name=trim(axis_name), valueList=buffer, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + deallocate(buffer) + + call ESMF_AttributeSet(field, convention="NetCDF", purpose="FV3-dim", & + name=trim(axis_name)//"cartesian_axis", value="Z", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + end subroutine add_zaxis_to_field + + +end module fv_ufs_restart_io_mod From c9ca0770d18b68ca045662df621fedd5085a6b81 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 15 Feb 2023 20:19:56 +0000 Subject: [PATCH 2/7] Add fv_core support to fv_ufs_restart_io.F90,compute checksums --- driver/fvGFS/fv_ufs_restart_io.F90 | 310 +++++++++++++++++++++++++---- 1 file changed, 269 insertions(+), 41 deletions(-) diff --git a/driver/fvGFS/fv_ufs_restart_io.F90 b/driver/fvGFS/fv_ufs_restart_io.F90 index fa6ef248f..d8221d3dc 100644 --- a/driver/fvGFS/fv_ufs_restart_io.F90 +++ b/driver/fvGFS/fv_ufs_restart_io.F90 @@ -1,53 +1,64 @@ module fv_ufs_restart_io_mod + use esmf use mpp_mod, only: mpp_pe, mpp_root_pe, mpp_chksum, mpp_pe use fv_arrays_mod, only: fv_atmos_type - use tracer_manager_mod, only: tr_get_tracer_names=>get_tracer_names, & - get_tracer_names, get_number_tracers, & - set_tracer_profile, & - get_tracer_index - use field_manager_mod, only: MODEL_ATMOS + use tracer_manager_mod, only: get_tracer_names + use field_manager_mod, only: MODEL_ATMOS + use atmosphere_mod, only: atmosphere_resolution implicit none - public fv_ufs_restart_register + public fv_dyn_restart_register + public fv_dyn_restart_output - public fv_srf_wnd_restart_output + public fv_core_restart_bundle_setup public fv_srf_wnd_restart_bundle_setup - - public fv_tracer_restart_output public fv_tracer_restart_bundle_setup private ! fv_core.res + integer :: nvar3d_core_center, core_zsize + real, allocatable, target, dimension(:,:,:) :: core_center_var2 + real, allocatable, target, dimension(:,:,:,:) :: core_center_var3, core_east_var3, core_south_var3 + character(len=32), allocatable,dimension(:) :: core_center_var2_names + character(len=32), allocatable,dimension(:) :: core_center_var3_names, core_east_var3_names, core_south_var3_names ! fv_srf_wnd.res integer :: nvar2d_srf_wnd real, allocatable, target, dimension(:,:,:) :: srf_wnd_var2 - character(32), allocatable,dimension(:) :: srf_wnd_var2_names + character(len=32), allocatable, dimension(:) :: srf_wnd_var2_names + character(len=32), allocatable, dimension(:) :: srf_wnd_var2_cksum ! fv_tracers integer :: nvar3d_tracers integer :: ntprog, ntdiag, tracers_zsize real, allocatable, target, dimension(:,:,:,:) :: tracers_var3 - character(32), allocatable,dimension(:) :: tracers_var3_names + character(len=32), allocatable, dimension(:) :: tracers_var3_names + character(len=32), allocatable, dimension(:) :: tracers_var3_cksum + + type(ESMF_FieldBundle) :: core_bundle, srf_wnd_bundle, tracer_bundle contains - subroutine fv_ufs_restart_register (Atm) + subroutine fv_dyn_restart_register (Atm) ! this subroutine must allocate all data buffers and set the variable names ! for restart bundles + ! must be consistent with fv_io_register_restart in atmos_cubed_sphere/tools/fv_io.F90 implicit none type(fv_atmos_type), intent(inout) :: Atm - integer :: isc, iec, jsc, jec, nx, ny - integer :: num, nt + integer :: isc, iec, jsc, jec, nx, ny, nx_c, ny_c, nx_s, ny_s, nx_e, ny_e + integer :: nlon, nlat, mlon, mlat + integer :: num, nt, n character(len=64) :: tracer_name + ! if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_dyn_restart_register' + isc = Atm%bd%isc iec = Atm%bd%iec jsc = Atm%bd%jsc @@ -55,11 +66,84 @@ subroutine fv_ufs_restart_register (Atm) nx = (iec - isc + 1) ny = (jec - jsc + 1) - if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_ufs_restart_register ' + ! core + ! 'u', 'v' + nvar3d_core_center = 0 + if (.not.atm%flagstruct%hydrostatic) then + if (atm%flagstruct%make_nh) then ! hydrostatic restarts dont have these variables + nvar3d_core_center = nvar3d_core_center + 2 ! 'w', 'dz' + if ( atm%flagstruct%hybrid_z ) then + nvar3d_core_center = nvar3d_core_center + 1 ! 'ze0' + endif + else !the restart file has the non-hydrostatic variables + nvar3d_core_center = nvar3d_core_center + 2 ! 'w', 'dz' + if ( atm%flagstruct%hybrid_z ) then + nvar3d_core_center = nvar3d_core_center + 1 ! 'ze0' + endif + endif + endif + nvar3d_core_center = nvar3d_core_center + 2 ! 't', 'delp' + !--- include agrid winds in restarts for use in data assimilation + if (atm%flagstruct%agrid_vel_rst) then + nvar3d_core_center = nvar3d_core_center + 2 ! 'ua', 'va' + endif + core_zsize = size(Atm%u,3) + + call atmosphere_resolution (mlon, mlat, global=.true.) + nx_c = nx + ny_c = ny + if (iec == mlon) then + ! we are on task at the 'east' edge of the cubed sphere face or regional domain + ! corner arrays should have one extra element in 'i' direction + nx_c = nx + 1 + end if + if (jec == mlat) then + ! we are on task at the 'north' edge of the cubed sphere face or regional domain + ! corner arrays should have one extra element in 'j' direction + ny_c = ny + 1 + end if + + allocate(core_south_var3 (nx , ny_c,core_zsize,1)) + allocate(core_east_var3 (nx_c, ny ,core_zsize,1)) + allocate(core_center_var3 (nx , ny ,core_zsize,nvar3d_core_center)) + allocate(core_center_var2 (nx , ny ,1)) + + allocate(core_south_var3_names (1)) + allocate(core_east_var3_names (1)) + allocate(core_center_var3_names (nvar3d_core_center)) + allocate(core_center_var2_names (1)) + + core_south_var3_names(1) = 'u' + core_east_var3_names(1) = 'v' + n = 1 + if (.not.atm%flagstruct%hydrostatic) then + if (atm%flagstruct%make_nh) then ! hydrostatic restarts dont have these variables + core_center_var3_names(n) = 'W'; n=n+1 + core_center_var3_names(n) = 'DZ'; n=n+1 + if ( atm%flagstruct%hybrid_z ) then + core_center_var3_names(n) = 'ze0'; n=n+1 + endif + else !the restart file has the non-hydrostatic variables + core_center_var3_names(n) = 'W'; n=n+1 + core_center_var3_names(n) = 'DZ'; n=n+1 + if ( atm%flagstruct%hybrid_z ) then + core_center_var3_names(n) = 'ze0'; n=n+1 + endif + endif + endif + core_center_var3_names(n) = 'T'; n=n+1 + core_center_var3_names(n) = 'delp'; n=n+1 + !--- include agrid winds in restarts for use in data assimilation + if (atm%flagstruct%agrid_vel_rst) then + core_center_var3_names(n) = 'ua'; n=n+1 + core_center_var3_names(n) = 'va'; n=n+1 + endif + + core_center_var2_names(1) = 'phis' ! srf_wnd nvar2d_srf_wnd = 2 - allocate (srf_wnd_var2(nx,ny,nvar2d_srf_wnd), srf_wnd_var2_names(nvar2d_srf_wnd)) + allocate (srf_wnd_var2(nx,ny,nvar2d_srf_wnd), srf_wnd_var2_names(nvar2d_srf_wnd), srf_wnd_var2_cksum(nvar2d_srf_wnd)) srf_wnd_var2_names(1) = 'u_srf' srf_wnd_var2_names(2) = 'v_srf' @@ -68,7 +152,7 @@ subroutine fv_ufs_restart_register (Atm) ntdiag = size(Atm%qdiag,4) nvar3d_tracers = ntprog+ntdiag tracers_zsize = size(Atm%q,3) - allocate (tracers_var3(nx,ny,tracers_zsize,nvar3d_tracers), tracers_var3_names(nvar3d_tracers)) + allocate (tracers_var3(nx,ny,tracers_zsize,nvar3d_tracers), tracers_var3_names(nvar3d_tracers), tracers_var3_cksum(nvar3d_tracers)) do nt = 1, ntprog call get_tracer_names(MODEL_ATMOS, nt, tracer_name) @@ -79,45 +163,144 @@ subroutine fv_ufs_restart_register (Atm) tracers_var3_names(nt) = tracer_name enddo - end subroutine fv_ufs_restart_register + end subroutine fv_dyn_restart_register - subroutine fv_srf_wnd_restart_output(Atm) + subroutine fv_dyn_restart_output(Atm) implicit none type(fv_atmos_type), intent(inout) :: Atm - if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_srf_wnd_restart_output ' - - srf_wnd_var2(:,:,1) = Atm%u_srf - srf_wnd_var2(:,:,2) = Atm%v_srf - - end subroutine fv_srf_wnd_restart_output - - subroutine fv_tracer_restart_output(Atm) - - implicit none - - type(fv_atmos_type), intent(inout) :: Atm - - integer :: nt - integer :: isc, iec, jsc, jec + integer :: isc, iec, jsc, jec, nx, ny + integer :: n, nt + integer :: mlon, mlat, nx_c, ny_c - if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_tracer_restart_output ' + ! if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_dyn_restart_output' isc = Atm%bd%isc iec = Atm%bd%iec jsc = Atm%bd%jsc jec = Atm%bd%jec + nx = (iec - isc + 1) + ny = (jec - jsc + 1) + + call atmosphere_resolution (mlon, mlat, global=.true.) + nx_c = nx + ny_c = ny + if (iec == mlon) then + ! we are on task at the 'east' edge of the cubed sphere face or regional domain + ! corner arrays should have one extra element in 'i' direction + nx_c = nx + 1 + end if + if (jec == mlat) then + ! we are on task at the 'north' edge of the cubed sphere face or regional domain + ! corner arrays should have one extra element in 'j' direction + ny_c = ny + 1 + end if + + ! ---- fv_core.res + + core_south_var3(:,:,:,1) = Atm%u(isc:iec,jsc:(jsc+ny_c-1),:) + core_east_var3(:,:,:,1) = Atm%v(isc:(isc+nx_c-1),jsc:jec,:) + + n = 1 + if (.not.atm%flagstruct%hydrostatic) then + if (atm%flagstruct%make_nh) then ! hydrostatic restarts dont have these variables + ! core_center_var3(n) = 'w'; n=n+1 + ! core_center_var3(n) = 'dz'; n=n+1 + ! if ( atm%flagstruct%hybrid_z ) then + ! core_center_var3(n) = 'ze0'; n=n+1 + ! endif + else !the restart file has the non-hydrostatic variables + core_center_var3(:,:,:,n) = Atm%w(isc:iec,jsc:jec,:); n=n+1 + core_center_var3(:,:,:,n) = Atm%delz; n=n+1 + if ( atm%flagstruct%hybrid_z ) then + core_center_var3(:,:,:,n) = Atm%ze0; n=n+1 + endif + endif + endif + core_center_var3(:,:,:,n) = Atm%pt(isc:iec,jsc:jec,:); n=n+1 + core_center_var3(:,:,:,n) = Atm%delp(isc:iec,jsc:jec,:); n=n+1 + !--- include agrid winds in restarts for use in data assimilation + if (atm%flagstruct%agrid_vel_rst) then + core_center_var3(:,:,:,n) = Atm%ua(isc:iec,jsc:jec,:); n=n+1 + core_center_var3(:,:,:,n) = Atm%va(isc:iec,jsc:jec,:); n=n+1 + endif + + core_center_var2(:,:,1) = Atm%phis(isc:iec,jsc:jec) + + ! ---- fv_srf_wnd.res + srf_wnd_var2(:,:,1) = Atm%u_srf + srf_wnd_var2(:,:,2) = Atm%v_srf + write(srf_wnd_var2_cksum(1),'(Z16)') mpp_chksum(Atm%u_srf) + write(srf_wnd_var2_cksum(2),'(Z16)') mpp_chksum(Atm%v_srf) + call update_chksums(srf_wnd_bundle, 2, srf_wnd_var2_names, srf_wnd_var2_cksum) + + ! ---- fv_tracer_res + ! computing checksum for arrays that have halos is not straightforward as simply calling + ! mpp_chksum on full array or computational domain sub-array, unfortunately. do nt = 1, ntprog tracers_var3(:,:,:,nt) = Atm%q(isc:iec,jsc:jec,:,nt) + ! write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%q(:,:,:,nt)) + ! write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%q(isc:iec,jsc:jec,:,nt)) enddo do nt = ntprog+1, nvar3d_tracers tracers_var3(:,:,:,nt) = Atm%qdiag(isc:iec,jsc:jec,:,nt) + ! write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%qdiag(:,:,:,nt)) + ! write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%qdiag(isc:iec,jsc:jec,:,nt)) enddo + ! call update_chksums(tracer_bundle, nvar3d_tracers, tracers_var3_names, tracers_var3_cksum) + + end subroutine fv_dyn_restart_output - end subroutine fv_tracer_restart_output + subroutine fv_core_restart_bundle_setup(bundle, grid, rc) +! +!------------------------------------------------------------- +!*** set esmf bundle for phys restart fields +!------------------------------------------------------------ +! + use esmf + + implicit none + + type(ESMF_FieldBundle),intent(inout) :: bundle + type(ESMF_Grid),intent(inout) :: grid + integer,intent(out) :: rc + +!*** local variables + integer i, j, k, n + character(128) :: bdl_name + type(ESMF_Field) :: field + character(128) :: outputfile + integer :: num + real,dimension(:,:),pointer :: temp_r2d + real,dimension(:,:,:),pointer :: temp_r3d + + ! if(mpp_pe() == mpp_root_pe()) write(0,*) 'in fv_core_restart_bundle_setup' + + core_bundle = bundle + + call ESMF_FieldBundleGet(bundle, name=bdl_name,rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + outputfile = trim(bdl_name) + + temp_r3d => core_south_var3(:,:,:,1) + call create_3d_field_and_add_to_bundle(temp_r3d, trim(core_south_var3_names(1)), "zaxis_1", core_zsize, trim(outputfile), grid, bundle, staggerloc=ESMF_STAGGERLOC_EDGE2) + + temp_r3d => core_east_var3(:,:,:,1) + call create_3d_field_and_add_to_bundle(temp_r3d, trim(core_east_var3_names(1)), "zaxis_1", core_zsize, trim(outputfile), grid, bundle, staggerloc=ESMF_STAGGERLOC_EDGE1) + + do n = 1, nvar3d_core_center + temp_r3d => core_center_var3(:,:,:,n) + call create_3d_field_and_add_to_bundle(temp_r3d, trim(core_center_var3_names(n)), "zaxis_1", core_zsize, trim(outputfile), grid, bundle) + end do + + temp_r2d => core_center_var2(:,:,1) + call create_2d_field_and_add_to_bundle(temp_r2d, trim(core_center_var2_names(1)), trim(outputfile), grid, bundle) + + end subroutine fv_core_restart_bundle_setup subroutine fv_srf_wnd_restart_bundle_setup(bundle, grid, rc) ! @@ -142,7 +325,9 @@ subroutine fv_srf_wnd_restart_bundle_setup(bundle, grid, rc) real,dimension(:,:),pointer :: temp_r2d real,dimension(:,:,:),pointer :: temp_r3d - if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_srf_wnd_restart_bundle_setup ' + ! if(mpp_pe() == mpp_root_pe()) write(0,*) 'in fv_srf_wnd_restart_bundle_setup' + + srf_wnd_bundle = bundle call ESMF_FieldBundleGet(bundle, name=bdl_name,rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -179,7 +364,9 @@ subroutine fv_tracer_restart_bundle_setup(bundle, grid, rc) real,dimension(:,:),pointer :: temp_r2d real,dimension(:,:,:),pointer :: temp_r3d - if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_tracer_restart_bundle_setup ' + ! if(mpp_pe() == mpp_root_pe()) write(0,*) 'in fv_tracer_restart_bundle_setup' + + tracer_bundle = bundle call ESMF_FieldBundleGet(bundle, name=bdl_name,rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -225,7 +412,7 @@ subroutine create_2d_field_and_add_to_bundle(temp_r2d, field_name, outputfile, g end subroutine create_2d_field_and_add_to_bundle - subroutine create_3d_field_and_add_to_bundle(temp_r3d, field_name, axis_name, num_levels, outputfile, grid, bundle) + subroutine create_3d_field_and_add_to_bundle(temp_r3d, field_name, axis_name, num_levels, outputfile, grid, bundle, staggerloc) use esmf @@ -238,14 +425,20 @@ subroutine create_3d_field_and_add_to_bundle(temp_r3d, field_name, axis_name, nu character(len=*), intent(in) :: outputfile type(ESMF_Grid), intent(in) :: grid type(ESMF_FieldBundle), intent(inout) :: bundle + type(ESMF_StaggerLoc),optional, intent(in) :: staggerloc type(ESMF_Field) :: field + type(ESMF_StaggerLoc) :: stagger integer :: rc, i - ! if(mpp_pe() == mpp_root_pe()) write(0,*)'in add 3d field to bundle ', trim(field_name) + stagger = ESMF_STAGGERLOC_CENTER + if(present(staggerloc)) then + stagger = staggerloc + end if + field = ESMF_FieldCreate(grid, temp_r3d, datacopyflag=ESMF_DATACOPY_REFERENCE, & - name=trim(field_name), indexFlag=ESMF_INDEX_DELOCAL, rc=rc) + name=trim(field_name), indexFlag=ESMF_INDEX_DELOCAL, staggerloc=stagger, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU,line=__LINE__, file=__FILE__)) & call ESMF_Finalize(endflag=ESMF_END_ABORT) @@ -301,5 +494,40 @@ subroutine add_zaxis_to_field(field, axis_name, num_levels) end subroutine add_zaxis_to_field +#if 0 + subroutine update_chksums(bundle, var_name, var_cksum) + + type(ESMF_FieldBundle), intent(inout) :: bundle + character(len=*), intent(in) :: var_names, var_cksums + + integer :: rc + type(ESMF_Field) :: field + + call ESMF_FieldBundleGet(bundle, var_names field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeSet(field, convention="NetCDF", purpose="FV3", name="checksum", value=var_cksums, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + end subroutine update_chksums +#endif + subroutine update_chksums(bundle, nvar, var_names, var_cksums) + + type(ESMF_FieldBundle), intent(inout) :: bundle + integer, intent(in) :: nvar + character(len=*), dimension(:), intent(in) :: var_names, var_cksums + + integer :: n, rc + type(ESMF_Field) :: field + + do n = 1, nvar + call ESMF_FieldBundleGet(bundle, var_names(n), field=field, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + + call ESMF_AttributeSet(field, convention="NetCDF", purpose="FV3", name="checksum", value=var_cksums(n), rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + end do + + end subroutine update_chksums end module fv_ufs_restart_io_mod From a1d6a6bc97fde3b54e8aa76086034aea8b141632 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 15 Feb 2023 21:40:09 +0000 Subject: [PATCH 3/7] Add subroutine that writes fv_core.res.nc file (ak, bk arrays). --- driver/fvGFS/fv_ufs_restart_io.F90 | 68 ++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/driver/fvGFS/fv_ufs_restart_io.F90 b/driver/fvGFS/fv_ufs_restart_io.F90 index d8221d3dc..c0f9eb82f 100644 --- a/driver/fvGFS/fv_ufs_restart_io.F90 +++ b/driver/fvGFS/fv_ufs_restart_io.F90 @@ -1,7 +1,7 @@ module fv_ufs_restart_io_mod use esmf - use mpp_mod, only: mpp_pe, mpp_root_pe, mpp_chksum, mpp_pe + use mpp_mod, only: mpp_pe, mpp_root_pe, mpp_chksum, mpp_npes, mpp_get_current_pelist use fv_arrays_mod, only: fv_atmos_type use tracer_manager_mod, only: get_tracer_names use field_manager_mod, only: MODEL_ATMOS @@ -165,11 +165,12 @@ subroutine fv_dyn_restart_register (Atm) end subroutine fv_dyn_restart_register - subroutine fv_dyn_restart_output(Atm) + subroutine fv_dyn_restart_output(Atm, timestamp) implicit none - type(fv_atmos_type), intent(inout) :: Atm + type(fv_atmos_type), intent(in) :: Atm + character(len=*), intent(in) :: timestamp integer :: isc, iec, jsc, jec, nx, ny integer :: n, nt @@ -252,6 +253,9 @@ subroutine fv_dyn_restart_output(Atm) enddo ! call update_chksums(tracer_bundle, nvar3d_tracers, tracers_var3_names, tracers_var3_cksum) + ! Instead of creating yet another esmf bundle just to write Atm%ak and Atm%bk, write them here synchronously + call write_ak_bk(Atm, timestamp) + end subroutine fv_dyn_restart_output subroutine fv_core_restart_bundle_setup(bundle, grid, rc) @@ -489,7 +493,7 @@ subroutine add_zaxis_to_field(field, axis_name, num_levels) deallocate(buffer) call ESMF_AttributeSet(field, convention="NetCDF", purpose="FV3-dim", & - name=trim(axis_name)//"cartesian_axis", value="Z", rc=rc) + name=trim(axis_name)//"axis", value="Z", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return end subroutine add_zaxis_to_field @@ -530,4 +534,60 @@ subroutine update_chksums(bundle, nvar, var_names, var_cksums) end subroutine update_chksums + subroutine write_ak_bk(Atm, timestamp) + + + use fms2_io_mod, only: FmsNetcdfFile_t, & + register_restart_field, register_axis, unlimited, & + open_file, write_restart, & + close_file, register_field, write_data, & + register_variable_attribute + implicit none + + type(fv_atmos_type), intent(in) :: Atm + character(len=*), intent(in) :: timestamp + + character(len=120) :: fname + integer, allocatable, dimension(:) :: pes + type(FmsNetcdfFile_t) :: Fv_restart + logical :: Fv_restart_is_open + integer, dimension(:), allocatable :: buffer + character(len=8), dimension(2) :: dim_names_2d + integer :: j + + dim_names_2d(1) = "xaxis_1" + dim_names_2d(2) = "Time" + + fname = 'RESTART_new/'//trim(timestamp)//'.fv_core.res.nc' + + allocate(pes(mpp_npes())) + call mpp_get_current_pelist(pes) + + Fv_restart_is_open = open_file(Fv_restart, fname, "overwrite", is_restart=.true., pelist=pes) + + call register_axis(Fv_restart, "xaxis_1", size(Atm%ak(:), 1)) + call register_axis(Fv_restart, "Time", unlimited) + call register_field(Fv_restart, "xaxis_1", "double", (/"xaxis_1"/)) + call register_variable_attribute(Fv_restart,"xaxis_1", "axis", "X", str_len=1) + if (allocated(buffer)) deallocate(buffer) + allocate(buffer(size(Atm%ak(:), 1))) + do j = 1, size(Atm%ak(:), 1) + buffer(j) = j + end do + call write_data(Fv_restart, "xaxis_1", buffer) + deallocate(buffer) + call register_field(Fv_restart, "Time", "double", (/"Time"/)) + call register_variable_attribute(Fv_restart, dim_names_2d(2), "cartesian_axis", "T", str_len=1) + call register_variable_attribute(Fv_restart, dim_names_2d(2), "units", "time level", str_len=len("time level")) + call register_variable_attribute(Fv_restart, dim_names_2d(2), "long_name", dim_names_2d(2), str_len=len(dim_names_2d(2))) + call write_data(Fv_restart, "Time", 1) + call register_restart_field (Fv_restart, 'ak', Atm%ak(:), dim_names_2d) + call register_restart_field (Fv_restart, 'bk', Atm%bk(:), dim_names_2d) + + call write_restart(Fv_restart) + call close_file(Fv_restart) + deallocate(pes) + + end subroutine write_ak_bk + end module fv_ufs_restart_io_mod From 67db148617082575869dd5840ed8574f27b9347d Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 21 Feb 2023 15:18:03 +0000 Subject: [PATCH 4/7] More updates to compute restart file checksums --- driver/fvGFS/fv_ufs_restart_io.F90 | 46 ++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/driver/fvGFS/fv_ufs_restart_io.F90 b/driver/fvGFS/fv_ufs_restart_io.F90 index c0f9eb82f..795b08fdb 100644 --- a/driver/fvGFS/fv_ufs_restart_io.F90 +++ b/driver/fvGFS/fv_ufs_restart_io.F90 @@ -22,8 +22,8 @@ module fv_ufs_restart_io_mod integer :: nvar3d_core_center, core_zsize real, allocatable, target, dimension(:,:,:) :: core_center_var2 real, allocatable, target, dimension(:,:,:,:) :: core_center_var3, core_east_var3, core_south_var3 - character(len=32), allocatable,dimension(:) :: core_center_var2_names - character(len=32), allocatable,dimension(:) :: core_center_var3_names, core_east_var3_names, core_south_var3_names + character(len=32), allocatable,dimension(:) :: core_center_var2_names, core_center_var3_names, core_east_var3_names, core_south_var3_names + character(len=32), allocatable,dimension(:) :: core_center_var2_cksum, core_center_var3_cksum, core_east_var3_cksum, core_south_var3_cksum ! fv_srf_wnd.res integer :: nvar2d_srf_wnd @@ -108,11 +108,21 @@ subroutine fv_dyn_restart_register (Atm) allocate(core_center_var3 (nx , ny ,core_zsize,nvar3d_core_center)) allocate(core_center_var2 (nx , ny ,1)) + core_south_var3 = 0.0 + core_east_var3 = 0.0 + core_center_var3 = 0.0 + core_center_var2 = 0.0 + allocate(core_south_var3_names (1)) allocate(core_east_var3_names (1)) allocate(core_center_var3_names (nvar3d_core_center)) allocate(core_center_var2_names (1)) + allocate(core_south_var3_cksum (1)) + allocate(core_east_var3_cksum (1)) + allocate(core_center_var3_cksum (nvar3d_core_center)) + allocate(core_center_var2_cksum (1)) + core_south_var3_names(1) = 'u' core_east_var3_names(1) = 'v' n = 1 @@ -202,7 +212,9 @@ subroutine fv_dyn_restart_output(Atm, timestamp) ! ---- fv_core.res core_south_var3(:,:,:,1) = Atm%u(isc:iec,jsc:(jsc+ny_c-1),:) - core_east_var3(:,:,:,1) = Atm%v(isc:(isc+nx_c-1),jsc:jec,:) + core_east_var3 (:,:,:,1) = Atm%v(isc:(isc+nx_c-1),jsc:jec,:) + write(core_south_var3_cksum(1),'(Z16)') mpp_chksum(Atm%u(isc:iec,jsc:(jsc+ny_c-1),:)) + write(core_east_var3_cksum (1),'(Z16)') mpp_chksum(Atm%v(isc:(isc+nx_c-1),jsc:jec,:)) n = 1 if (.not.atm%flagstruct%hydrostatic) then @@ -228,7 +240,17 @@ subroutine fv_dyn_restart_output(Atm, timestamp) core_center_var3(:,:,:,n) = Atm%va(isc:iec,jsc:jec,:); n=n+1 endif + do n = 1, nvar3d_core_center + write(core_center_var3_cksum(n),'(Z16)') mpp_chksum(core_center_var3(:,:,:,n)) + enddo + core_center_var2(:,:,1) = Atm%phis(isc:iec,jsc:jec) + write(core_center_var2_cksum(1),'(Z16)') mpp_chksum(core_center_var2(:,:,1)) + + call update_chksums(core_bundle, 1, core_south_var3_names, core_south_var3_cksum) + call update_chksums(core_bundle, 1, core_east_var3_names, core_east_var3_cksum) + call update_chksums(core_bundle, nvar3d_core_center, core_center_var3_names, core_center_var3_cksum) + call update_chksums(core_bundle, 1, core_center_var2_names, core_center_var2_cksum) ! ---- fv_srf_wnd.res srf_wnd_var2(:,:,1) = Atm%u_srf @@ -238,20 +260,15 @@ subroutine fv_dyn_restart_output(Atm, timestamp) call update_chksums(srf_wnd_bundle, 2, srf_wnd_var2_names, srf_wnd_var2_cksum) ! ---- fv_tracer_res - - ! computing checksum for arrays that have halos is not straightforward as simply calling - ! mpp_chksum on full array or computational domain sub-array, unfortunately. do nt = 1, ntprog tracers_var3(:,:,:,nt) = Atm%q(isc:iec,jsc:jec,:,nt) - ! write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%q(:,:,:,nt)) - ! write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%q(isc:iec,jsc:jec,:,nt)) + write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%q(isc:iec,jsc:jec,:,nt)) enddo do nt = ntprog+1, nvar3d_tracers tracers_var3(:,:,:,nt) = Atm%qdiag(isc:iec,jsc:jec,:,nt) - ! write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%qdiag(:,:,:,nt)) - ! write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%qdiag(isc:iec,jsc:jec,:,nt)) + write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%qdiag(isc:iec,jsc:jec,:,nt)) enddo - ! call update_chksums(tracer_bundle, nvar3d_tracers, tracers_var3_names, tracers_var3_cksum) + call update_chksums(tracer_bundle, nvar3d_tracers, tracers_var3_names, tracers_var3_cksum) ! Instead of creating yet another esmf bundle just to write Atm%ak and Atm%bk, write them here synchronously call write_ak_bk(Atm, timestamp) @@ -558,7 +575,12 @@ subroutine write_ak_bk(Atm, timestamp) dim_names_2d(1) = "xaxis_1" dim_names_2d(2) = "Time" - fname = 'RESTART_new/'//trim(timestamp)//'.fv_core.res.nc' + ! fname = 'RESTART_new/'//trim(timestamp)//'.fv_core.res.nc' + if (trim(timestamp) == '') then + fname = 'RESTART/'//'fv_core.res.nc' + else + fname = 'RESTART/'//trim(timestamp)//'.fv_core.res.nc' + endif allocate(pes(mpp_npes())) call mpp_get_current_pelist(pes) From 4e533a4da60abd53e75da83727f821a5abf15a9d Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Thu, 23 Feb 2023 14:04:08 +0000 Subject: [PATCH 5/7] Move checksum computation to write grid component --- driver/fvGFS/fv_ufs_restart_io.F90 | 68 +----------------------------- 1 file changed, 2 insertions(+), 66 deletions(-) diff --git a/driver/fvGFS/fv_ufs_restart_io.F90 b/driver/fvGFS/fv_ufs_restart_io.F90 index 795b08fdb..b14ed1093 100644 --- a/driver/fvGFS/fv_ufs_restart_io.F90 +++ b/driver/fvGFS/fv_ufs_restart_io.F90 @@ -23,20 +23,17 @@ module fv_ufs_restart_io_mod real, allocatable, target, dimension(:,:,:) :: core_center_var2 real, allocatable, target, dimension(:,:,:,:) :: core_center_var3, core_east_var3, core_south_var3 character(len=32), allocatable,dimension(:) :: core_center_var2_names, core_center_var3_names, core_east_var3_names, core_south_var3_names - character(len=32), allocatable,dimension(:) :: core_center_var2_cksum, core_center_var3_cksum, core_east_var3_cksum, core_south_var3_cksum ! fv_srf_wnd.res integer :: nvar2d_srf_wnd real, allocatable, target, dimension(:,:,:) :: srf_wnd_var2 character(len=32), allocatable, dimension(:) :: srf_wnd_var2_names - character(len=32), allocatable, dimension(:) :: srf_wnd_var2_cksum ! fv_tracers integer :: nvar3d_tracers integer :: ntprog, ntdiag, tracers_zsize real, allocatable, target, dimension(:,:,:,:) :: tracers_var3 character(len=32), allocatable, dimension(:) :: tracers_var3_names - character(len=32), allocatable, dimension(:) :: tracers_var3_cksum type(ESMF_FieldBundle) :: core_bundle, srf_wnd_bundle, tracer_bundle @@ -118,11 +115,6 @@ subroutine fv_dyn_restart_register (Atm) allocate(core_center_var3_names (nvar3d_core_center)) allocate(core_center_var2_names (1)) - allocate(core_south_var3_cksum (1)) - allocate(core_east_var3_cksum (1)) - allocate(core_center_var3_cksum (nvar3d_core_center)) - allocate(core_center_var2_cksum (1)) - core_south_var3_names(1) = 'u' core_east_var3_names(1) = 'v' n = 1 @@ -153,7 +145,7 @@ subroutine fv_dyn_restart_register (Atm) ! srf_wnd nvar2d_srf_wnd = 2 - allocate (srf_wnd_var2(nx,ny,nvar2d_srf_wnd), srf_wnd_var2_names(nvar2d_srf_wnd), srf_wnd_var2_cksum(nvar2d_srf_wnd)) + allocate (srf_wnd_var2(nx,ny,nvar2d_srf_wnd), srf_wnd_var2_names(nvar2d_srf_wnd)) srf_wnd_var2_names(1) = 'u_srf' srf_wnd_var2_names(2) = 'v_srf' @@ -162,7 +154,7 @@ subroutine fv_dyn_restart_register (Atm) ntdiag = size(Atm%qdiag,4) nvar3d_tracers = ntprog+ntdiag tracers_zsize = size(Atm%q,3) - allocate (tracers_var3(nx,ny,tracers_zsize,nvar3d_tracers), tracers_var3_names(nvar3d_tracers), tracers_var3_cksum(nvar3d_tracers)) + allocate (tracers_var3(nx,ny,tracers_zsize,nvar3d_tracers), tracers_var3_names(nvar3d_tracers)) do nt = 1, ntprog call get_tracer_names(MODEL_ATMOS, nt, tracer_name) @@ -186,8 +178,6 @@ subroutine fv_dyn_restart_output(Atm, timestamp) integer :: n, nt integer :: mlon, mlat, nx_c, ny_c - ! if(mpp_pe() == mpp_root_pe()) write(0,*) ' in fv_dyn_restart_output' - isc = Atm%bd%isc iec = Atm%bd%iec jsc = Atm%bd%jsc @@ -213,8 +203,6 @@ subroutine fv_dyn_restart_output(Atm, timestamp) core_south_var3(:,:,:,1) = Atm%u(isc:iec,jsc:(jsc+ny_c-1),:) core_east_var3 (:,:,:,1) = Atm%v(isc:(isc+nx_c-1),jsc:jec,:) - write(core_south_var3_cksum(1),'(Z16)') mpp_chksum(Atm%u(isc:iec,jsc:(jsc+ny_c-1),:)) - write(core_east_var3_cksum (1),'(Z16)') mpp_chksum(Atm%v(isc:(isc+nx_c-1),jsc:jec,:)) n = 1 if (.not.atm%flagstruct%hydrostatic) then @@ -240,35 +228,19 @@ subroutine fv_dyn_restart_output(Atm, timestamp) core_center_var3(:,:,:,n) = Atm%va(isc:iec,jsc:jec,:); n=n+1 endif - do n = 1, nvar3d_core_center - write(core_center_var3_cksum(n),'(Z16)') mpp_chksum(core_center_var3(:,:,:,n)) - enddo - core_center_var2(:,:,1) = Atm%phis(isc:iec,jsc:jec) - write(core_center_var2_cksum(1),'(Z16)') mpp_chksum(core_center_var2(:,:,1)) - - call update_chksums(core_bundle, 1, core_south_var3_names, core_south_var3_cksum) - call update_chksums(core_bundle, 1, core_east_var3_names, core_east_var3_cksum) - call update_chksums(core_bundle, nvar3d_core_center, core_center_var3_names, core_center_var3_cksum) - call update_chksums(core_bundle, 1, core_center_var2_names, core_center_var2_cksum) ! ---- fv_srf_wnd.res srf_wnd_var2(:,:,1) = Atm%u_srf srf_wnd_var2(:,:,2) = Atm%v_srf - write(srf_wnd_var2_cksum(1),'(Z16)') mpp_chksum(Atm%u_srf) - write(srf_wnd_var2_cksum(2),'(Z16)') mpp_chksum(Atm%v_srf) - call update_chksums(srf_wnd_bundle, 2, srf_wnd_var2_names, srf_wnd_var2_cksum) ! ---- fv_tracer_res do nt = 1, ntprog tracers_var3(:,:,:,nt) = Atm%q(isc:iec,jsc:jec,:,nt) - write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%q(isc:iec,jsc:jec,:,nt)) enddo do nt = ntprog+1, nvar3d_tracers tracers_var3(:,:,:,nt) = Atm%qdiag(isc:iec,jsc:jec,:,nt) - write(tracers_var3_cksum(nt),'(Z16)') mpp_chksum(Atm%qdiag(isc:iec,jsc:jec,:,nt)) enddo - call update_chksums(tracer_bundle, nvar3d_tracers, tracers_var3_names, tracers_var3_cksum) ! Instead of creating yet another esmf bundle just to write Atm%ak and Atm%bk, write them here synchronously call write_ak_bk(Atm, timestamp) @@ -515,42 +487,6 @@ subroutine add_zaxis_to_field(field, axis_name, num_levels) end subroutine add_zaxis_to_field -#if 0 - subroutine update_chksums(bundle, var_name, var_cksum) - - type(ESMF_FieldBundle), intent(inout) :: bundle - character(len=*), intent(in) :: var_names, var_cksums - - integer :: rc - type(ESMF_Field) :: field - - call ESMF_FieldBundleGet(bundle, var_names field=field, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeSet(field, convention="NetCDF", purpose="FV3", name="checksum", value=var_cksums, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - end subroutine update_chksums -#endif - subroutine update_chksums(bundle, nvar, var_names, var_cksums) - - type(ESMF_FieldBundle), intent(inout) :: bundle - integer, intent(in) :: nvar - character(len=*), dimension(:), intent(in) :: var_names, var_cksums - - integer :: n, rc - type(ESMF_Field) :: field - - do n = 1, nvar - call ESMF_FieldBundleGet(bundle, var_names(n), field=field, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - - call ESMF_AttributeSet(field, convention="NetCDF", purpose="FV3", name="checksum", value=var_cksums(n), rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - end do - - end subroutine update_chksums - subroutine write_ak_bk(Atm, timestamp) From 83f596debedf6ae996144726de5659f53d9439c0 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 7 Mar 2023 15:20:19 +0000 Subject: [PATCH 6/7] Rename staggered variables (south -> north). Cleanup. --- driver/fvGFS/fv_ufs_restart_io.F90 | 35 ++++++++++++------------------ 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/driver/fvGFS/fv_ufs_restart_io.F90 b/driver/fvGFS/fv_ufs_restart_io.F90 index b14ed1093..204573fbd 100644 --- a/driver/fvGFS/fv_ufs_restart_io.F90 +++ b/driver/fvGFS/fv_ufs_restart_io.F90 @@ -21,8 +21,8 @@ module fv_ufs_restart_io_mod ! fv_core.res integer :: nvar3d_core_center, core_zsize real, allocatable, target, dimension(:,:,:) :: core_center_var2 - real, allocatable, target, dimension(:,:,:,:) :: core_center_var3, core_east_var3, core_south_var3 - character(len=32), allocatable,dimension(:) :: core_center_var2_names, core_center_var3_names, core_east_var3_names, core_south_var3_names + real, allocatable, target, dimension(:,:,:,:) :: core_center_var3, core_east_var3, core_north_var3 + character(len=32), allocatable,dimension(:) :: core_center_var2_names, core_center_var3_names, core_east_var3_names, core_north_var3_names ! fv_srf_wnd.res integer :: nvar2d_srf_wnd @@ -100,22 +100,22 @@ subroutine fv_dyn_restart_register (Atm) ny_c = ny + 1 end if - allocate(core_south_var3 (nx , ny_c,core_zsize,1)) + allocate(core_north_var3 (nx , ny_c,core_zsize,1)) allocate(core_east_var3 (nx_c, ny ,core_zsize,1)) allocate(core_center_var3 (nx , ny ,core_zsize,nvar3d_core_center)) allocate(core_center_var2 (nx , ny ,1)) - core_south_var3 = 0.0 + core_north_var3 = 0.0 core_east_var3 = 0.0 core_center_var3 = 0.0 core_center_var2 = 0.0 - allocate(core_south_var3_names (1)) + allocate(core_north_var3_names (1)) allocate(core_east_var3_names (1)) allocate(core_center_var3_names (nvar3d_core_center)) allocate(core_center_var2_names (1)) - core_south_var3_names(1) = 'u' + core_north_var3_names(1) = 'u' core_east_var3_names(1) = 'v' n = 1 if (.not.atm%flagstruct%hydrostatic) then @@ -201,7 +201,7 @@ subroutine fv_dyn_restart_output(Atm, timestamp) ! ---- fv_core.res - core_south_var3(:,:,:,1) = Atm%u(isc:iec,jsc:(jsc+ny_c-1),:) + core_north_var3(:,:,:,1) = Atm%u(isc:iec,jsc:(jsc+ny_c-1),:) core_east_var3 (:,:,:,1) = Atm%v(isc:(isc+nx_c-1),jsc:jec,:) n = 1 @@ -250,7 +250,7 @@ end subroutine fv_dyn_restart_output subroutine fv_core_restart_bundle_setup(bundle, grid, rc) ! !------------------------------------------------------------- -!*** set esmf bundle for phys restart fields +!*** set esmf bundle for fv_core restart fields !------------------------------------------------------------ ! use esmf @@ -270,8 +270,6 @@ subroutine fv_core_restart_bundle_setup(bundle, grid, rc) real,dimension(:,:),pointer :: temp_r2d real,dimension(:,:,:),pointer :: temp_r3d - ! if(mpp_pe() == mpp_root_pe()) write(0,*) 'in fv_core_restart_bundle_setup' - core_bundle = bundle call ESMF_FieldBundleGet(bundle, name=bdl_name,rc=rc) @@ -279,8 +277,8 @@ subroutine fv_core_restart_bundle_setup(bundle, grid, rc) outputfile = trim(bdl_name) - temp_r3d => core_south_var3(:,:,:,1) - call create_3d_field_and_add_to_bundle(temp_r3d, trim(core_south_var3_names(1)), "zaxis_1", core_zsize, trim(outputfile), grid, bundle, staggerloc=ESMF_STAGGERLOC_EDGE2) + temp_r3d => core_north_var3(:,:,:,1) + call create_3d_field_and_add_to_bundle(temp_r3d, trim(core_north_var3_names(1)), "zaxis_1", core_zsize, trim(outputfile), grid, bundle, staggerloc=ESMF_STAGGERLOC_EDGE2) temp_r3d => core_east_var3(:,:,:,1) call create_3d_field_and_add_to_bundle(temp_r3d, trim(core_east_var3_names(1)), "zaxis_1", core_zsize, trim(outputfile), grid, bundle, staggerloc=ESMF_STAGGERLOC_EDGE1) @@ -298,7 +296,7 @@ end subroutine fv_core_restart_bundle_setup subroutine fv_srf_wnd_restart_bundle_setup(bundle, grid, rc) ! !------------------------------------------------------------- -!*** set esmf bundle for phys restart fields +!*** set esmf bundle for fv_srf_wnd restart fields !------------------------------------------------------------ ! use esmf @@ -318,8 +316,6 @@ subroutine fv_srf_wnd_restart_bundle_setup(bundle, grid, rc) real,dimension(:,:),pointer :: temp_r2d real,dimension(:,:,:),pointer :: temp_r3d - ! if(mpp_pe() == mpp_root_pe()) write(0,*) 'in fv_srf_wnd_restart_bundle_setup' - srf_wnd_bundle = bundle call ESMF_FieldBundleGet(bundle, name=bdl_name,rc=rc) @@ -337,7 +333,7 @@ end subroutine fv_srf_wnd_restart_bundle_setup subroutine fv_tracer_restart_bundle_setup(bundle, grid, rc) ! !------------------------------------------------------------- -!*** set esmf bundle for phys restart fields +!*** set esmf bundle for fv_tracer restart fields !------------------------------------------------------------ ! use esmf @@ -357,8 +353,6 @@ subroutine fv_tracer_restart_bundle_setup(bundle, grid, rc) real,dimension(:,:),pointer :: temp_r2d real,dimension(:,:,:),pointer :: temp_r3d - ! if(mpp_pe() == mpp_root_pe()) write(0,*) 'in fv_tracer_restart_bundle_setup' - tracer_bundle = bundle call ESMF_FieldBundleGet(bundle, name=bdl_name,rc=rc) @@ -511,11 +505,10 @@ subroutine write_ak_bk(Atm, timestamp) dim_names_2d(1) = "xaxis_1" dim_names_2d(2) = "Time" - ! fname = 'RESTART_new/'//trim(timestamp)//'.fv_core.res.nc' if (trim(timestamp) == '') then - fname = 'RESTART/'//'fv_core.res.nc' + fname = 'RESTART/'//'fv_core.res.nc' else - fname = 'RESTART/'//trim(timestamp)//'.fv_core.res.nc' + fname = 'RESTART/'//trim(timestamp)//'.fv_core.res.nc' endif allocate(pes(mpp_npes())) From 437253cd1e37f2836d00cab6f02a2bbbd8f491d1 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Mon, 13 Mar 2023 15:14:40 +0000 Subject: [PATCH 7/7] Cleanup fv_ufs_restart_io --- driver/fvGFS/fv_ufs_restart_io.F90 | 49 ++++++++---------------------- 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/driver/fvGFS/fv_ufs_restart_io.F90 b/driver/fvGFS/fv_ufs_restart_io.F90 index 204573fbd..fb554b832 100644 --- a/driver/fvGFS/fv_ufs_restart_io.F90 +++ b/driver/fvGFS/fv_ufs_restart_io.F90 @@ -67,17 +67,10 @@ subroutine fv_dyn_restart_register (Atm) ! 'u', 'v' nvar3d_core_center = 0 if (.not.atm%flagstruct%hydrostatic) then - if (atm%flagstruct%make_nh) then ! hydrostatic restarts dont have these variables - nvar3d_core_center = nvar3d_core_center + 2 ! 'w', 'dz' - if ( atm%flagstruct%hybrid_z ) then - nvar3d_core_center = nvar3d_core_center + 1 ! 'ze0' - endif - else !the restart file has the non-hydrostatic variables - nvar3d_core_center = nvar3d_core_center + 2 ! 'w', 'dz' - if ( atm%flagstruct%hybrid_z ) then - nvar3d_core_center = nvar3d_core_center + 1 ! 'ze0' - endif - endif + nvar3d_core_center = nvar3d_core_center + 2 ! 'w', 'dz' + if (atm%flagstruct%hybrid_z) then + nvar3d_core_center = nvar3d_core_center + 1 ! 'ze0' + endif endif nvar3d_core_center = nvar3d_core_center + 2 ! 't', 'delp' !--- include agrid winds in restarts for use in data assimilation @@ -119,19 +112,11 @@ subroutine fv_dyn_restart_register (Atm) core_east_var3_names(1) = 'v' n = 1 if (.not.atm%flagstruct%hydrostatic) then - if (atm%flagstruct%make_nh) then ! hydrostatic restarts dont have these variables - core_center_var3_names(n) = 'W'; n=n+1 - core_center_var3_names(n) = 'DZ'; n=n+1 - if ( atm%flagstruct%hybrid_z ) then - core_center_var3_names(n) = 'ze0'; n=n+1 - endif - else !the restart file has the non-hydrostatic variables - core_center_var3_names(n) = 'W'; n=n+1 - core_center_var3_names(n) = 'DZ'; n=n+1 - if ( atm%flagstruct%hybrid_z ) then - core_center_var3_names(n) = 'ze0'; n=n+1 - endif - endif + core_center_var3_names(n) = 'W'; n=n+1 + core_center_var3_names(n) = 'DZ'; n=n+1 + if (atm%flagstruct%hybrid_z) then + core_center_var3_names(n) = 'ze0'; n=n+1 + endif endif core_center_var3_names(n) = 'T'; n=n+1 core_center_var3_names(n) = 'delp'; n=n+1 @@ -206,18 +191,10 @@ subroutine fv_dyn_restart_output(Atm, timestamp) n = 1 if (.not.atm%flagstruct%hydrostatic) then - if (atm%flagstruct%make_nh) then ! hydrostatic restarts dont have these variables - ! core_center_var3(n) = 'w'; n=n+1 - ! core_center_var3(n) = 'dz'; n=n+1 - ! if ( atm%flagstruct%hybrid_z ) then - ! core_center_var3(n) = 'ze0'; n=n+1 - ! endif - else !the restart file has the non-hydrostatic variables - core_center_var3(:,:,:,n) = Atm%w(isc:iec,jsc:jec,:); n=n+1 - core_center_var3(:,:,:,n) = Atm%delz; n=n+1 - if ( atm%flagstruct%hybrid_z ) then - core_center_var3(:,:,:,n) = Atm%ze0; n=n+1 - endif + core_center_var3(:,:,:,n) = Atm%w(isc:iec,jsc:jec,:); n=n+1 + core_center_var3(:,:,:,n) = Atm%delz; n=n+1 + if (atm%flagstruct%hybrid_z) then + core_center_var3(:,:,:,n) = Atm%ze0; n=n+1 endif endif core_center_var3(:,:,:,n) = Atm%pt(isc:iec,jsc:jec,:); n=n+1