Skip to content

Component level PIO initialization for external applications such as HAFS #158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
uturuncoglu merged 10 commits into from
Mar 5, 2021
Merged

Component level PIO initialization for external applications such as HAFS #158

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
e18514b
update internal PIO initialization
Feb 14, 2021
2a1977a
more work related to component level PIO initialization
Feb 16, 2021
7770513
add missing ESMF call to use statement
Feb 18, 2021
6e36233
remove shr_pio_mod.F90 from util/CMakeLists.txt
Feb 18, 2021
42c82d7
fix for performance issue in S2S
Feb 23, 2021
21b2d86
change default for pio_root
Feb 23, 2021
dced80b
remove extra print
Feb 23, 2021
48b9136
add extra print
Feb 23, 2021
216ab10
add support to set pio debug level and set uninitialized variable
Feb 26, 2021
3f9de91
minor fix for CDEPS data ocean
Feb 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion mediator/med.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2389,7 +2389,8 @@ subroutine DataInitialize(gcomp, rc)
!---------------------------------------
! Initialize mediator IO
!---------------------------------------
call med_io_init()
call med_io_init(gcomp, rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

!---------------------------------------
! Initialize mediator water/heat budget diags
Expand Down
351 changes: 330 additions & 21 deletions mediator/med_io_mod.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ module med_io_mod
use shr_const_mod , only : fillvalue => SHR_CONST_SPVAL
use ESMF , only : ESMF_VM, ESMF_LogWrite, ESMF_LOGMSG_INFO
use ESMF , only : ESMF_SUCCESS, ESMF_FAILURE
use ESMF , only : ESMF_VMGetCurrent, ESMF_VMGet, ESMF_VMBroadCast
use ESMF , only : ESMF_VMGetCurrent, ESMF_VMGet, ESMF_VMBroadCast, ESMF_GridComp
use NUOPC , only : NUOPC_FieldDictionaryGetEntry
use NUOPC , only : NUOPC_FieldDictionaryHasEntry
use pio , only : file_desc_t, iosystem_desc_t
Expand Down Expand Up @@ -119,41 +119,350 @@ logical function med_io_file_exists(vm, iam, filename)
end function med_io_file_exists

!===============================================================================
subroutine med_io_init()
subroutine med_io_init(gcomp, rc)

!---------------
! initialize pio
!---------------

#ifdef CESMCOUPLED
use shr_pio_mod , only : shr_pio_getiosys, shr_pio_getiotype, shr_pio_getioformat
#ifdef INTERNAL_PIO_INIT
! if CMEPS is the only component using PIO, then it needs to initialize PIO
use shr_pio_mod , only : shr_pio_init1, shr_pio_init2
#else
use pio , only : pio_init, pio_set_rearr_opts
use pio , only : PIO_64BIT_OFFSET, PIO_64BIT_DATA
use pio , only : PIO_IOTYPE_NETCDF, PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF4C, PIO_IOTYPE_NETCDF4P
use pio , only : PIO_REARR_BOX, PIO_REARR_SUBSET
use pio , only : PIO_REARR_COMM_P2P, PIO_REARR_COMM_COLL
use pio , only : PIO_REARR_COMM_FC_2D_ENABLE, PIO_REARR_COMM_FC_2D_DISABLE
use pio , only : PIO_REARR_COMM_FC_1D_COMP2IO, PIO_REARR_COMM_FC_1D_IO2COMP
use ESMF , only : ESMF_GridComp, ESMF_UtilStringUpperCase
use NUOPC, only : NUOPC_CompAttributeGet
#endif

! input/output arguments
type(ESMF_GridComp), intent(in) :: gcomp
integer , intent(out) :: rc

type(ESMF_VM) :: vm
integer :: comms(1), comps(1)
logical :: comp_iamin(1)
integer :: comp_comm_iam(1)
character(len=32) :: compLabels(1)
integer :: rc
#ifndef CESMCOUPLED
! local variables
type(ESMF_VM) :: vm
integer :: ret
integer :: comm
integer :: localPet, petCount
integer :: pio_numiotasks
integer :: pio_stride
integer :: pio_rearranger
integer :: pio_root
integer :: pio_rearr_comm_type
integer :: pio_rearr_comm_fcd
logical :: pio_rearr_comm_enable_hs_comp2io
logical :: pio_rearr_comm_enable_isend_comp2io
integer :: pio_rearr_comm_max_pend_req_comp2io
logical :: pio_rearr_comm_enable_hs_io2comp
logical :: pio_rearr_comm_enable_isend_io2comp
integer :: pio_rearr_comm_max_pend_req_io2comp
logical :: isPresent, isSet
character(len=CS) :: cvalue
character(*), parameter :: subname = '(med_io_init)'
!-------------------------------------------------------------------------------
#endif

#ifdef CESMCOUPLED
io_subsystem => shr_pio_getiosys(med_id)
pio_iotype = shr_pio_getiotype(med_id)
pio_ioformat = shr_pio_getioformat(med_id)
#else
! query VM
call ESMF_VMGetCurrent(vm=vm, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

call ESMF_VMGet(vm, mpiCommunicator=comms(1), localPet=comp_comm_iam(1), rc=rc)
call ESMF_VMGet(vm, mpiCommunicator=comm, localPet=localPet, petCount=petCount, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

comps(1) = med_id
compLabels(1) = "MED"
comp_iamin(1) = .true.
! query component specific PIO attributes
! pio_netcdf_format
call NUOPC_CompAttributeGet(gcomp, name='pio_netcdf_format', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
if (isPresent .and. isSet) then
cvalue = ESMF_UtilStringUpperCase(cvalue)
if (trim(cvalue) .eq. 'CLASSIC') then
pio_ioformat = 0
else if (trim(cvalue) .eq. '64BIT_OFFSET') then
pio_ioformat = PIO_64BIT_OFFSET
else if (trim(cvalue) .eq. '64BIT_DATA') then
pio_ioformat = PIO_64BIT_DATA
else
call ESMF_LogWrite(trim(subname)//': need to provide valid option for pio_ioformat (CLASSIC|64BIT_OFFSET|64BIT_DATA)', ESMF_LOGMSG_INFO)
rc = ESMF_FAILURE
return
end if
else
cvalue = '64BIT_OFFSET'
pio_ioformat = PIO_64BIT_OFFSET
end if
if (localPet == 0) write(logunit,*) trim(subname), ' : pio_netcdf_format = ', trim(cvalue), pio_ioformat

! pio_typename
call NUOPC_CompAttributeGet(gcomp, name='pio_typename', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
if (isPresent .and. isSet) then
cvalue = ESMF_UtilStringUpperCase(cvalue)
if (trim(cvalue) .eq. 'NETCDF') then
pio_iotype = PIO_IOTYPE_NETCDF
else if (trim(cvalue) .eq. 'PNETCDF') then
pio_iotype = PIO_IOTYPE_PNETCDF
else if (trim(cvalue) .eq. 'NETCDF4C') then
pio_iotype = PIO_IOTYPE_NETCDF4C
else if (trim(cvalue) .eq. 'NETCDF4P') then
pio_iotype = PIO_IOTYPE_NETCDF4P
else
call ESMF_LogWrite(trim(subname)//': need to provide valid option for pio_typename (NETCDF|PNETCDF|NETCDF4C|NETCDF4P)', ESMF_LOGMSG_INFO)
rc = ESMF_FAILURE
return
end if
else
cvalue = 'NETCDF'
pio_iotype = PIO_IOTYPE_NETCDF
end if
if (localPet == 0) write(logunit,*) trim(subname), ' : pio_typename = ', trim(cvalue), pio_iotype

! pio_root
call NUOPC_CompAttributeGet(gcomp, name='pio_root', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
if (isPresent .and. isSet) then
read(cvalue,*) pio_root
if (pio_root < 0) then
pio_root = 1
endif
pio_root = min(pio_root, petCount-1)
else
pio_root = 1
end if
if (localPet == 0) write(logunit,*) trim(subname), ' : pio_root = ', pio_root

call shr_pio_init1(1, "pio_in", comms(1))
call shr_pio_init2(comps, compLabels, comp_iamin, comms, comp_comm_iam)
#endif
! pio_stride
call NUOPC_CompAttributeGet(gcomp, name='pio_stride', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
if (isPresent .and. isSet) then
read(cvalue,*) pio_stride
else
pio_stride = -99
end if
if (localPet == 0) write(logunit,*) trim(subname), ' : pio_stride = ', pio_stride

io_subsystem => shr_pio_getiosys(med_id)
pio_iotype = shr_pio_getiotype(med_id)
pio_ioformat = shr_pio_getioformat(med_id)
! pio_numiotasks
call NUOPC_CompAttributeGet(gcomp, name='pio_numiotasks', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
if (isPresent .and. isSet) then
read(cvalue,*) pio_numiotasks
else
pio_numiotasks = -99
end if
if (localPet == 0) write(logunit,*) trim(subname), ' : pio_numiotasks = ', pio_numiotasks

! check for parallel IO, it requires at least two io pes
if (petCount > 1 .and. pio_numiotasks == 1 .and. &
(pio_iotype .eq. PIO_IOTYPE_PNETCDF .or. pio_iotype .eq. PIO_IOTYPE_NETCDF4P)) then
pio_numiotasks = 2
pio_stride = min(pio_stride, petCount/2)
if (localPet == 0) then
write(logunit,*) ' parallel io requires at least two io pes - following parameters are updated:'
write(logunit,*) trim(subname), ' : pio_stride = ', pio_stride
write(logunit,*) trim(subname), ' : pio_numiotasks = ', pio_numiotasks
end if
endif

! check/set/correct io pio parameters
if (pio_stride > 0 .and. pio_numiotasks < 0) then
pio_numiotasks = max(1, petCount/pio_stride)
if (localPet == 0) write(logunit,*) trim(subname), ' : update pio_numiotasks = ', pio_numiotasks
else if(pio_numiotasks > 0 .and. pio_stride < 0) then
pio_stride = max(1, petCount/pio_numiotasks)
if (localPet == 0) write(logunit,*) trim(subname), ' : update pio_stride = ', pio_stride
else if(pio_numiotasks < 0 .and. pio_stride < 0) then
pio_stride = max(1,petCount/4)
pio_numiotasks = max(1,petCount/pio_stride)
if (localPet == 0) write(logunit,*) trim(subname), ' : update pio_numiotasks = ', pio_numiotasks
if (localPet == 0) write(logunit,*) trim(subname), ' : update pio_stride = ', pio_stride
end if
if (pio_stride == 1) then
pio_root = 0
endif

if (pio_root + (pio_stride)*(pio_numiotasks-1) >= petCount .or. &
pio_stride <= 0 .or. pio_numiotasks <= 0 .or. pio_root < 0 .or. pio_root > petCount-1) then
if (petCount < 100) then
pio_stride = max(1, petCount/4)
else if(petCount < 1000) then
pio_stride = max(1, petCount/8)
else
pio_stride = max(1, petCount/16)
end if
if(pio_stride > 1) then
pio_numiotasks = petCount/pio_stride
pio_root = min(1, petCount-1)
else
pio_numiotasks = petCount
pio_root = 0
end if
if (localPet == 0) then
write(logunit,*) 'pio_stride, iotasks or root out of bounds - resetting to defaults:'
write(logunit,*) trim(subname), ' : pio_root = ', pio_root
write(logunit,*) trim(subname), ' : pio_stride = ', pio_stride
write(logunit,*) trim(subname), ' : pio_numiotasks = ', pio_numiotasks
end if
end if

! pio_rearranger
call NUOPC_CompAttributeGet(gcomp, name='pio_rearranger', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return
if (isPresent .and. isSet) then
cvalue = ESMF_UtilStringUpperCase(cvalue)
if (trim(cvalue) .eq. 'BOX') then
pio_rearranger = PIO_REARR_BOX
else if (trim(cvalue) .eq. 'SUBSET') then
pio_rearranger = PIO_REARR_SUBSET
else
call ESMF_LogWrite(trim(subname)//': need to provide valid option for pio_rearranger (BOX|SUBSET)', ESMF_LOGMSG_INFO)
rc = ESMF_FAILURE
return
end if
else
cvalue = 'BOX'
pio_rearranger = PIO_REARR_BOX
end if
if (localPet == 0) write(logunit,*) trim(subname), ' : pio_rearranger = ', trim(cvalue), pio_rearranger

! init PIO
allocate(io_subsystem)
if (localPet == 0) write(logunit,*) trim(subname),' calling pio init'
call pio_init(localPet, comm, pio_numiotasks, 0, pio_stride, pio_rearranger, io_subsystem, base=pio_root)

! query shared PIO rearranger attributes
! pio_rearr_comm_type
call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_type', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

if (isPresent .and. isSet) then
cvalue = ESMF_UtilStringUpperCase(cvalue)
if (trim(cvalue) .eq. 'P2P') then
pio_rearr_comm_type = PIO_REARR_COMM_P2P
else if (trim(cvalue) .eq. 'COLL') then
pio_rearr_comm_type = PIO_REARR_COMM_COLL
else
call ESMF_LogWrite(trim(subname)//': need to provide valid option for pio_rearr_comm_type (P2P|COLL)', ESMF_LOGMSG_INFO)
rc = ESMF_FAILURE
return
end if
else
cvalue = 'P2P'
pio_rearr_comm_type = PIO_REARR_COMM_P2P
end if
if (localPet == 0) write(logunit,*) trim(subname), ' : pio_rearr_comm_type = ', trim(cvalue), pio_rearr_comm_type

! pio_rearr_comm_fcd
call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_fcd', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

if (isPresent .and. isSet) then
cvalue = ESMF_UtilStringUpperCase(cvalue)
if (trim(cvalue) .eq. '2DENABLE') then
pio_rearr_comm_fcd = PIO_REARR_COMM_FC_2D_ENABLE
else if (trim(cvalue) .eq. 'IO2COMP') then
pio_rearr_comm_fcd = PIO_REARR_COMM_FC_1D_IO2COMP
else if (trim(cvalue) .eq. 'COMP2IO') then
pio_rearr_comm_fcd = PIO_REARR_COMM_FC_1D_COMP2IO
else if (trim(cvalue) .eq. '2DDISABLE') then
pio_rearr_comm_fcd = PIO_REARR_COMM_FC_2D_DISABLE
else
call ESMF_LogWrite(trim(subname)//': need to provide valid option for pio_rearr_comm_fcd (2DENABLE|IO2COMP|COMP2IO|2DDISABLE)', ESMF_LOGMSG_INFO)
rc = ESMF_FAILURE
return
end if
else
cvalue = '2DENABLE'
pio_rearr_comm_fcd = PIO_REARR_COMM_FC_2D_ENABLE
end if
if (localPet == 0) write(logunit,*) trim(subname), ' : pio_rearr_comm_fcd = ', trim(cvalue), pio_rearr_comm_fcd

! pio_rearr_comm_enable_hs_comp2io
call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_enable_hs_comp2io', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

if (isPresent .and. isSet) then
read(cvalue,*) pio_rearr_comm_enable_hs_comp2io
else
pio_rearr_comm_enable_hs_comp2io = .true.
end if

! pio_rearr_comm_enable_isend_comp2io
call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_enable_isend_comp2io', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

if (isPresent .and. isSet) then
read(cvalue,*) pio_rearr_comm_enable_isend_comp2io
else
pio_rearr_comm_enable_isend_comp2io = .false.
end if

! pio_rearr_comm_max_pend_req_comp2io
call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_max_pend_req_comp2io', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

if (isPresent .and. isSet) then
read(cvalue,*) pio_rearr_comm_max_pend_req_comp2io
else
pio_rearr_comm_max_pend_req_comp2io = 0
end if

! pio_rearr_comm_enable_hs_io2comp
call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_enable_hs_io2comp', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

if (isPresent .and. isSet) then
read(cvalue,*) pio_rearr_comm_enable_hs_io2comp
else
pio_rearr_comm_enable_hs_io2comp = .false.
end if

! pio_rearr_comm_enable_isend_io2comp
call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_enable_isend_io2comp', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

if (isPresent .and. isSet) then
read(cvalue,*) pio_rearr_comm_enable_isend_io2comp
else
pio_rearr_comm_enable_isend_io2comp = .true.
end if

! pio_rearr_comm_max_pend_req_io2comp
call NUOPC_CompAttributeGet(gcomp, name='pio_rearr_comm_max_pend_req_io2comp', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
if (ChkErr(rc,__LINE__,u_FILE_u)) return

if (isPresent .and. isSet) then
read(cvalue,*) pio_rearr_comm_max_pend_req_io2comp
else
pio_rearr_comm_max_pend_req_io2comp = 64
end if

! print out PIO rearranger parameters
if (localPet == 0) then
write(logunit,*) trim(subname), ' : pio_rearr_comm_enable_hs_comp2io = ', pio_rearr_comm_enable_hs_comp2io
write(logunit,*) trim(subname), ' : pio_rearr_comm_enable_isend_comp2io = ', pio_rearr_comm_enable_isend_comp2io
write(logunit,*) trim(subname), ' : pio_rearr_comm_max_pend_req_comp2io = ', pio_rearr_comm_max_pend_req_comp2io
write(logunit,*) trim(subname), ' : pio_rearr_comm_enable_hs_io2comp = ', pio_rearr_comm_enable_hs_io2comp
write(logunit,*) trim(subname), ' : pio_rearr_comm_enable_isend_io2comp = ', pio_rearr_comm_enable_isend_io2comp
write(logunit,*) trim(subname), ' : pio_rearr_comm_max_pend_req_io2comp = ', pio_rearr_comm_max_pend_req_io2comp
end if

! set PIO rearranger options
if (localPet == 0) write(logunit,*) trim(subname),' calling pio_set_rearr_opts'
ret = pio_set_rearr_opts(io_subsystem, pio_rearr_comm_type, pio_rearr_comm_fcd, &
pio_rearr_comm_enable_hs_comp2io, pio_rearr_comm_enable_isend_comp2io, &
pio_rearr_comm_max_pend_req_comp2io, &
pio_rearr_comm_enable_hs_io2comp, pio_rearr_comm_enable_isend_io2comp, &
pio_rearr_comm_max_pend_req_io2comp)
#endif

end subroutine med_io_init

Expand Down
2 changes: 1 addition & 1 deletion util/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@ project(CMEPS_share Fortran)
include(ExternalProject)

add_library(cmeps_share shr_abort_mod.F90 shr_flux_mod.F90 shr_log_mod.F90 shr_mpi_mod.F90 shr_sys_mod.F90
glc_elevclass_mod.F90 perf_mod.F90 shr_const_mod.F90 shr_kind_mod.F90 shr_mem_mod.F90 shr_pio_mod.F90)
glc_elevclass_mod.F90 perf_mod.F90 shr_const_mod.F90 shr_kind_mod.F90 shr_mem_mod.F90)

target_include_directories (cmeps_share PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${ESMF_F90COMPILEPATHS} ${PIO_Fortran_INCLUDE_DIRS})
Loading