Skip to content

redo multiinstance support #337

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jedwards4b merged 4 commits into from
Jan 11, 2023
Merged

redo multiinstance support #337

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6ad71d9
redo multiinstance support
jedwards4b Jan 9, 2023
d17cd89
move timer init function
jedwards4b Jan 11, 2023
72c8b2c
Merge branch 'main' into esmf_multidriver
jedwards4b Jan 11, 2023
c7fec3b
fix merge issues
jedwards4b Jan 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 37 additions & 15 deletions cesm/driver/ensemble_driver.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,17 +79,17 @@ subroutine SetModelServices(ensemble_driver, rc)
use ESMF , only : ESMF_CalendarSetDefault
use ESMF , only : ESMF_CALKIND_NOLEAP, ESMF_CALKIND_GREGORIAN
use NUOPC , only : NUOPC_CompAttributeGet, NUOPC_CompAttributeSet, NUOPC_CompAttributeAdd
use NUOPC_Driver , only : NUOPC_DriverAddComp
use NUOPC_Driver , only : NUOPC_DriverAddComp, NUOPC_DriverGetComp
use esm , only : ESMSetServices => SetServices, ReadAttributes
use esm_time_mod , only : esm_time_clockInit

use perf_mod , only : t_startf, t_stopf, t_initf
! input/output variables
type(ESMF_GridComp) :: ensemble_driver
integer, intent(out) :: rc

! local variables
type(ESMF_VM) :: vm
type(ESMF_GridComp) :: driver, gridcomptmp
type(ESMF_GridComp) :: driver
type(ESMF_Config) :: config
integer :: n
integer, pointer :: petList(:)
Expand All @@ -102,6 +102,7 @@ subroutine SetModelServices(ensemble_driver, rc)
integer :: inst
integer :: number_of_members
integer :: ntasks_per_member
integer :: Global_Comm
character(CL) :: start_type ! Type of startup
character(len=7) :: drvrinst
character(len=5) :: inst_suffix
Expand All @@ -116,10 +117,21 @@ subroutine SetModelServices(ensemble_driver, rc)

rc = ESMF_SUCCESS
call ESMF_LogWrite(trim(subname)//": called", ESMF_LOGMSG_INFO)

call ESMF_GridCompGet(ensemble_driver, config=config, vm=vm, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

call ESMF_VMGet(vm, localPet=localPet, mpiCommunicator=global_comm, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

if (localPet == 0) then
maintask=.true.
else
maintask = .false.
end if

call t_initf('drv_in', LogPrint=.true., LogUnit=logunit, mpicom=global_comm, mastertask=maintask)
call t_startf(subname)

!-------------------------------------------
! Initialize clocks
!-------------------------------------------
Expand Down Expand Up @@ -200,23 +212,32 @@ subroutine SetModelServices(ensemble_driver, rc)
!-------------------------------------------

allocate(petList(ntasks_per_member))
! which driver instance is this?
inst = localPet/ntasks_per_member + 1
! We need to loop over instances
call t_startf('compute_drivers')
do inst = 1, number_of_members

! Determine pet list for driver instance
petList(1) = (inst-1) * ntasks_per_member
do n=2,ntasks_per_member
petList(n) = petList(n-1) + 1
enddo

! Add driver instance to ensemble driver
write(drvrinst,'(a,i4.4)') "ESM",inst
call NUOPC_DriverAddComp(ensemble_driver, drvrinst, ESMSetServices, petList=petList, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
enddo
call t_stopf('compute_drivers')

! Determine pet list for driver instance
inst = localPet/ntasks_per_member + 1
petList(1) = (inst-1) * ntasks_per_member
do n=2,ntasks_per_member
petList(n) = petList(n-1) + 1
enddo

! Add driver instance to ensemble driver
write(drvrinst,'(a,i4.4)') "ESM",inst
call NUOPC_DriverAddComp(ensemble_driver, drvrinst, ESMSetServices, petList=petList, comp=gridcomptmp, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

if (localpet >= petlist(1) .and. localpet <= petlist(ntasks_per_member)) then

driver = gridcomptmp
write(drvrinst,'(a,i4.4)') "ESM",inst
call NUOPC_DriverGetComp(ensemble_driver, drvrinst, comp=driver, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

if(number_of_members > 1) then
call NUOPC_CompAttributeAdd(driver, attrList=(/'inst_suffix'/), rc=rc)
Expand Down Expand Up @@ -264,6 +285,7 @@ subroutine SetModelServices(ensemble_driver, rc)
endif

deallocate(petList)
call t_stopf(subname)

call ESMF_LogWrite(trim(subname)//": done", ESMF_LOGMSG_INFO)

Expand Down
10 changes: 1 addition & 9 deletions cesm/driver/esm.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ module ESM
use shr_mem_mod , only : shr_mem_init
use shr_log_mod , only : shr_log_setLogunit
use esm_utils_mod, only : logunit, maintask, dbug_flag, chkerr
use perf_mod , only : t_initf, t_setLogUnit

implicit none
private
Expand Down Expand Up @@ -151,8 +150,6 @@ subroutine SetModelServices(driver, rc)
call ESMF_VMGet(vm, localPet=localPet, mpiCommunicator=global_comm, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return

call ESMF_VMGet(vm, localPet=localPet, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
if (localPet == 0) then
maintask=.true.
else
Expand Down Expand Up @@ -211,11 +208,6 @@ subroutine SetModelServices(driver, rc)
write(logunit,*) trim(meminitstr)
end if

!-------------------------------------------
! Timer initialization (has to be after pelayouts are determined)
!-------------------------------------------
call t_initf('drv_in', LogPrint=.true., LogUnit=logunit, mpicom=global_comm, mastertask=maintask, MaxThreads=maxthreads)

call ESMF_LogWrite(trim(subname)//": done", ESMF_LOGMSG_INFO)

end subroutine SetModelServices
Expand Down Expand Up @@ -636,7 +628,7 @@ subroutine AddAttributes(gcomp, driver, config, compid, compname, inst_suffix, n
if (chkerr(rc,__LINE__,u_FILE_u)) return

!------
! Add driver restart flag a to gcomp attributes
! Add driver restart flag to gcomp attributes
!------
attribute = 'read_restart'
call NUOPC_CompAttributeGet(driver, name=trim(attribute), value=cvalue, rc=rc)
Expand Down
6 changes: 4 additions & 2 deletions cesm/nuopc_cap_share/nuopc_shr_methods.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ subroutine set_component_logging(gcomp, maintask, logunit, shrlogunit, rc)
character(len=CL) :: logfile
character(len=CL) :: inst_suffix
integer :: inst_index ! not used here
integer :: n
!-----------------------------------------------------------------------

rc = ESMF_SUCCESS
Expand All @@ -157,8 +158,9 @@ subroutine set_component_logging(gcomp, maintask, logunit, shrlogunit, rc)
call get_component_instance(gcomp, inst_suffix, inst_index, rc=rc)
if (chkerr(rc,__LINE__,u_FILE_u)) return
! Multiinstance logfile name needs a correction
if(logfile(4:4) == '_') then
logfile = logfile(1:3)//trim(inst_suffix)//logfile(9:)
if(len_trim(inst_suffix) > 0) then
n = index(logfile, '.')
logfile = logfile(1:n-1)//trim(inst_suffix)//logfile(n:)
endif

open(newunit=logunit,file=trim(diro)//"/"//trim(logfile))
Expand Down