remove shr_mpi_mod

.github/actions/buildnetcdff/action.yml

@@ -23,7 +23,7 @@ runs:
       with:
         repository: Unidata/netcdf-fortran
         path: netcdf-fortran-src
-        ref: ${{ env.NETCDF_FORTRAN_VERSION }}
+        ref: ${{ inputs.NETCDF_FORTRAN_VERSION }}
     - id: netcdf-fortran-build
       shell: bash
       run: |

.github/workflows/extbuild.yml

@@ -94,6 +94,6 @@ jobs:
           export SRC_ROOT=
           mkdir build-cdeps
           pushd build-cdeps
-          cmake -DCMAKE_BUILD_TYPE=DEBUG -DCMAKE_Fortran_FLAGS="-DCPRGNU -g -Wall -ffree-form -ffree-line-length-none -fallow-argument-mismatch " ../
+          cmake -Wno-dev -DCMAKE_BUILD_TYPE=DEBUG -DCMAKE_Fortran_FLAGS="-DCPRGNU -g -Wall -ffree-form -ffree-line-length-none -fallow-argument-mismatch " ../
           make VERBOSE=1
           popd

datm/atm_comp_nuopc.F90

@@ -8,15 +8,16 @@ module cdeps_datm_comp
   ! This is the NUOPC cap for DATM
   !----------------------------------------------------------------------------
 
+  use ESMF             , only : ESMF_VM, ESMF_VMBroadcast
   use ESMF             , only : ESMF_Mesh, ESMF_GridComp, ESMF_SUCCESS, ESMF_LogWrite
   use ESMF             , only : ESMF_GridCompSetEntryPoint, ESMF_METHOD_INITIALIZE
   use ESMF             , only : ESMF_MethodRemove, ESMF_State, ESMF_Clock, ESMF_TimeInterval
   use ESMF             , only : ESMF_State, ESMF_Field, ESMF_LOGMSG_INFO, ESMF_ClockGet
   use ESMF             , only : ESMF_Time, ESMF_Alarm, ESMF_TimeGet, ESMF_TimeInterval
   use ESMF             , only : operator(+), ESMF_TimeIntervalGet, ESMF_ClockGetAlarm
   use ESMF             , only : ESMF_AlarmIsRinging, ESMF_AlarmRingerOff, ESMF_StateGet
-  use ESMF             , only : ESMF_FieldGet, ESMF_MAXSTR
-  use ESMF             , only : ESMF_TraceRegionEnter, ESMF_TraceRegionExit
+  use ESMF             , only : ESMF_FieldGet, ESMF_MAXSTR, ESMF_VMBroadcast
+  use ESMF             , only : ESMF_TraceRegionEnter, ESMF_TraceRegionExit, ESMF_GridCompGet
   use NUOPC            , only : NUOPC_CompDerive, NUOPC_CompSetEntryPoint, NUOPC_CompSpecialize
   use NUOPC            , only : NUOPC_CompAttributeGet, NUOPC_Advertise
   use NUOPC_Model      , only : model_routine_SS        => SetServices
@@ -28,7 +29,6 @@ module cdeps_datm_comp
   use shr_const_mod    , only : shr_const_cday
   use shr_sys_mod      , only : shr_sys_abort
   use shr_cal_mod      , only : shr_cal_ymd2date
-  use shr_mpi_mod      , only : shr_mpi_bcast
   use shr_log_mod     , only : shr_log_setLogUnit
   use dshr_methods_mod , only : dshr_state_diagnose, chkerr, memcheck
   use dshr_strdata_mod , only : shr_strdata_type, shr_strdata_init_from_config, shr_strdata_advance
@@ -222,17 +222,32 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     integer           :: nu         ! unit number
     integer           :: ierr       ! error code
     logical           :: exists     ! check for file existence
+    integer           :: bcasttmp(9)
+    type(ESMF_VM)     :: vm
     character(len=*),parameter :: subname=trim(modName) // ':(InitializeAdvertise) '
     character(*)    ,parameter :: F00 = "('(" // trim(modName) // ") ',8a)"
     character(*)    ,parameter :: F01 = "('(" // trim(modName) // ") ',a,2x,i8)"
     character(*)    ,parameter :: F02 = "('(" // trim(modName) // ") ',a,l6)"
     !-------------------------------------------------------------------------------
 
-    namelist / datm_nml / datamode, &
-         model_meshfile, model_maskfile, &
-         nx_global, ny_global, restfilm, iradsw, factorFn_data, factorFn_mesh, &
-         flds_presaero, flds_co2, flds_wiso, bias_correct, anomaly_forcing, &
-         skip_restart_read, flds_presndep, flds_preso3
+    namelist / datm_nml / &
+         datamode, &
+         model_meshfile, &
+         model_maskfile, &
+         nx_global, &
+         ny_global, &
+         restfilm, &
+         iradsw, &
+         factorFn_data, &
+         factorFn_mesh, &
+         flds_presaero, &
+         flds_co2, &
+         flds_wiso, &
+         bias_correct, &
+         anomaly_forcing, &
+         skip_restart_read, &
+         flds_presndep, &
+         flds_preso3
 
     rc = ESMF_SUCCESS
 
@@ -259,23 +274,44 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
           write(logunit,*) 'ERROR: reading input namelist, '//trim(nlfilename)//' iostat=',ierr
           call shr_sys_abort(subName//': namelist read error '//trim(nlfilename))
        end if
+       bcasttmp = 0
+       bcasttmp(1) = nx_global
+       bcasttmp(2) = ny_global
+       bcasttmp(3) = iradsw
+       if(flds_presaero)     bcasttmp(4) = 1
+       if(flds_presndep)     bcasttmp(5) = 1
+       if(flds_preso3)       bcasttmp(6) = 1
+       if(flds_co2)          bcasttmp(7) = 1
+       if(flds_wiso)         bcasttmp(8) = 1
+       if(skip_restart_read) bcasttmp(9) = 1
+
     end if
+    call ESMF_GridCompGet(gcomp, vm=vm, rc=rc)
+    if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
-    call shr_mpi_bcast(datamode                  , mpicom, 'datamode')
-    call shr_mpi_bcast(model_meshfile            , mpicom, 'model_meshfile')
-    call shr_mpi_bcast(model_maskfile            , mpicom, 'model_maskfile')
-    call shr_mpi_bcast(nx_global                 , mpicom, 'nx_global')
-    call shr_mpi_bcast(ny_global                 , mpicom, 'ny_global')
-    call shr_mpi_bcast(iradsw                    , mpicom, 'iradsw')
-    call shr_mpi_bcast(factorFn_data             , mpicom, 'factorFn_data')
-    call shr_mpi_bcast(factorFn_mesh             , mpicom, 'factorFn_mesh')
-    call shr_mpi_bcast(restfilm                  , mpicom, 'restfilm')
-    call shr_mpi_bcast(flds_presaero             , mpicom, 'flds_presaero')
-    call shr_mpi_bcast(flds_presndep             , mpicom, 'flds_presndep')
-    call shr_mpi_bcast(flds_preso3               , mpicom, 'flds_preso3')
-    call shr_mpi_bcast(flds_co2                  , mpicom, 'flds_co2')
-    call shr_mpi_bcast(flds_wiso                 , mpicom, 'flds_wiso')
-    call shr_mpi_bcast(skip_restart_read         , mpicom, 'skip_restart_read')
+    call ESMF_VMBroadcast(vm, datamode, CL, main_task, rc=rc)
+    if (ChkErr(rc,__LINE__,u_FILE_u)) return
+    call ESMF_VMBroadcast(vm, model_meshfile, CL, main_task, rc=rc)
+    if (ChkErr(rc,__LINE__,u_FILE_u)) return
+    call ESMF_VMBroadcast(vm, model_maskfile, CL, main_task, rc=rc)
+    if (ChkErr(rc,__LINE__,u_FILE_u)) return
+    call ESMF_VMBroadcast(vm, factorFn_data, CL, main_task, rc=rc)
+    if (ChkErr(rc,__LINE__,u_FILE_u)) return
+    call ESMF_VMBroadcast(vm, factorFn_mesh, CL, main_task, rc=rc)
+    if (ChkErr(rc,__LINE__,u_FILE_u)) return
+    call ESMF_VMBroadcast(vm, restfilm, CL, main_task, rc=rc)
+    if (ChkErr(rc,__LINE__,u_FILE_u)) return
+    call ESMF_VMBroadcast(vm, bcasttmp, 9, main_task, rc=rc)
+    if (ChkErr(rc,__LINE__,u_FILE_u)) return
+    nx_global         = bcasttmp(1)
+    ny_global         = bcasttmp(2)
+    iradsw            = bcasttmp(3)
+    flds_presaero     = (bcasttmp(4) == 1)
+    flds_presndep     = (bcasttmp(5) == 1)
+    flds_preso3       = (bcasttmp(6) == 1)
+    flds_co2          = (bcasttmp(7) == 1)
+    flds_wiso         = (bcasttmp(8) == 1)
+    skip_restart_read = (bcasttmp(9) == 1)
 
     ! write namelist input to standard out
     if (my_task == main_task) then

datm/datm_datamode_cfsr_mod.F90

@@ -5,7 +5,6 @@ module datm_datamode_cfsr_mod
   use shr_kind_mod     , only : r8=>shr_kind_r8, i8=>shr_kind_i8, cl=>shr_kind_cl, cs=>shr_kind_cs
   use shr_sys_mod      , only : shr_sys_abort
   use shr_precip_mod   , only : shr_precip_partition_rain_snow_ramp
-  use shr_mpi_mod      , only : shr_mpi_max
   use shr_const_mod    , only : shr_const_tkfrz, shr_const_rhofw, shr_const_rdair
   use dshr_methods_mod , only : dshr_state_getfldptr, chkerr
   use dshr_strdata_mod , only : shr_strdata_type, shr_strdata_get_stream_pointer
@@ -51,7 +50,7 @@ module datm_datamode_cfsr_mod
   real(r8) , parameter :: tKFrz    = SHR_CONST_TKFRZ
   real(r8) , parameter :: rdair    = SHR_CONST_RDAIR ! dry air gas constant ~ J/K/kg
   real(r8) , parameter :: rhofw    = SHR_CONST_RHOFW ! density of fresh water ~ kg/m^3
-  
+
   character(*), parameter :: nullstr = 'undefined'
   character(*), parameter :: rpfile  = 'rpointer.atm'
   character(*), parameter :: u_FILE_u = &
@@ -166,6 +165,7 @@ end subroutine datm_datamode_cfsr_init_pointers
 
   !===============================================================================
   subroutine datm_datamode_cfsr_advance(exportstate, mainproc, logunit, mpicom, target_ymd, target_tod, model_calendar, rc)
+    use ESMF, only: ESMF_VMGetCurrent, ESMF_VMAllReduce, ESMF_REDUCE_MAX, ESMF_VM
 
     ! input/output variables
     type(ESMF_State)       , intent(inout) :: exportState
@@ -181,8 +181,9 @@ subroutine datm_datamode_cfsr_advance(exportstate, mainproc, logunit, mpicom, ta
     logical  :: first_time = .true.
     integer  :: n                   ! indices
     integer  :: lsize               ! size of attr vect
-    real(r8) :: rtmp
+    real(r8) :: rtmp(2)
     real(r8) :: tbot, pbot
+    type(ESMF_VM) :: vm
     character(len=*), parameter :: subname='(datm_datamode_cfsr_advance): '
     !-------------------------------------------------------------------------------
 
@@ -191,14 +192,18 @@ subroutine datm_datamode_cfsr_advance(exportstate, mainproc, logunit, mpicom, ta
     lsize = size(strm_mask)
 
     if (first_time) then
+       call ESMF_VMGetCurrent(vm, rc=rc)
+       if (ChkErr(rc,__LINE__,u_FILE_u)) return
        ! determine tbotmax (see below for use)
-       rtmp = maxval(Sa_tbot(:))
-       call shr_mpi_max(rtmp, tbotmax, mpicom, 'datm_tbot', all=.true.)
+       rtmp(1) = maxval(Sa_tbot(:))
+       call ESMF_VMAllReduce(vm, rtmp, rtmp(2:), 1, ESMF_REDUCE_MAX, rc=rc)
+       tbotmax = rtmp(2)
        if (mainproc) write(logunit,*) trim(subname),' tbotmax = ',tbotmax
 
        ! determine maskmax (see below for use)
-       rtmp = maxval(strm_mask(:))
-       call shr_mpi_max(rtmp, maskmax, mpicom, 'datm_mask', all=.true.)
+       rtmp(1) = maxval(strm_mask(:))
+       call ESMF_VMAllReduce(vm, rtmp, rtmp(2:), 1, ESMF_REDUCE_MAX, rc=rc)
+       maskmax = rtmp(2)
        if (mainproc) write(logunit,*) trim(subname),' maskmax = ',maskmax
 
        ! reset first_time
@@ -217,7 +222,7 @@ end subroutine datm_datamode_cfsr_advance
   !===============================================================================
   subroutine datm_datamode_cfsr_restart_write(case_name, inst_suffix, ymd, tod, &
        logunit, my_task, sdat)
-    
+
     ! input/output variables
     character(len=*)            , intent(in)    :: case_name
     character(len=*)            , intent(in)    :: inst_suffix

datm/datm_datamode_clmncep_mod.F90

@@ -6,7 +6,6 @@ module datm_datamode_clmncep_mod
   use shr_kind_mod     , only : r8=>shr_kind_r8, i8=>shr_kind_i8, cl=>shr_kind_cl, cs=>shr_kind_cs
   use shr_sys_mod      , only : shr_sys_abort
   use shr_precip_mod   , only : shr_precip_partition_rain_snow_ramp
-  use shr_mpi_mod      , only : shr_mpi_max
   use shr_const_mod    , only : shr_const_spval, shr_const_tkfrz, shr_const_pi
   use shr_const_mod    , only : shr_const_pstd, shr_const_stebol, shr_const_rdair
   use dshr_methods_mod , only : dshr_state_getfldptr, chkerr
@@ -345,6 +344,7 @@ end subroutine datm_datamode_clmncep_init_pointers
 
   !===============================================================================
   subroutine datm_datamode_clmncep_advance(mainproc, logunit, mpicom, rc)
+    use ESMF, only: ESMF_VMGetCurrent, ESMF_VMAllReduce, ESMF_REDUCE_MAX, ESMF_VM
 
     ! input/output variables
     logical                , intent(in)    :: mainproc
@@ -356,7 +356,7 @@ subroutine datm_datamode_clmncep_advance(mainproc, logunit, mpicom, rc)
     logical  :: first_time = .true.
     integer  :: n                   ! indices
     integer  :: lsize               ! size of attr vect
-    real(r8) :: rtmp
+    real(r8) :: rtmp(2)
     real(r8) :: swndr
     real(r8) :: swndf
     real(r8) :: swvdr
@@ -365,6 +365,7 @@ subroutine datm_datamode_clmncep_advance(mainproc, logunit, mpicom, rc)
     real(r8) :: tbot, pbot
     real(r8) :: vp
     real(r8) :: ea, e, qsat, frac
+    type(ESMF_VM) :: vm
     character(len=*), parameter :: subname='(datm_datamode_clmncep_advance): '
     !-------------------------------------------------------------------------------
 
@@ -373,27 +374,35 @@ subroutine datm_datamode_clmncep_advance(mainproc, logunit, mpicom, rc)
     lsize = size(Sa_u)
 
     if (first_time) then
+       call ESMF_VMGetCurrent(vm, rc=rc)
+       if (ChkErr(rc,__LINE__,u_FILE_u)) return
        ! determine tbotmax (see below for use)
-       rtmp = maxval(Sa_tbot(:))
-       call shr_mpi_max(rtmp, tbotmax, mpicom, 'datm_tbot', all=.true.)
+       rtmp(1) = maxval(Sa_tbot(:))
+       call ESMF_VMAllReduce(vm, rtmp, rtmp(2:), 1, ESMF_REDUCE_MAX, rc=rc)
+       if (ChkErr(rc,__LINE__,u_FILE_u)) return
+       tbotmax = rtmp(2)
        if (mainproc) write(logunit,*) trim(subname),' tbotmax = ',tbotmax
        if(tbotmax <= 0) then
           call shr_sys_abort(subname//'ERROR: bad value in tbotmax')
        endif
 
        ! determine anidrmax (see below for use)
        if (atm_prognostic) then
-          rtmp = maxval(Sx_anidr(:))
-          call shr_mpi_max(rtmp, anidrmax, mpicom, 'datm_ani', all=.true.)
+          rtmp(1) = maxval(Sx_anidr(:))
+          call ESMF_VMAllReduce(vm, rtmp, rtmp(2:), 1, ESMF_REDUCE_MAX, rc=rc)
+          if (ChkErr(rc,__LINE__,u_FILE_u)) return
+          anidrmax = rtmp(2)
        else
           anidrmax = SHR_CONST_SPVAL
        end if
        if (mainproc) write(logunit,*) trim(subname),' anidrmax = ',anidrmax
 
        ! determine tdewmax (see below for use)
        if (associated(strm_tdew)) then
-          rtmp = maxval(strm_tdew(:))
-          call shr_mpi_max(rtmp, tdewmax, mpicom, 'datm_tdew', all=.true.)
+          rtmp(1) = maxval(strm_tdew(:))
+          call ESMF_VMAllReduce(vm, rtmp, rtmp(2:), 1, ESMF_REDUCE_MAX, rc=rc)
+          if (ChkErr(rc,__LINE__,u_FILE_u)) return
+          tdewmax = rtmp(2)
           if (mainproc) write(logunit,*) trim(subname),' tdewmax = ',tdewmax
        endif
 

datm/datm_datamode_era5_mod.F90

@@ -5,7 +5,6 @@ module datm_datamode_era5_mod
   use shr_kind_mod     , only : r8=>shr_kind_r8, i8=>shr_kind_i8, cl=>shr_kind_cl, cs=>shr_kind_cs
   use shr_sys_mod      , only : shr_sys_abort
   use shr_precip_mod   , only : shr_precip_partition_rain_snow_ramp
-  use shr_mpi_mod      , only : shr_mpi_max
   use shr_const_mod    , only : shr_const_tkfrz, shr_const_rhofw, shr_const_rdair
   use dshr_methods_mod , only : dshr_state_getfldptr, chkerr
   use dshr_strdata_mod , only : shr_strdata_type, shr_strdata_get_stream_pointer
@@ -196,6 +195,7 @@ end subroutine datm_datamode_era5_init_pointers
 
   !===============================================================================
   subroutine datm_datamode_era5_advance(exportstate, mainproc, logunit, mpicom, target_ymd, target_tod, model_calendar, rc)
+    use ESMF, only: ESMF_VMGetCurrent, ESMF_VMAllReduce, ESMF_REDUCE_MAX, ESMF_VM
 
     ! input/output variables
     type(ESMF_State)       , intent(inout) :: exportState
@@ -211,28 +211,33 @@ subroutine datm_datamode_era5_advance(exportstate, mainproc, logunit, mpicom, ta
     logical  :: first_time = .true.
     integer  :: n                   ! indices
     integer  :: lsize               ! size of attr vect
-    real(r8) :: rtmp
+    real(r8) :: rtmp(2)
     real(r8) :: t2, pslv
     real(r8) :: vp
     real(r8) :: e, qsat
+    type(ESMF_VM) :: vm
     character(len=*), parameter :: subname='(datm_datamode_era5_advance): '
     !-------------------------------------------------------------------------------
 
     rc = ESMF_SUCCESS
 
     lsize = size(strm_tdew)
-
     if (first_time) then
+       call ESMF_VMGetCurrent(vm, rc=rc)
        ! determine t2max (see below for use)
        if (associated(Sa_t2m)) then
-         rtmp = maxval(Sa_t2m(:))
-         call shr_mpi_max(rtmp, t2max, mpicom, 'datm_t2m', all=.true.)
+         rtmp(1) = maxval(Sa_t2m(:))
+
+         call ESMF_VMAllReduce(vm, rtmp, rtmp(2:), 1, ESMF_REDUCE_MAX, rc=rc)
+         t2max = rtmp(2)
          if (mainproc) write(logunit,*) trim(subname),' t2max = ',t2max
        end if
 
        ! determine tdewmax (see below for use)
-       rtmp = maxval(strm_tdew(:))
-       call shr_mpi_max(rtmp, td2max, mpicom, 'datm_td2m', all=.true.)
+       rtmp(1) = maxval(strm_tdew(:))
+       call ESMF_VMAllReduce(vm, rtmp, rtmp(2:), 1, ESMF_REDUCE_MAX, rc=rc)
+       td2max = rtmp(2)
+
        if (mainproc) write(logunit,*) trim(subname),' td2max = ',td2max
 
        ! reset first_time
@@ -272,7 +277,7 @@ subroutine datm_datamode_era5_advance(exportstate, mainproc, logunit, mpicom, ta
     if (associated(Faxa_swvdf)) Faxa_swvdf(:) = Faxa_swdn(:)*Faxa_swvdf(:)
     if (associated(Faxa_swndf)) Faxa_swndf(:) = Faxa_swdn(:)*Faxa_swndf(:)
 
-    !--- TODO: need to understand relationship between shortwave bands and 
+    !--- TODO: need to understand relationship between shortwave bands and
     !--- net shortwave rad. currently it is provided directly from ERA5
     !--- and the total of the bands are not consistent with the swnet
     !--- swnet: a diagnostic quantity ---