NOAA-EMC · DeniseWorthen · Aug 25, 2023 · Dec 27, 2022 · Dec 27, 2022 · Dec 27, 2022
diff --git a/.github/workflows/srt.yml b/.github/workflows/srt.yml
@@ -26,8 +26,8 @@ jobs:
       CPPFLAGS: "-I/usr/include -I/usr/local/include "
       LDFLAGS: "-L/usr/lib/x86_64-linux-gnu -lnetcdf -lnetcdff -lpnetcdf"     
       # Versions of all dependencies can be updated here
-      ESMF_VERSION: v8.4.0
-      PARALLELIO_VERSION: pio2_5_10
+      ESMF_VERSION: v8.5.0
+      PARALLELIO_VERSION: pio2_6_0
       CIME_MODEL: cesm
       CIME_DRIVER: nuopc
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -153,6 +153,7 @@ jobs:
           mkdir -p $HOME/cesm/scratch
           mkdir -p $HOME/cesm/inputdata
           pushd $GITHUB_WORKSPACE/cesm/cime/CIME/tests
+          export SRCROOT=$GITHUB_WORKSPACE/cesm/
           export CIME_TEST_PLATFORM=ubuntu-latest
           export PIO_INCDIR=$HOME/pio/include
           export PIO_LIBDIR=$HOME/pio/lib
@@ -175,6 +176,6 @@ jobs:
           popd
 #     the following can be used by developers to login to the github server in case of errors
 #     see https://github.com/marketplace/actions/debugging-with-tmate for further details
-#      - name: Setup tmate session
-#        if: ${{ failure() }}
-#        uses: mxschmitt/action-tmate@v3
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml
@@ -784,6 +784,34 @@
       If TRUE, the component libraries are always built with OpenMP capability.</desc>
   </entry>
 
+  <entry id="GPU_TYPE">
+    <type>char</type>
+    <valid_values></valid_values>
+    <default_value></default_value>
+    <group>build_def</group>
+    <file>env_build.xml</file>
+    <desc>If set will compile and submit with this gpu type enabled </desc>
+  </entry>
+
+  <entry id="GPU_OFFLOAD">
+    <type>char</type>
+    <valid_values></valid_values>
+    <default_value></default_value>
+    <group>build_def</group>
+    <file>env_build.xml</file>
+    <desc>If set will compile and submit with this gpu offload method enabled </desc>
+  </entry>
+
+  <entry id="MPI_GPU_WRAPPER_SCRIPT">
+    <type>char</type>
+    <valid_values></valid_values>
+    <default_value></default_value>
+    <group>build_def</group>
+    <file>env_build.xml</file>
+    <desc>If set will attach this script to the MPI run command, mapping
+    different MPI ranks to different GPUs within the same compute node</desc>
+  </entry>
+
   <entry id="SMP_PRESENT">
     <type>logical</type>
     <valid_values>TRUE,FALSE</valid_values>
@@ -1798,12 +1826,22 @@
     <desc>pes or cores per node for accounting purposes </desc>
   </entry>
 
+  <entry id="MAX_CPUTASKS_PER_GPU_NODE">
+    <type>integer</type>
+    <default_value>0</default_value>
+    <values>
+      <value compiler="nvhpc">1</value>
+    </values>
+    <group>mach_pes_last</group>
+    <file>env_mach_pes.xml</file>
+    <desc> Number of CPU cores per GPU node used for simulation </desc>
+  </entry>
+
   <entry id="NGPUS_PER_NODE">
     <type>integer</type>
     <default_value>0</default_value>
     <values>
-      <value compiler="pgi-gpu">1</value>
-      <value compiler="nvhpc-gpu">1</value>
+      <value compiler="nvhpc">1</value>
     </values>
     <group>mach_pes</group>
     <file>env_mach_pes.xml</file>

diff --git a/cime_config/namelist_definition_drv.xml b/cime_config/namelist_definition_drv.xml
@@ -938,7 +938,7 @@
       default: ogrid
     </desc>
     <values>
-      <value>xgrid</value>
+      <value>ogrid</value>
     </values>
   </entry>
   <entry id="ocn_surface_flux_scheme">

diff --git a/mediator/esmFldsExchange_cesm_mod.F90 b/mediator/esmFldsExchange_cesm_mod.F90
@@ -2158,7 +2158,7 @@ subroutine esmFldsExchange_cesm(gcomp, phase, rc)
           ! liquid from river and possibly flood from river to ocean
           if (fldchk(is_local%wrap%FBImp(comprof, comprof), 'Forr_rofl' , rc=rc)) then
              if (trim(rof2ocn_liq_rmap) == 'unset') then
-                call addmap_from(comprof, 'Forr_rofl', compocn, mapconsd, 'none', 'unset')
+                call addmap_from(comprof, 'Forr_rofl', compocn, mapconsd, 'one', 'unset')
              else
                 call addmap_from(comprof, 'Forr_rofl', compocn, map_rof2ocn_liq, 'none', rof2ocn_liq_rmap)
              end if
@@ -2182,7 +2182,7 @@ subroutine esmFldsExchange_cesm(gcomp, phase, rc)
           ! ice from river to ocean
           if (fldchk(is_local%wrap%FBImp(comprof, comprof), 'Forr_rofi' , rc=rc)) then
              if (trim(rof2ocn_ice_rmap) == 'unset') then
-                call addmap_from(comprof, 'Forr_rofi', compocn, mapconsd, 'none', 'unset')
+                call addmap_from(comprof, 'Forr_rofi', compocn, mapconsd, 'one', 'unset')
              else
                 call addmap_from(comprof, 'Forr_rofi', compocn, map_rof2ocn_ice, 'none', rof2ocn_ice_rmap)
              end if

diff --git a/mediator/esmFldsExchange_nems_mod.F90 b/mediator/esmFldsExchange_nems_mod.F90
@@ -38,6 +38,8 @@ subroutine esmFldsExchange_nems(gcomp, phase, rc)
     use esmFlds               , only : addmap_from => med_fldList_addmap_from
     use esmFlds               , only : addfld_aoflux => med_fldList_addfld_aoflux
     use esmFlds               , only : addmap_aoflux => med_fldList_addmap_aoflux
+    use esmFlds               , only : addfld_ocnalb => med_fldList_addfld_ocnalb
+    use esmFlds               , only : addmap_ocnalb => med_fldList_addmap_ocnalb
 
     ! input/output parameters:
     type(ESMF_GridComp)              :: gcomp
@@ -167,9 +169,12 @@ subroutine esmFldsExchange_nems(gcomp, phase, rc)
        deallocate(flds)
     end if
 
-    ! TODO: unused, but required to maintain B4B repro for mediator restarts; should be removed
+    ! Advertise the ocean albedos. These are not sent to the ATM in UFS.
     if (phase == 'advertise') then
-       call addfld_from(compice, 'mean_sw_pen_to_ocn')
+       call addfld_ocnalb('So_avsdr')
+       call addfld_ocnalb('So_avsdf')
+       call addfld_ocnalb('So_anidr')
+       call addfld_ocnalb('So_anidf')
     end if
 
     !=====================================================================
@@ -329,6 +334,17 @@ subroutine esmFldsExchange_nems(gcomp, phase, rc)
        end if
     end if
 
+    ! to ocn: swpen thru ice w/o bands
+    if (phase == 'advertise') then
+       if (is_local%wrap%comp_present(compice) .and. is_local%wrap%comp_present(compocn)) then
+          call addfld_from(compice, 'Fioi_swpen')
+       end if
+    else
+       if (fldchk(is_local%wrap%FBImp(compice,compice), 'Fioi_swpen', rc=rc)) then
+          call addmap_from(compice, 'Fioi_swpen', compocn, mapfcopy, 'unset', 'unset')
+       end if
+    end if
+
     ! to ocn: from sw from atm and sw net from ice (custom merge in med_phases_prep_ocn)
     ! - downward direct  near-infrared ("n" or "i") incident solar radiation
     ! - downward diffuse near-infrared ("n" or "i") incident solar radiation

diff --git a/mediator/med.F90 b/mediator/med.F90
@@ -1804,7 +1804,8 @@ subroutine DataInitialize(gcomp, rc)
          call esmFldsExchange_cesm(gcomp, phase='initialize', rc=rc)
          if (ChkErr(rc,__LINE__,u_FILE_u)) return
       else if (trim(coupling_mode(1:4)) == 'nems') then
-       call esmFldsExchange_nems(gcomp, phase='initialize', rc=rc)
+         call esmFldsExchange_nems(gcomp, phase='initialize', rc=rc)
+         if (ChkErr(rc,__LINE__,u_FILE_u)) return
       else if (trim(coupling_mode) == 'hafs') then
          call esmFldsExchange_hafs(gcomp, phase='initialize', rc=rc)
          if (ChkErr(rc,__LINE__,u_FILE_u)) return
@@ -1939,14 +1940,12 @@ subroutine DataInitialize(gcomp, rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
     !----------------------------------------------------------
-    ! Initialize ocean albedos (this is needed for cesm and hafs)
+    ! Initialize ocean albedos
     !----------------------------------------------------------
 
-    if (trim(coupling_mode(1:5)) == 'cesm_') then
-       if (is_local%wrap%comp_present(compocn) .or. is_local%wrap%comp_present(compatm)) then
-          call med_phases_ocnalb_run(gcomp, rc=rc)
-          if (ChkErr(rc,__LINE__,u_FILE_u)) return
-       end if
+    if (is_local%wrap%comp_present(compocn) .or. is_local%wrap%comp_present(compatm)) then
+       call med_phases_ocnalb_run(gcomp, rc=rc)
+       if (ChkErr(rc,__LINE__,u_FILE_u)) return
     end if
 
     !---------------------------------------

diff --git a/mediator/med_internalstate_mod.F90 b/mediator/med_internalstate_mod.F90
@@ -262,7 +262,6 @@ subroutine med_internalstate_init(gcomp, rc)
        end do
     end if
     is_local%wrap%num_icesheets = num_icesheets
-
     call NUOPC_CompAttributeGet(gcomp, name='mediator_present', value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
     if (isPresent .and. isSet) then

diff --git a/mediator/med_io_mod.F90 b/mediator/med_io_mod.F90
@@ -75,7 +75,7 @@ module med_io_mod
   character(*),parameter         :: prefix    = "med_io_"
   character(*),parameter         :: modName   = "(med_io_mod) "
   character(*),parameter         :: version   = "cmeps0"
-  
+
   integer                        :: pio_iotype
   integer                        :: pio_ioformat
   type(iosystem_desc_t), pointer :: io_subsystem
@@ -1738,8 +1738,8 @@ subroutine med_io_read_init_iodesc(FB, name1, pioid, iodesc, rc)
 
        deallocate(minIndexPTile, maxIndexPTile)
     else
-       if(maintask) write(logunit,*) trim(subname),' ERROR: '//trim(name1)//' is not present, aborting '
-       call ESMF_LogWrite(trim(subname)//' ERROR: '//trim(name1)//' is not present, aborting ', ESMF_LOGMSG_INFO)
+       if(maintask) write(logunit,'(a)') trim(subname)//' ERROR: '//trim(name1)//' is not present, aborting '
+       call ESMF_LogWrite(trim(subname)//' ERROR: '//trim(name1)//' is not present, aborting ', ESMF_LOGMSG_ERROR)
        rc = ESMF_FAILURE
     end if ! end if rcode check
 

diff --git a/mediator/med_map_mod.F90 b/mediator/med_map_mod.F90
@@ -111,7 +111,7 @@ subroutine med_map_RouteHandles_initfrom_esmflds(gcomp, flds_scalar_name, llogun
     type(ESMF_Mesh)           :: mesh_dst
     type(med_fldlist_type), pointer :: FldListFr
     type(med_fldlist_entry_type), pointer :: fldptr
-    character(len=*), parameter :: subname=' (module_med_map: RouteHandles_init) '
+    character(len=*), parameter :: subname=' (med_map_mod: RouteHandles_init) '
     !-----------------------------------------------------------
 
     call t_startf('MED:'//subname)
@@ -259,7 +259,8 @@ subroutine med_map_RouteHandles_initfrom_esmflds(gcomp, flds_scalar_name, llogun
                       if (chkerr(rc,__LINE__,u_FILE_u)) return
                       if (maintask) then
                          write(logunit,'(a)') trim(subname)//' created field_NormOne for '&
-                              //compname(n1)//'->'//compname(n2)//' with mapping '//trim(mapnames(mapindex))
+                              //trim(compname(n1))//'->'//trim(compname(n2))//' with mapping '&
+                              //trim(mapnames(mapindex))
                       end if
                    end if
                 end do ! end of loop over map_indiex mappers
@@ -304,7 +305,7 @@ subroutine med_map_routehandles_initfrom_fieldbundle(n1, n2, FBsrc, FBdst, mapin
     ! local variables
     type(ESMF_Field)   :: fldsrc
     type(ESMF_Field)   :: flddst
-    character(len=*), parameter :: subname=' (module_MED_map:med_map_routehandles_initfrom_fieldbundle) '
+    character(len=*), parameter :: subname=' (med_map_mod:med_map_routehandles_initfrom_fieldbundle) '
     !---------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -653,7 +654,7 @@ logical function med_map_RH_is_created_RH3d(RHs,n1,n2,mapindex,rc)
     integer                , intent(out)   :: rc
 
     ! local variables
-    character(len=*), parameter :: subname=' (module_MED_map:med_map_RH_is_created_RH3d) '
+    character(len=*), parameter :: subname=' (med_map_mod:med_map_RH_is_created_RH3d) '
     !-----------------------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -678,7 +679,7 @@ logical function med_map_RH_is_created_RH1d(RHs,mapindex,rc)
     ! local variables
     integer :: rc1, rc2
     logical :: mapexists
-    character(len=*), parameter :: subname=' (module_MED_map:med_map_RH_is_created_RH1d) '
+    character(len=*), parameter :: subname=' (med_map_mod:med_map_RH_is_created_RH1d) '
     !-----------------------------------------------------------
 
     rc  = ESMF_SUCCESS
@@ -750,7 +751,7 @@ subroutine med_map_packed_field_create(destcomp, flds_scalar_name, &
     character(CL), allocatable :: fieldNameList(:)
     character(CS)              :: mapnorm_mapindex
     character(len=CX)          :: tmpstr
-    character(len=*), parameter :: subname=' (module_MED_map:med_packed_field_create) '
+    character(len=*), parameter :: subname=' (med_map_mod:med_packed_field_create) '
     !-----------------------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -818,6 +819,7 @@ subroutine med_map_packed_field_create(destcomp, flds_scalar_name, &
                       //'  '//trim(fieldnamelist(nf))
                    call ESMF_LogWrite(trim(tmpstr), ESMF_LOGMSG_INFO)
                 else
+                   !if(rof_name .ne. 'xrof' .and. compname(destcomp) .ne. 'ocn') then
                    if (mapnorm_mapindex /= packed_data(mapindex)%mapnorm) then
                      write(tmpstr,*)'Map type '//trim(mapnames(mapindex)) &
                         //', destcomp '//trim(compname(destcomp)) &
@@ -953,7 +955,7 @@ subroutine med_map_field_packed(FBSrc, FBDst, FBFracSrc, field_normOne, packed_d
     type(ESMF_Field), pointer  :: fieldlist_dst(:)
     real(r8), pointer          :: data_norm(:)
     real(r8), pointer          :: data_dst(:,:)
-    character(len=*), parameter  :: subname=' (module_MED_map:med_map_field_packed) '
+    character(len=*), parameter  :: subname=' (med_map_mod:med_map_field_packed) '
     !-----------------------------------------------------------
 
     call t_startf('MED:'//subname)
@@ -1165,7 +1167,7 @@ subroutine med_map_field_normalized(field_src, field_dst, routehandles, maptype,
     integer           :: ungriddedUBound(1)     ! currently the size must equal 1 for rank 2 fields
     integer           :: lsize_src
     integer           :: lsize_dst
-    character(len=*), parameter  :: subname=' (module_MED_map:med_map_field_normalized) '
+    character(len=*), parameter  :: subname=' (med_map_mod:med_map_field_normalized) '
     !-----------------------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -1278,7 +1280,7 @@ subroutine med_map_field(field_src, field_dst, routehandles, maptype, fldname, r
     logical :: checkflag = .false.
     character(len=CS) :: lfldname
     real(ESMF_KIND_R8), parameter :: fillValue = 9.99e20_ESMF_KIND_R8
-    character(len=*), parameter :: subname='(module_MED_map:med_map_field) '
+    character(len=*), parameter :: subname='(med_map_mod:med_map_field) '
     !---------------------------------------------------
 
     rc = ESMF_SUCCESS
@@ -1381,7 +1383,7 @@ subroutine med_map_uv_cart3d(FBsrc, FBdst, routehandles, mapindex, rc)
     integer             :: spatialDim
     real(r8), parameter :: deg2rad = shr_const_pi/180.0_R8  ! deg to rads
     logical             :: first_time = .true.
-    character(len=*), parameter :: subname=' (module_MED_map:med_map_uv_cart3d) '
+    character(len=*), parameter :: subname=' (med_map_mod:med_map_uv_cart3d) '
     !-------------------------------------------------------------------------------
 
     rc = ESMF_SUCCESS

diff --git a/mediator/med_methods_mod.F90 b/mediator/med_methods_mod.F90
@@ -1354,7 +1354,10 @@ subroutine med_methods_FB_accum(FBout, FBin, copy, rc)
         call med_methods_Field_GetFldPtr(lfield, fldptr1=dataptro1, fldptr2=dataptro2, rank=lranko, rc=rc)
         if (chkerr(rc,__LINE__,u_FILE_u)) return
 
-        if (lranki == 1 .and. lranko == 1) then
+        if (lranki == 0 .and. lranko == 0) then
+           ! do nothing
+          call ESMF_LogWrite(trim(subname)//": Both ranki and ranko are 0", ESMF_LOGMSG_INFO)
+        elseif (lranki == 1 .and. lranko == 1) then
 
           if (.not.med_methods_FieldPtr_Compare(dataPtro1, dataPtri1, subname, rc)) then
             call ESMF_LogWrite(trim(subname)//": ERROR in dataPtr1 size ", ESMF_LOGMSG_ERROR)
@@ -1397,7 +1400,7 @@ subroutine med_methods_FB_accum(FBout, FBin, copy, rc)
         else
 
           write(msgString,'(a,2i8)') trim(subname)//": ranki, ranko = ",lranki,lranko
-          call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_INFO)
+          call ESMF_LogWrite(trim(msgString), ESMF_LOGMSG_ERROR)
           call ESMF_LogWrite(trim(subname)//": ERROR ranki ranko not supported "//trim(lfieldnamelist(n)), &
                ESMF_LOGMSG_ERROR)
           rc = ESMF_FAILURE

diff --git a/mediator/med_phases_history_mod.F90 b/mediator/med_phases_history_mod.F90
@@ -25,7 +25,7 @@ module med_phases_history_mod
   use med_io_mod            , only : med_io_write, med_io_wopen, med_io_enddef, med_io_close
   use perf_mod              , only : t_startf, t_stopf
   use pio                   , only : file_desc_t
-  
+
   implicit none
   private