Add CDEPS component, add CDEPS tests and fix bugs in the datm_bulk_gefs test

.gitmodules

@@ -4,8 +4,8 @@
   branch = develop
 [submodule "NEMS"]
   path = NEMS
-  url = https://github.com/NOAA-EMC/NEMS
-  branch = develop
+  url = https://github.com/binli2337/NEMS
+  branch = feature/new_cdeps
 [submodule "FMS"]
   path = FMS
   url = https://github.com/NOAA-GFDL/FMS
@@ -24,8 +24,8 @@
   branch = develop
 [submodule "CMEPS"]
   path = CMEPS-interface/CMEPS
-  url = https://github.com/NOAA-EMC/CMEPS.git
-  branch = emc/develop
+  url = https://github.com/DeniseWorthen/CMEPS
+  branch = feature/updcmeps
 [submodule "MOM6"]
   path = MOM6-interface/MOM6
   url = https://github.com/NOAA-EMC/MOM6
@@ -34,6 +34,10 @@
   path = CICE-interface/CICE
   url = https://github.com/NOAA-EMC/CICE
   branch = emc/develop
+[submodule "CDEPS"]
+  path = CDEPS
+  url = https://github.com/binli2337/CDEPS
+  branch = feature/add_gefs
 [submodule "DATM"]
   path = DATM
   url = https://github.com/NOAA-EMC/NEMSdatm

CMEPS-interface/CMakeLists.txt

@@ -18,7 +18,6 @@ endif()
 list(APPEND _nems_util_files
   CMEPS/util/shr_abort_mod.F90
   CMEPS/util/shr_log_mod.F90
-  CMEPS/util/shr_pio_mod.F90
   CMEPS/util/shr_sys_mod.F90
   CMEPS/util/shr_flux_mod.F90
   CMEPS/util/shr_mpi_mod.F90
@@ -66,8 +65,7 @@ list(APPEND _mediator_files
   CMEPS/mediator/med_phases_post_wav_mod.F90)
 
 list(APPEND _mediator_defs ESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR}
-                           ESMF_VERSION_MINOR=${ESMF_VERSION_MINOR}
-                           INTERNAL_PIO_INIT)
+                           ESMF_VERSION_MINOR=${ESMF_VERSION_MINOR})
 
 ### Create target library and set PUBLIC interfaces on the library
 add_library(cmeps STATIC ${_nems_util_files} ${_mediator_files})

CMakeLists.txt

@@ -240,7 +240,8 @@ if(NEMSdatm)
 endif()
 
 if(CDEPS)
-  message("ADD CDEPS-interface directory")
+  add_subdirectory(CDEPS)
+#  message("ADD CDEPS-interface directory")
 #  add_subdirectory(CDEPS-interface)
 endif()
 
@@ -302,10 +303,18 @@ endif()
 
 if(NEMSdatm)
   add_dependencies(ufs datatm)
-  list(APPEND _ufs_defs_private FRONT_DATM=datm)
+  list(APPEND _ufs_defs_private FRONT_NEMS_DATM=datm)
   list(APPEND _ufs_libs_public datatm)
 endif()
 
+if(CDEPS)
+  list(APPEND _ufs_defs_private CDEPS
+                                FRONT_CDEPS_DATM=atm_comp_nuopc)
+  add_dependencies(ufs datm)
+  include_directories(${CMAKE_CURRENT_BINARY_DIR}/CDEPS/datm)
+  target_link_libraries(ufs PUBLIC datm)
+endif()
+
 if(FV3)
   add_dependencies(ufs fv3atm)
   list(APPEND _ufs_defs_private FRONT_FV3=fv3gfs_cap_mod)
@@ -331,7 +340,9 @@ if(CMEPS)
 endif()
 
 if(CDEPS)
-  message("Do CDEPS things here")
+  #message("Do CDEPS things here")
+  add_dependencies(ufs streams dshr cdeps_share FoX_dom)
+  target_link_libraries(ufs PUBLIC streams dshr cdeps_share FoX_dom)
 endif()
 
 target_compile_definitions(ufs PRIVATE "${_ufs_defs_private}")
@@ -357,7 +368,7 @@ endif()
 
 target_compile_definitions(ufs_model PRIVATE "${_ufs_model_defs_private}")
 
-if(NEMSdatm)
+if(CDEPS OR NEMSdatm)
   target_link_libraries(ufs_model PUBLIC ufs w3nco::w3nco_d)
 endif()
 

tests/default_vars.sh

@@ -57,6 +57,14 @@ elif [[ $MACHINE_ID = wcoss_dell_p3 || $MACHINE_ID = wcoss2 ]]; then
   MPB_datm_025="0 39"; APB_datm_025="0 39"
   OPB_datm_025="40 159"; IPB_datm_025="160 207"
 
+  TASKS_cdeps_100=40; TPN_cdeps_100=28
+  MPB_cdeps_100="0 11"; APB_cdeps_100="0 11"
+  OPB_cdeps_100="12 27"; IPB_cdeps_100="28 39"
+
+  TASKS_cdeps_025=208; TPN_cdeps_025=28
+  MPB_cdeps_025="0 39"; APB_cdeps_025="0 39"
+  OPB_cdeps_025="40 159"; IPB_cdeps_025="160 207"
+
 elif [[ $MACHINE_ID = orion.* ]]; then
 
   TASKS_dflt=150 ; TPN_dflt=40 ; INPES_dflt=3 ; JNPES_dflt=8
@@ -100,6 +108,14 @@ elif [[ $MACHINE_ID = orion.* ]]; then
   MPB_datm_025="0 39"; APB_datm_025="0 39"
   OPB_datm_025="40 159"; IPB_datm_025="160 207"
 
+  TASKS_cdeps_100=40; TPN_cdeps_100=40
+  MPB_cdeps_100="0 11"; APB_cdeps_100="0 11"
+  OPB_cdeps_100="12 27"; IPB_cdeps_100="28 39"
+
+  TASKS_cdeps_025=208; TPN_cdeps_025=40
+  MPB_cdeps_025="0 39"; APB_cdeps_025="0 39"
+  OPB_cdeps_025="40 159"; IPB_cdeps_025="160 207"
+
 elif [[ $MACHINE_ID = hera.* ]]; then
 
   TASKS_dflt=150 ; TPN_dflt=40 ; INPES_dflt=3 ; JNPES_dflt=8
@@ -143,6 +159,14 @@ elif [[ $MACHINE_ID = hera.* ]]; then
   MPB_datm_025="0 39"; APB_datm_025="0 39"
   OPB_datm_025="40 159"; IPB_datm_025="160 207"
 
+  TASKS_cdeps_100=40; TPN_cdeps_100=40
+  MPB_cdeps_100="0 11"; APB_cdeps_100="0 11"
+  OPB_cdeps_100="12 27"; IPB_cdeps_100="28 39"
+
+  TASKS_cdeps_025=208; TPN_cdeps_025=40
+  MPB_cdeps_025="0 39"; APB_cdeps_025="0 39"
+  OPB_cdeps_025="40 159"; IPB_cdeps_025="160 207"
+
 elif [[ $MACHINE_ID = linux.* ]]; then
 
   if [[ $CI_TEST = true ]]; then
@@ -204,6 +228,14 @@ elif [[ $MACHINE_ID = gaea.* ]]; then
   MPB_datm_025="0 39"; APB_datm_025="0 39"
   OPB_datm_025="40 159"; IPB_datm_025="160 207"
 
+  TASKS_cdeps_100=40; TPN_cdeps_100=36
+  MPB_cdeps_100="0 11"; APB_cdeps_100="0 11"
+  OPB_cdeps_100="12 27"; IPB_cdeps_100="28 39"
+
+  TASKS_cdeps_025=208; TPN_cdeps_025=36
+  MPB_cdeps_025="0 39"; APB_cdeps_025="0 39"
+  OPB_cdeps_025="40 159"; IPB_cdeps_025="160 207"
+
 elif [[ $MACHINE_ID = cheyenne.* ]]; then
 
   TASKS_dflt=150 ; TPN_dflt=36 ; INPES_dflt=3 ; JNPES_dflt=8
@@ -247,6 +279,14 @@ elif [[ $MACHINE_ID = cheyenne.* ]]; then
   MPB_datm_025="0 39"; APB_datm_025="0 39"
   OPB_datm_025="40 159"; IPB_datm_025="160 207"
 
+  TASKS_cdeps_100=40; TPN_cdeps_100=36
+  MPB_cdeps_100="0 11"; APB_cdeps_100="0 11"
+  OPB_cdeps_100="12 27"; IPB_cdeps_100="28 39"
+
+  TASKS_cdeps_025=208; TPN_cdeps_025=36
+  MPB_cdeps_025="0 39"; APB_cdeps_025="0 39"
+  OPB_cdeps_025="40 159"; IPB_cdeps_025="160 207"
+
 elif [[ $MACHINE_ID = stampede.* ]]; then
 
   TASKS_dflt=150 ; TPN_dflt=48 ; INPES_dflt=3 ; JNPES_dflt=8
@@ -289,6 +329,14 @@ elif [[ $MACHINE_ID = stampede.* ]]; then
   MPB_datm_025="0 39"; APB_datm_025="0 39"
   OPB_datm_025="40 159"; IPB_datm_025="160 207"
 
+  TASKS_cdeps_100=40; TPN_cdeps_100=48
+  MPB_cdeps_100="0 11"; APB_cdeps_100="0 11"
+  OPB_cdeps_100="12 27"; IPB_cdeps_100="28 39"
+
+  TASKS_cdeps_025=208; TPN_cdeps_025=48
+  MPB_cdeps_025="0 39"; APB_cdeps_025="0 39"
+  OPB_cdeps_025="40 159"; IPB_cdeps_025="160 207"
+
 else
 
   echo "Unknown MACHINE_ID ${MACHINE_ID}"
@@ -307,7 +355,8 @@ export_fv3 ()
 {
 export FV3=true
 export S2S=false
-export DATM=false
+export DATM_NEMS=false
+export DATM_CDEPS=false
 export THRD=1
 export WLCLK=$WLCLK_dflt
 export INPES=$INPES_dflt
@@ -474,7 +523,8 @@ export_cpl ()
 {
 export FV3=true
 export S2S=true
-export DATM=false
+export DATM_NEMS=false
+export DATM_CDEPS=false
 
 export DAYS="1"
 export FHMAX="24"
@@ -619,7 +669,8 @@ export_datm ()
 {
 export FV3=false
 export S2S=false
-export DATM=true
+export DATM_NEMS=true
+export DATM_CDEPS=false
 export DAYS=1
 export FHMAX=24
 export WLCLK=30
@@ -643,7 +694,7 @@ export NY_GLB=320
 # nems.configure
 export NEMS_CONFIGURE="nems.configure.datm.IN"
 export med_model="nems"
-export atm_model="datm"
+export atm_model="nems_datm"
 export ocn_model="mom6"
 export ice_model="cice6"
 export atm_petlist_bounds=$APB_datm_100
@@ -718,3 +769,112 @@ export CICE_HIST_AVG='.true.'
 export BL_SUFFIX=""
 export RT_SUFFIX=""
 }
+export_datm_cdeps ()
+{
+export FV3=false
+export S2S=false
+export DATM_NEMS=false
+export DATM_CDEPS=true
+export DAYS=1
+export FHMAX=24
+export WLCLK=30
+export THRD=1
+export FHROT='0'
+export WARM_START=.F.
+
+# atm/ocn/ice resolution
+export IATM=1760
+export JATM=880
+export ATM_NX_GLB=$IATM
+export ATM_NY_GLB=$JATM
+export ATMRES='1760x880'
+export OCNRES='100'
+export ICERES='1.00'
+export NX_GLB=360
+export NY_GLB=320
+
+# nems.configure
+export NEMS_CONFIGURE="nems.configure.datm_cdeps.IN"
+export med_model="nems"
+export atm_model="datm"
+export ocn_model="mom6"
+export ice_model="cice6"
+export atm_petlist_bounds=$APB_cdeps_100
+export med_petlist_bounds=$MPB_cdeps_100
+export ocn_petlist_bounds=$OPB_cdeps_100
+export ice_petlist_bounds=$IPB_cdeps_100
+export TASKS=$TASKS_cdeps_100
+export TPN=$TPN_cdeps_100
+export NPROC_ICE='12'
+
+export ENS_NUM=1
+export SYEAR='2011'
+export SMONTH='10'
+export SDAY='01'
+export SHOUR='00'
+export CDATE=${SYEAR}${SMONTH}${SDAY}${SHOUR}
+
+export NFHOUT=6
+export FDIAG=6
+export DT_ATMOS='900'
+export DT_CICE=${DT_ATMOS}
+export DT_DYNAM_MOM6='1800'
+export DT_THERM_MOM6='3600'
+export CPL_SLOW=${DT_THERM_MOM6}
+export CPL_FAST=${DT_ATMOS}
+export coupling_interval_slow_sec=${CPL_SLOW}
+export coupling_interval_fast_sec=${CPL_FAST}
+
+export RESTART_N=${FHMAX}
+export CPLMODE='nems_orig_data'
+export cap_dbug_flag="0"
+export use_coldstart=".false."
+export use_mommesh=".false."
+export RUNTYPE='startup'
+export flux_convergence='0.0'
+export flux_iteration='2'
+export flux_scheme='0'
+
+export INPUT_NML=input.mom6.nml.IN
+export MODEL_CONFIGURE=datm_cdeps_configure.IN
+export FIELD_TABLE="field_table"
+export DIAG_TABLE="diag_table_template"
+
+# atm defaults
+export DATM_SRC="CFSR"
+export FILENAME_BASE='cfsr.'
+export mesh_file="cfsr_mesh.nc"
+export MESH_ATM="DATM_INPUT/${mesh_file}"
+export atm_datamode=${DATM_SRC}
+
+# MOM6 defaults; 1 degree
+export MOM_INPUT=MOM_input_template_100
+export MOM6_RESTART_SETTING='n'
+export MOM6_RIVER_RUNOFF='False'
+export FRUNOFF=""
+export CHLCLIM='"seawifs_1998-2006_smoothed_2X.nc"'
+# this must be set False for restart repro
+export MOM6_REPRO_LA='False'
+# since CPL_SLOW is set to DT_THERM, this should be always be false
+export MOM6_THERMO_SPAN='False'
+# no WW3
+export MOM6_USE_WAVES='False'
+
+# CICE6 defaults; 1 degree
+export MESHOCN_ICE="mesh.mx${OCNRES}.nc"
+export CICEGRID="grid_cice_NEMS_mx${OCNRES}.nc"
+export CICEMASK="kmtu_cice_NEMS_mx${OCNRES}.nc"
+export RUNID='unknown'
+# set large; restart frequency now controlled by restart_n in nems.configure
+export DUMPFREQ='d'
+export DUMPFREQ_N=1000
+export USE_RESTART_TIME='.false.'
+export RESTART_EXT='.false.'
+# setting to true will allow Frazil FW and Salt to be
+# included in fluxes sent to ocean
+export FRAZIL_FWSALT='.true.'
+# default to write CICE average history files
+export CICE_HIST_AVG='.true.'
+export BL_SUFFIX=""
+export RT_SUFFIX=""
+}

tests/fv3_conf/cpld_datm_cdeps.IN

@@ -0,0 +1,39 @@
+mkdir INPUT DATM_INPUT RESTART history MOM6_OUTPUT
+
+# DATM fixed input
+ln -s  @[INPUTDATA_ROOT]/DATM_CDEPS/@[DATM_SRC]/@[mesh_file] DATM_INPUT
+ln -s  @[INPUTDATA_ROOT]/DATM_CDEPS/@[DATM_SRC]/201110/*201110*nc DATM_INPUT
+
+# MOM6 fixed input
+ICERES="${OCNRES:0:1}.${OCNRES:1}"
+cp    @[INPUTDATA_ROOT]/MOM6_FIX/@[OCNRES]/* ./INPUT
+cp    @[INPUTDATA_ROOT]/MOM6_FIX_DATM/@[OCNRES]/* ./INPUT
+
+# CICE fixed input
+cp    @[INPUTDATA_ROOT]/CICE_FIX/@[OCNRES]/grid_cice_NEMS_mx@[OCNRES].nc .
+cp    @[INPUTDATA_ROOT]/CICE_FIX/@[OCNRES]/kmtu_cice_NEMS_mx@[OCNRES].nc .
+cp    @[INPUTDATA_ROOT]/CICE_FIX/@[OCNRES]/mesh.mx@[OCNRES].nc .
+
+if [ $WARM_START = .F. ]; then
+# cold start
+ if [[ $OCNRES == '025' ]]; then
+  cp    @[INPUTDATA_ROOT]/MOM6_IC/@[OCNRES]/2011100100/MOM*.nc ./INPUT
+  cp    @[INPUTDATA_ROOT]/CICE_IC/@[OCNRES]/cice_model_@[ICERES].cpc*.nc cice_model.res.nc
+ else
+  cp    @[INPUTDATA_ROOT]/MOM6_IC/@[OCNRES]/2011100100/MOM6_IC_TS*.nc ./INPUT/MOM6_IC_TS.nc
+  cp    @[INPUTDATA_ROOT]/CICE_IC/@[OCNRES]/cice_model_@[ICERES].cpc*.nc cice_model.res.nc
+ fi
+else
+# warm start
+ cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/MOM.res.2011-10-01-12*.nc ./INPUT/MOM.res.nc
+
+# CMEPS restart and pointer files
+ RFILE="DATM_${DATM_SRC}.cpl.r.2011-10-01-43200.nc"
+ cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/${RFILE} .
+ ls -1 ${RFILE}>rpointer.cpl
+
+# CICE restart and pointer files
+ cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/iced.2011-10-01-43200.nc ./INPUT
+ RFILE="iced.2011-10-01-43200.nc"
+ ls -1 "./INPUT/"${RFILE}>ice.restart_file
+fi

tests/fv3_conf/cpld_datm_cfsr.IN

@@ -1,6 +1,5 @@
 mkdir INPUT DATM_INPUT RESTART history MOM6_OUTPUT
 
-export DATM=true
 # DATM fixed input
 export IATM=1760
 export JATM=880