Bring in CDEPS, CMEPS/CDEPS component level PIO initialization and add HAFS specific RTs

.gitmodules

@@ -44,5 +44,5 @@
   branch = develop
 [submodule "CDEPS"]
   path = CDEPS-interface/CDEPS
-  url = https://github.com/hafs-community/CDEPS
-  branch = emc/develop
+  url = https://github.com/hafs-community/CDEPS.git
+  branch = feature/hafs_couplehycom_cdeps

CMakeLists.txt

@@ -16,7 +16,11 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 ###############################################################################
 
 # Valid applications and choices
-list(APPEND VALID_APPS ATM ATMW S2S S2SW HAFS HAFSW DATM DATM_NEMS)
+list(APPEND VALID_APPS 
+     ATM ATMW 
+     S2S S2SW 
+     HAFS HAFSW HAFS_DATM HAFS_DOCN 
+     DATM DATM_NEMS)
 set(APP NONE CACHE BOOL "Application Name")
 if(NOT (APP IN_LIST VALID_APPS))
   message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
@@ -134,7 +138,7 @@ if(INLINE_POST)
 endif()
 
 # See https://github.com/NOAA-EMC/NCEPLIBS-nemsio/pull/22
-target_link_libraries(nemsio::nemsio INTERFACE w3emc::w3emc_d bacio::bacio_4)
+target_link_libraries(nemsio::nemsio INTERFACE w3nco::w3nco_d w3emc::w3emc_d bacio::bacio_4)
 
 # Configure Python
 find_package(Python 3 REQUIRED COMPONENTS Interpreter)
@@ -311,11 +315,17 @@ if(CMEPS)
 endif()
 
 if(CDEPS)
-  add_dependencies(ufs cdeps::datm)
-  list(APPEND _ufs_defs_private CDEPS-interface/CDEPS
-                                FRONT_CDEPS_DATM=atm_comp_nuopc)
-  include_directories(${CMAKE_CURRENT_BINARY_DIR}/CDEPS-interface/CDEPS/datm)
-  target_link_libraries(ufs PUBLIC cdeps::datm)
+   add_dependencies(ufs cdeps::datm)
+   list(APPEND _ufs_defs_private CDEPS-interface/CDEPS
+                                 FRONT_CDEPS_DATM=cdeps_datm_comp)
+   include_directories(${CMAKE_CURRENT_BINARY_DIR}/CDEPS-interface/CDEPS/datm)
+   target_link_libraries(ufs PUBLIC cdeps::datm)
+
+   add_dependencies(ufs cdeps::docn)
+   list(APPEND _ufs_defs_private CDEPS-interface/CDEPS
+                                 FRONT_CDEPS_DOCN=cdeps_docn_comp)
+   include_directories(${CMAKE_CURRENT_BINARY_DIR}/CDEPS-interface/CDEPS/docn)
+   target_link_libraries(ufs PUBLIC cdeps::docn)
 endif()
 
 target_compile_definitions(ufs PRIVATE "${_ufs_defs_private}")
@@ -349,6 +359,13 @@ target_link_libraries(ufs_model PRIVATE ufs
                                         esmf
                                         NetCDF::NetCDF_Fortran)
 
+if(S2S OR CMEPS OR CDEPS)
+  if(PNETCDF)
+    target_link_libraries (ufs_model PRIVATE ${PNETCDF_LIBRARY_DIRS})
+    target_include_directories(ufs_model PRIVATE ${PNETCDF_INCLUDE_DIRS})
+  endif()
+endif()
+
 ###############################################################################
 ### Install
 ###############################################################################

HYCOM-interface/hycom_files.cmake

@@ -66,4 +66,3 @@ list(APPEND _hycom_nuopc_src_files
 list(APPEND _hycom_offline_src_files
   HYCOM/hycom.F90
 )
-

cmake/configure_apps.cmake

@@ -14,56 +14,64 @@
 ### Configure Application Components
 ###############################################################################
 if(APP MATCHES "^(ATM|ATMW)$")
-  set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
-  set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
-  set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
+  set(FMS          ON  CACHE BOOL "Enable FMS"                 FORCE)
+  set(FV3          ON  CACHE BOOL "Enable FV3"                 FORCE)
+  set(STOCH_PHYS   ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
   if(APP MATCHES "ATMW")
-    set(WW3      ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
+    set(WW3        ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
     message("Configuring UFS app in Atmosphere with Waves mode")
   else()
     message("Configuring UFS app in Atmosphere Only mode")
   endif()
 endif()
 
 if(APP MATCHES "^(DATM|DATM_NEMS)$")
-  set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
-  set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
-  set(MOM6       ON  CACHE BOOL "Enable MOM6"                FORCE)
-  set(CICE6      ON  CACHE BOOL "Enable CICE6"               FORCE)
+  set(CMEPS        ON  CACHE BOOL "Enable CMEPS"               FORCE)
+  set(FMS          ON  CACHE BOOL "Enable FMS"                 FORCE)
+  set(MOM6         ON  CACHE BOOL "Enable MOM6"                FORCE)
+  set(CICE6        ON  CACHE BOOL "Enable CICE6"               FORCE)
   if(APP MATCHES "DATM_NEMS")
-    set(NEMSdatm ON  CACHE BOOL "Enable NEMS DataAtm"        FORCE)
+    set(NEMSdatm   ON  CACHE BOOL "Enable NEMS DataAtm"        FORCE)
     message("Configuring UFS app in (NEMS) Data Atmosphere mode")
   elseif(APP MATCHES "DATM")
-    set(CDEPS    ON  CACHE BOOL "Enable CDEPS"               FORCE)
+    set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)
     message("Configuring UFS app in (CDEPS) Data Atmosphere mode")
   endif()
 endif()
 
 if(APP MATCHES "^(S2S|S2SW)$")
-  set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
-  set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
-  set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
-  set(MOM6       ON  CACHE BOOL "Enable MOM6"                FORCE)
-  set(CICE6      ON  CACHE BOOL "Enable CICE6"               FORCE)
-  set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
+  set(CMEPS        ON  CACHE BOOL "Enable CMEPS"               FORCE)
+  set(FMS          ON  CACHE BOOL "Enable FMS"                 FORCE)
+  set(FV3          ON  CACHE BOOL "Enable FV3"                 FORCE)
+  set(MOM6         ON  CACHE BOOL "Enable MOM6"                FORCE)
+  set(CICE6        ON  CACHE BOOL "Enable CICE6"               FORCE)
+  set(STOCH_PHYS   ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
   if(APP MATCHES "S2SW")
-    set(WW3      ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
+    set(WW3        ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
     message("Configuring UFS app in S2S with Waves mode")
   else()
     message("Configuring UFS app in S2S mode")
   endif()
 endif()
 
-if(APP MATCHES "^(HAFS|HAFSW)$")
-  set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
-  set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
-  set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
-  set(HYCOM      ON  CACHE BOOL "Enable HYCOM"               FORCE)
-  set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
+if(APP MATCHES "^(HAFS|HAFSW|HAFS_DATM|HAFS_DOCN)$")
+  set(CMEPS        ON  CACHE BOOL "Enable CMEPS"               FORCE)
+  if(APP MATCHES "HAFS_DATM")
+    set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)
+    message("Configuring UFS app in HAFS with Data Atmosphere mode")
+  else()
+    set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
+    set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
+    set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
+  endif()
+  if(APP MATCHES "HAFS_DOCN")
+    set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)
+    message("Configuring UFS app in HAFS with Data Ocean mode")
+  else()
+    set(HYCOM      ON  CACHE BOOL "Enable HYCOM"               FORCE)
+  endif()
   if(APP MATCHES "HAFSW")
-    set(WW3      ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
+    set(WW3        ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
     message("Configuring UFS app in HAFS with Waves mode")
-  else()
-    message("Configuring UFS app in HAFS mode")
   endif()
 endif()

tests/compile.sh

@@ -96,6 +96,10 @@ if [[ "${MAKE_OPT}" == *"OPENMP=N"* ]]; then
   CMAKE_FLAGS="${CMAKE_FLAGS} -DOPENMP=OFF"
 fi
 
+if [[ "${MAKE_OPT}" == *"PNETCDF=Y"* ]]; then
+  CMAKE_FLAGS="${CMAKE_FLAGS} -DPNETCDF=Y"
+fi
+
 if [[ "${MAKE_OPT}" == *"MULTI_GASES=Y"* ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DMULTI_GASES=ON"
 else
@@ -124,7 +128,7 @@ else
   fi
 fi
 
-  # Check if suites argument is provided or not
+# Check if suites argument is provided or not
 set +ex
 TEST=$( echo $MAKE_OPT | grep -e "SUITES=" )
 if [[ $? -eq 0 ]]; then
@@ -134,36 +138,47 @@ fi
 set -ex
 
 # Valid applications
-if [[ "${MAKE_OPT}" == *"APP=ATM"* ]]; then
+APP_OPT=$( echo $MAKE_OPT | tr " " "\n" | sed -n '/APP/p;/APP/q' )
+echo "APP_OPT = ${APP_OPT}"
+
+if [[ "${APP_OPT}" == "APP=ATM" ]]; then
     echo "MAKE_OPT = ${MAKE_OPT}"
     CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=ATM"
 fi
 
-if [[ "${MAKE_OPT}" == *"APP=ATMW"* ]]; then
+if [[ "${APP_OPT}" == "APP=ATMW" ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=ATMW"
 fi
 
-if [[ "${MAKE_OPT}" == *"APP=S2S"* ]]; then
+if [[ "${APP_OPT}" == "APP=S2S" ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=S2S -DMOM6SOLO=ON"
 fi
 
-if [[ "${MAKE_OPT}" == *"APP=S2SW"* ]]; then
+if [[ "${APP_OPT}" == "APP=S2SW" ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=S2SW -DMOM6SOLO=ON"
 fi
 
-if [[ "${MAKE_OPT}" == *"APP=HAFS"* ]]; then
+if [[ "${APP_OPT}" == "APP=HAFS" ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=HAFS"
 fi
 
-if [[ "${MAKE_OPT}" == *"APP=HAFSW"* ]]; then
+if [[ "${APP_OPT}" == "APP=HAFSW" ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=HAFSW"
 fi
 
-if [[ "${MAKE_OPT}" == *"APP=DATM"* ]]; then
+if [[ "${APP_OPT}" == "APP=HAFS_DATM" ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=HAFS_DATM"
+fi
+
+if [[ "${APP_OPT}" == "APP=HAFS_DOCN" ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=HAFS_DOCN"
+fi
+
+if [[ "${APP_OPT}" == "APP=DATM" ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=DATM"
 fi
 
-if [[ "${MAKE_OPT}" == *"APP=DATM_NEMS"* ]]; then
+if [[ "${APP_OPT}" == "APP=DATM_NEMS" ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DAPP=DATM_NEMS"
 fi
 

tests/default_vars.sh

@@ -401,6 +401,7 @@ export FV3=true
 export S2S=false
 export DATM_NEMS=false
 export DATM_CDEPS=false
+export DOCN_CDEPS=false
 export THRD=1
 export WLCLK=$WLCLK_dflt
 export INPES=$INPES_dflt
@@ -734,6 +735,7 @@ export FV3=false
 export S2S=false
 export DATM_NEMS=true
 export DATM_CDEPS=false
+export DOCN_CDEPS=false
 export CPLWAV=.F.
 export DAYS=1
 export FHMAX=24
@@ -945,3 +947,93 @@ export CICE_HIST_AVG='.true.'
 export BL_SUFFIX=""
 export RT_SUFFIX=""
 }
+export_cdeps_datm ()
+{
+export FV3=false
+export CDEPS_DATM=true
+export DATM=false
+export THRD=1
+export WLCLK=$WLCLK_dflt
+export INPES=$INPES_dflt
+export JNPES=$JNPES_dflt
+export TASKS=$TASKS_dflt
+export TPN=$TPN_dflt
+
+export atm_model="datm"
+
+export DATM_CONFIGURE_A="datm_in"
+export DATM_CONFIGURE_B="datm.streams.xml.era5"
+}
+export_cdeps_docn ()
+{
+export CDEPS_DOCN=true
+export THRD=1
+export WLCLK=$WLCLK_dflt
+export INPES=$INPES_dflt
+export JNPES=$JNPES_dflt
+export TASKS=$TASKS_dflt
+export TPN=$TPN_dflt
+
+export ocn_model="docn"
+export ocn_datamode="sstdata"
+
+export DOCN_CONFIGURE_A="docn_in"
+export DOCN_CONFIGURE_B="docn.streams.xml.IN"
+}
+export_cpl_regional ()
+{
+export S2S=false
+export CMEPS=true
+
+# model_configure
+export SYEAR='2019'
+export SMONTH='08'
+export SDAY='29'
+export SHOUR='00'
+export FHMAX=24
+export ENS_NUM=1
+export DT_ATMOS='900'
+export CPL='.true.'
+export RESTART_INTERVAL=0
+export FHROT=0
+export coupling_interval_fast_sec=0
+export QUILTING=.true.
+export WRITE_GROUP=1
+export WRTTASK_PER_GROUP=6
+export OUTPUT_HISTORY=.true.
+export WRITE_DOPOST=.false.
+export NUM_FILES=2
+export FILENAME_BASE="'dyn' 'phy'"
+export OUTPUT_GRID="'regional_latlon'"
+export OUTPUT_FILE="'netcdf'"
+export IDEFLATE=0
+export NBITS=0
+export WRITE_NEMSIOFLIP=.false.
+export WRITE_FSYNCFLAG=.false.
+export NFHOUT=3
+export NFHMAX_HF=-1
+export NFHOUT_HF=3
+export CEN_LON=-62.0
+export CEN_LAT=25.0
+export LON1=-114.5
+export LAT1=-5.0
+export LON2=-9.5
+export LAT2=55.0
+export DLON=0.03
+export DLAT=0.03
+
+# input.nml
+export MERGE_IMPORT=.true.
+
+# nems.configure
+export med_model="nems"
+export CAP_DBUG_FLAG="0"
+export RESTART_N=${FHMAX} 
+export CPLMODE="hafs"
+export RUNTYPE="startup"
+export USE_COLDSTART="false"
+
+# timing
+export ESMF_RUNTIME_PROFILE=ON
+export ESMF_RUNTIME_PROFILE_OUTPUT=SUMMARY
+}

tests/fv3_conf/ccpp_regional_natl_run.IN

@@ -0,0 +1,5 @@
+
+rm -rf INPUT RESTART
+mkdir  INPUT RESTART
+
+rsync -arv @[RTPWD]/FV3_regional_c192_natl_input_data/. .

tests/fv3_conf/cpld_datm_cdeps_era5.IN

@@ -0,0 +1,10 @@
+if [ -d "input" ]; then
+  echo "datm/ directory exists ..."
+else
+  echo "datm/ directory not found. Creating ..."
+  mkdir -p datm
+fi
+
+cd datm
+rsync -arv @[RTPWD]/DATM_ERA5_input_data/. . 
+cd -