Add data model support to HAFS workflow

.gitignore

@@ -7,3 +7,9 @@
 *.exe
 *.x
 *.log
+*~
+*.db
+*.db.bak
+system.conf
+fix/
+rocoto/*.xml

jobs/JHAFS_ATM_PREP

@@ -62,8 +62,13 @@ mkdir -p $OUTDIR $DATA
 cd $DATA
 
 # Execute ex-script
-${HOMEhafs}/scripts/exhafs_atm_prep.sh
-export err=$?
+if [[ "${run_datm:-no}" == yes ]] ; then
+    ${HOMEhafs}/scripts/exhafs_datm_prep.sh
+    export err=$?
+else
+    ${HOMEhafs}/scripts/exhafs_atm_prep.sh
+    export err=$?
+fi
 exit $err
 
 export KEEPDATA=${KEEPDATA:-YES}

jobs/JHAFS_OCN_PREP

@@ -51,8 +51,13 @@ mkdir -p $DATA
 cd $DATA
 
 # Execute ex-script
-${HOMEhafs}/scripts/exhafs_ocn_prep.py
-export err=$?
+if [[ "${run_docn:-no}" == yes ]] ; then
+    ${HOMEhafs}/scripts/exhafs_docn_prep.sh
+    export err=$?
+else
+    ${HOMEhafs}/scripts/exhafs_ocn_prep.py
+    export err=$?
+fi
 exit $err
 
 export KEEPDATA=${KEEPDATA:-YES}

modulefiles/modulefile.hafs.hera

@@ -69,3 +69,6 @@ module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles
 module load rocoto/1.3.3
 
 module load intelpython/3.6.8
+
+# For CDEPS data models:
+module load cdo/1.9.10

modulefiles/modulefile.hafs.jet

@@ -65,3 +65,6 @@ module load nco/4.9.1
 module load rocoto/1.3.3
 
 module load intelpython/3.6.5
+
+# For CDEPS data models:
+module load cdo

modulefiles/modulefile.hafs.orion

@@ -65,3 +65,6 @@ module load nco/4.9.3
 module load rocoto/1.3.3
 
 module load intelpython3/2020
+
+# For CDEPS data models:
+module load cdo

modulefiles/modulefile.hafs.wcoss_dell_p3

@@ -73,3 +73,6 @@ setenv CMAKE_Platform wcoss_dell_p3
 module use /usrx/local/dev/emc_rocoto/modulefiles
 module load ruby/2.5.1
 module load rocoto/1.3.0rc2
+
+# For CDEPS data models:
+module load cdo

parm/cdeps/datm_era5.streams

@@ -0,0 +1,15 @@
+stream_info:               ERA5_HOURLY01
+taxmode01:                 limit 
+mapalgo01:                 redist
+tInterpAlgo01:             linear
+readMode01:                single
+dtlimit01:                 1.5
+stream_offset01:           0
+yearFirst01:               _yearFirst_ 
+yearLast01:                _yearLast_
+yearAlign01:               _yearFirst_
+stream_vectors01:          "u:v"
+stream_mesh_file01:        _mesh_atm_ 
+stream_lev_dimname01:      null
+stream_data_files01:      
+stream_data_variables01:   "u10 Sa_u10m" "v10 Sa_v10m" "t2m Sa_t2m" "skt Sa_tskn" "d2m Sa_tdew" "msl Sa_pslv" "tp Faxa_rain" "cp Faxa_rainc" "lsp Faxa_rainl" "csf Faxa_snowc" "lsf Faxa_snowl" "ssrd Faxa_swdn" "ssr Faxa_swnet" "strd Faxa_lwdn" "str Faxa_lwnet" "aluvp Faxa_swvdr" "aluvd Faxa_swvdf" "alnip Faxa_swndr" "alnid Faxa_swndf" "sshf Faxa_sen" "slhf Faxa_lat" "ewss Faxa_taux" "nsss Faxa_tauy"

parm/cdeps/datm_in

@@ -0,0 +1,14 @@
+&datm_nml
+  datamode = "ERA5"
+  factorfn_data = "null"
+  factorfn_mesh = "null"
+  flds_co2 = .false.
+  flds_presaero = .false.
+  flds_wiso = .false.
+  iradsw = 1
+  model_maskfile = "_mesh_atm_"
+  model_meshfile = "_mesh_atm_"
+  nx_global = 1440
+  ny_global = 721
+  restfilm = "null"
+/

parm/cdeps/docn_ghrsst.streams

@@ -0,0 +1,15 @@
+stream_info:               PRESCRIBED01
+taxmode01:                 limit
+mapalgo01:                 redist
+tInterpAlgo01:             linear
+readMode01:                single
+dtlimit01:                 1.5
+stream_offset01:           0
+yearFirst01:               _yearFirst_
+yearLast01:                _yearLast_
+yearAlign01:               _yearFirst_
+stream_vectors01:          "null"
+stream_mesh_file01:        "INPUT/DOCN_ESMF_mesh.nc"
+stream_lev_dimname01:      null
+stream_data_files01:       
+stream_data_variables01:   "analysed_sst So_t"

parm/cdeps/docn_in

@@ -0,0 +1,9 @@
+&docn_nml
+  datamode = "sstdata"
+  model_maskfile = "_mesh_ocn_"
+  model_meshfile = "_mesh_ocn_"
+  nx_global = _nx_global_
+  ny_global = _ny_global_
+  restfilm = "null"
+  sst_constant_value = -1.0
+/

parm/cdeps/docn_oisst.streams

@@ -0,0 +1,15 @@
+stream_info:               PRESCRIBED01
+taxmode01:                 limit
+mapalgo01:                 redist
+tInterpAlgo01:             linear
+readMode01:                single
+dtlimit01:                 1.5
+stream_offset01:           0
+yearFirst01:               _yearFirst_
+yearLast01:                _yearLast_
+yearAlign01:               _yearFirst_
+stream_vectors01:          "null"
+stream_mesh_file01:        _mesh_ocn_
+stream_lev_dimname01:      null
+stream_data_files01:      
+stream_data_variables01:   "sst So_t"

parm/forecast/globnest/model_configure.tmp

@@ -1,4 +1,4 @@
-print_esmf:              .false.
+print_esmf:              _print_esmf_
 start_year:              YR
 start_month:             MN
 start_day:               DY

parm/forecast/globnest_hwrf/model_configure.tmp

@@ -1,4 +1,4 @@
-print_esmf:              .false.
+print_esmf:              _print_esmf_
 start_year:              YR
 start_month:             MN
 start_day:               DY

parm/forecast/regional/model_configure.tmp

@@ -1,4 +1,4 @@
-print_esmf:              .false.
+print_esmf:              _print_esmf_
 start_year:              YR
 start_month:             MN
 start_day:               DY

parm/forecast/regional/nems.configure.cdeps.tmp

@@ -0,0 +1,150 @@
+##############################################
+#####  NEMS Run-Time Configuration File  #####
+##############################################
+
+# EARTH #
+EARTH_component_list: MED ATM OCN
+EARTH_attributes::
+  Verbosity = 0
+::
+
+# MED #
+MED_model:                      cmeps
+_MED_petlist_bounds_
+MED_attributes::
+  Verbosity = 1
+  Diagnostic = 0
+  ATM_model = _atm_model_
+  OCN_model = _ocn_model_
+  MED_model = cmeps
+  history_n = 1
+  history_option = ndays
+  history_ymd = -999
+  coupling_mode = hafs
+  normalization = none
+  merge_type = copy
+::
+
+# ATM #
+ATM_model:                      _atm_model_
+_ATM_petlist_bounds_
+ATM_attributes::
+  Verbosity = 1
+  Diagnostic = 0
+  mesh_atm = _mesh_atm_
+::
+
+# OCN #
+OCN_model:                      _ocn_model_
+_OCN_petlist_bounds_
+OCN_attributes::
+  Verbosity = 1
+  Diagnostic = 0
+  mesh_ocn = _mesh_ocn_
+# The following are only used by the hycom ocean model. #
+  cdf_impexp_freq = 3
+  cpl_hour = 0
+  cpl_min = 0
+  cpl_sec = _cpl_dt_
+  base_dtg = _base_dtg_
+  merge_import = _merge_import_
+  skip_first_import = .true.
+  hycom_arche_output = .false.
+  hyc_esmf_exp_output = .true.
+  hyc_esmf_imp_output = .true.
+  import_diagnostics = .false.
+  import_setting = flexible
+  hyc_impexp_file = nems.configure
+  espc_show_impexp_minmax = .true.
+  ocean_start_dtg = _ocean_start_dtg_
+  start_hour = 0
+  start_min = 0
+  start_sec = 0
+  end_hour = _end_hour_
+  end_min = 0
+  end_sec = 0
+::
+
+# Run Sequence #
+runSeq::
+@_cpl_dt_
+  ATM -> MED :remapMethod=redist
+  MED med_phases_post_atm
+  OCN -> MED :remapMethod=redist
+  MED med_phases_post_ocn
+  MED med_phases_prep_atm
+  MED med_phases_prep_ocn_accum
+  MED med_phases_prep_ocn_avg
+  MED -> ATM :remapMethod=redist
+  MED -> OCN :remapMethod=redist
+  ATM
+  OCN
+  MED med_phases_restart_write
+  MED med_phases_history_write
+@
+::
+
+# Other Attributes #
+DRIVER_attributes::
+  start_type = startup
+::
+
+ALLCOMP_attributes::
+  ATM_model = _atm_model_
+  OCN_model = _ocn_model_
+  MED_model = cmeps
+  ScalarFieldCount = 3
+  ScalarFieldIdxGridNX = 1
+  ScalarFieldIdxGridNY = 2
+  ScalarFieldIdxNextSwCday = 3
+  ScalarFieldName = cpl_scalars
+  start_type = startup
+  case_name = ufs.hafs
+  restart_n = 1
+  restart_option = ndays
+  restart_ymd = -999
+  dbug_flag = 20
+  use_coldstart = true
+  orb_eccen = 1.e36
+  orb_iyear = 2000
+  orb_iyear_align = 2000
+  orb_mode = fixed_year
+  orb_mvelp = 1.e36
+  orb_obliq = 1.e36
+  mediator_read_restart = false
+  mediator_present = true
+::
+
+ATM_modelio::
+  diro = .
+  logfile = atm.log
+::
+
+OCN_modelio::
+  diro = .
+  logfile = ocn.log
+::
+
+MED_modelio::
+  diro = .
+  logfile = med.log
+::
+
+# The following are only used by the hycom ocean model. #
+ocn_export_fields::
+  'sst'         'sea_surface_temperature'   'K'
+  'mask'        'ocean_mask'                '1'
+  'cpl_scalars' 'cpl_scalars'               '1'
+::
+
+# The following are only used by the hycom ocean model. #
+ocn_import_fields::
+  'taux10'      'mean_zonal_moment_flx_atm' 'N_m-2'
+  'tauy10'      'mean_merid_moment_flx_atm' 'N_m-2'
+  'prcp'        'mean_prec_rate'            'kg_m-2_s-1'
+  'swflxd'      'mean_net_sw_flx'           'W_m-2'
+  'lwflxd'      'mean_net_lw_flx'           'W_m-2'
+  'mslprs'      'inst_pres_height_surface'  'Pa'
+  'sensflx'     'mean_sensi_heat_flx'       'W_m-2'
+  'latflx'      'mean_laten_heat_flx'       'W_m-2'
+::

parm/forecast/regional_hwrf/model_configure.tmp

@@ -1,4 +1,4 @@
-print_esmf:              .false.
+print_esmf:              _print_esmf_
 start_year:              YR
 start_month:             MN
 start_day:               DY

parm/hafs.conf

@@ -27,6 +27,9 @@ ENS=99                ;; The ensemble number (placeholder)
 # Specifies a section (default: [hafsdata]) to use: hafsdata, wcoss_fcst_nco
 input_catalog=fcst_{GFSVER}
 
+docn_source=NONE ;; Data source for data ocean model (GHRSST, OISST, or NONE)
+datm_source=NONE ;; Data source for data atmosphere model (ERA5 or NONE)
+
 ## Configure file and directory paths
 [dir]
 HOMEhafs={CDSAVE}/{EXPT}
@@ -57,9 +60,20 @@ gsistatus2=gsi_status.{vit[stormname]}{vit[stnum]:02d}{vit[basin1lc]}.{cycle}
 PARMforecast={PARMhafs}/forecast/regional   ;; The location where the forecast job will find its parm and namelist files
 PARMgsi={PARMhafs}/hafs-gsi/                ;; GSI input data for everything except CRTM
 FIXcrtm={FIXhafs}/hafs-crtm-2.2.3/          ;; GSI CRTM input data
+FIXcdeps={FIXhafs}/fix_cdeps                ;; CDEPS fix files
+FIXmeshes={FIXcdeps}/meshes                 ;; premade CDEPS meshes
 
 utilexec={HOMEhafs}/exec                    ;; utility exe location (placeholder)
 
+# Data model locations
+DOCNdir=/work/noaa/{disk_project}/{ENV[USER]}/DOCN
+DATMdir=/work/noaa/{disk_project}/{ENV[USER]}/DATM
+
+# Processed input files and meshes during workflow execution:
+docn_input_path={intercom}/cdeps
+datm_input_path={intercom}/cdeps
+
+
 ## Executable program locations
 # Currently not used in the workflow script system
 [exe]
@@ -226,6 +240,8 @@ npx=2561
 npy=2161
 npz=64
 
+print_esmf=.false.  ;; .true. to generate ESMF log files or .false. not to
+
 # The write_grid_component related options
 quilting=.true.
 write_groups=2
@@ -272,6 +288,15 @@ ocean_tasks=60 ;; Number of PEs for the OCN component
 ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31,0,0,0), e.g., 43340.00000
 merge_import=.true.
 
+# Data model defaults
+mesh_ocn_in=missing ;; premade mesh to use if make_mesh_ocn=no
+mesh_ocn_gen={WORKhafs}/intercom/cdeps/DOCN_ESMF_mesh.nc ;; do not change
+mesh_atm_in=missing ;; premade mesh to use if make_mesh_atm=no
+mesh_atm_gen={WORKhafs}/intercom/cdeps/DATM_ESMF_mesh.nc ;; do not change
+docn_mesh_nx_global=1440 ;; Dimensions of data ocean model in X direction
+docn_mesh_ny_global=720 ;; Dimensions of data ocean model in Y direction
+
+
 [forecast_ens]
 # ccpp suites
 ccpp_suite_regional_ens={forecast/ccpp_suite_regional}
@@ -368,6 +393,7 @@ COMgfs={dir/COMgfs}                 ;; input GFS com directory
 COMrtofs={dir/COMrtofs}             ;; input RTOFS com directory
 gtype={grid/gtype}                  ;; grid type: uniform, stretch, nest, or regional (currently only nest and regional have been tested and supported)
 GFSVER={config/GFSVER}              ;; Version of GFS input data, e.g., PROD2019, PROD2021
+
 # Specify the forecast job resources. Only a few combinations are provided. If
 # needed, you may add other options in the site entity files under rocoto/sites.
 #FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_omp2
@@ -385,9 +411,13 @@ RUN_ENSDA={run_ensda}               ;; Do we run the ensda system?
 RUN_ENKF={run_enkf}                 ;; Do we run the self-cycled ensda system with EnKF analysis
 RUN_OCEAN={run_ocean}               ;; Do we run with ocean coupling?
 RUN_WAVE={run_wave}                 ;; Do we run with wave coupling?
+RUN_DATM={run_datm}                 ;; Do we run with a data atmosphere using CDEPS?
+RUN_DOCN={run_docn}                 ;; Do we run with a data ocean using CDEPS?
+RUN_DWAV={run_dwav}                 ;; Do we run with data waves using CDEPS?
 RUN_VORTEXINIT={run_vortexinit}     ;; Do we enable vortex initialization?
 RUN_HRDGRAPHICS={run_hrdgraphics}   ;; Do we run HRD graphics?
 RUN_EMCGRAPHICS={run_emcgraphics}   ;; Do we run EMC graphics?
 SCRUB_COM={scrub_com}               ;; Should Rocoto scrub the COM directory?
 SCRUB_WORK={scrub_work}             ;; Should Rocoto scrub the WORK directory?
-
+MAKE_MESH_ATM={make_mesh_atm}       ;; Should the DATM mesh be generated by the workflow?
+MAKE_MESH_OCN={make_mesh_ocn}       ;; Should the DOCN mesh be generated by the workflow?