Add the CMEPS based HAFS-HYCOM coupling into the HAFS workflow

parm/forecast/globnest/nems.configure.atmonly

@@ -1,5 +1,18 @@
 EARTH_component_list: ATM
+
 ATM_model: fv3
+
 runSeq::
   ATM
 ::
+
+ATM_attributes::
+::
+
+DRIVER_attributes::
+  start_type = startup
+::
+
+ALLCOMP_attributes::
+  mediator_read_restart = false
+::

parm/forecast/regional/nems.configure.atm_ocn.tmp

@@ -12,10 +12,7 @@ _OCN_petlist_bounds_
 # run sequence
 runSeq::
     @_cpl_dt_
-      _runSeq_OCN2ATM_
-      _runSeq_ATM2OCN_
-      _runSeq_ATM_
-      _runSeq_OCN_
+ _runSeq_ALL_
     @
 ::
 
@@ -49,3 +46,35 @@ OCN_attributes::
   end_min = 0
   end_sec = 0
 ::
+
+DRIVER_attributes::
+  start_type = startup
+::
+
+ALLCOMP_attributes::
+  mediator_read_restart = false
+# ScalarFieldCount = 3
+# ScalarFieldIdxGridNX = 1
+# ScalarFieldIdxGridNY = 2
+# ScalarFieldIdxNextSwCday = 3
+# ScalarFieldName = cpl_scalars
+::
+
+ocn_export_fields::
+  'sst'         'sea_surface_temperature' 'K'       '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+# 'mask'        'So_omask'                '1'       '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+::
+
+ocn_import_fields::
+  'u10'    'inst_zonal_wind_height10m' 'm_s-1'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'v10'    'inst_merid_wind_height10m' 'm_s-1'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'taux10' 'mean_zonal_moment_flx_atm' 'N_m-2'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'tauy10' 'mean_merid_moment_flx_atm' 'N_m-2'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'airtmp' 'inst_temp_height2m'        'K'          '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'airhum' 'inst_spec_humid_height2m'  'kg_kg-1'    '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'prcp'   'mean_prec_rate'            'kg_m-2_s-1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'swflxd' 'mean_net_sw_flx'           'W_m-2'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'lwflxd' 'mean_net_lw_flx'           'W_m-2'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'mslprs' 'inst_pres_height_surface'  'Pa'         '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'gt'     'inst_temp_height_surface'  'K'          '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+::

parm/forecast/regional/nems.configure.atm_ocn_cmeps.tmp

@@ -0,0 +1,98 @@
+# component list
+EARTH_component_list: ATM OCN MED
+
+# component models
+ATM_model: fv3
+OCN_model: hycom
+MED_model: nems
+
+# component petLists
+_ATM_petlist_bounds_
+_OCN_petlist_bounds_
+_MED_petlist_bounds_
+
+# run sequence
+runSeq::
+    @_cpl_dt_
+ _runSeq_ALL_
+    @
+::
+
+# component attributes
+
+ATM_attributes::
+  Verbosity = 1
+  Diagnostic = 0
+::
+
+OCN_attributes::
+  Verbosity = 1
+  Diagnostic = 0
+  cdf_impexp_freq = 3
+  cpl_hour = 0
+  cpl_min = 0
+  cpl_sec = _cpl_dt_
+  base_dtg = _base_dtg_
+  merge_import = _merge_import_
+  skip_first_import = .true.
+  hycom_arche_output = .false.
+  hyc_esmf_exp_output = .true.
+  hyc_esmf_imp_output = .true.
+  hyc_impexp_file = nems.configure
+  espc_show_impexp_minmax = .true.
+  ocean_start_dtg = _ocean_start_dtg_
+  start_hour = 0
+  start_min = 0
+  start_sec = 0
+  end_hour = _end_hour_
+  end_min = 0
+  end_sec = 0
+::
+
+MED_attributes::
+  Verbosity = 1
+  Diagnostic = 0
+  coupling_mode = hafs
+  system_type = ufs
+  normalization = none
+  merge_type = copy
+  ATM_model = fv3
+  OCN_model = hycom
+  history_ymd = -999
+  ScalarFieldCount = 0
+  ScalarFieldIdxGridNX = 0
+  ScalarFieldIdxGridNY = 0
+  ScalarFieldName = cpl_scalars
+::
+
+DRIVER_attributes::
+  start_type = startup
+::
+
+ALLCOMP_attributes::
+  mediator_read_restart = false
+# ScalarFieldCount = 3
+# ScalarFieldIdxGridNX = 1
+# ScalarFieldIdxGridNY = 2
+# ScalarFieldIdxNextSwCday = 3
+# ScalarFieldName = cpl_scalars
+::
+
+ocn_export_fields::
+  'sst'         'sea_surface_temperature' 'K'       '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'mask'        'So_omask'                '1'       '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+::
+
+ocn_import_fields::
+  'u10'    'inst_zonal_wind_height10m' 'm_s-1'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'v10'    'inst_merid_wind_height10m' 'm_s-1'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'taux10' 'mean_zonal_moment_flx_atm' 'N_m-2'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'tauy10' 'mean_merid_moment_flx_atm' 'N_m-2'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'airtmp' 'inst_temp_height2m'        'K'          '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'airhum' 'inst_spec_humid_height2m'  'kg_kg-1'    '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'prcp'   'mean_prec_rate'            'kg_m-2_s-1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'swflxd' 'mean_net_sw_flx'           'W_m-2'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'lwflxd' 'mean_net_lw_flx'           'W_m-2'      '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'mslprs' 'inst_pres_height_surface'  'Pa'         '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+  'gt'     'inst_temp_height_surface'  'K'          '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1'
+::

parm/forecast/regional/nems.configure.atmonly

@@ -1,5 +1,18 @@
 EARTH_component_list: ATM
+
 ATM_model: fv3
+
 runSeq::
   ATM
 ::
+
+ATM_attributes::
+::
+
+DRIVER_attributes::
+  start_type = startup
+::
+
+ALLCOMP_attributes::
+  mediator_read_restart = false
+::

parm/hafs.conf

@@ -239,19 +239,18 @@ output_grid_dlat=0.03             ;; output grid spacing dlat . . . .
 #output_grid_dlon=0.03             ;; output grid spacing dlon . . . .
 #output_grid_dlat=0.03             ;; output grid spacing dlat . . . .
 
-cplflx=.false.
 # Ocean coupling related options. Only useful when run_ocean=yes
+# When run_ocean=yes, then the following options are available for cpl_ocean
+# 0, run ocean model side by side (no coupling)
+# 1, direct coupling through the nearest point regridding method
+# 2, direct coupling through the bilinear regridding method
+# 3, CMEPS based coupling through the bilinear regridding method
 cpl_ocean=0
-cpl_dt=360
-ocean_tasks=60
-runSeq_OCN2ATM=auto
-runSeq_ATM2OCN=auto
-runSeq_ATM=auto
-runSeq_OCN=auto
-ATM_petlist_bounds=auto
-OCN_petlist_bounds=auto
-ocean_start_dtg=auto
-merge_import=.false.
+cpl_dt=360  ;; coupling time step in seconds
+#ocean_tasks=120 ;; Number of PEs for the OCN component
+ocean_tasks=60 ;; Number of PEs for the OCN component
+ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31,0,0,0), e.g., 43340.00000 
+merge_import=.true.
 
 [post]
 # Grid definition for post and tracker, used by wgrib2

parm/hafs_holdvars.conf

@@ -85,14 +85,7 @@ synop_gridspecs={synop_gridspecs}
 trker_gridspecs={trker_gridspecs}
 
 cpl_dt={forecast/cpl_dt}
-cplflx={forecast/cplflx}
 cpl_ocean={forecast/cpl_ocean}
-runSeq_OCN2ATM={forecast/runSeq_OCN2ATM}
-runSeq_ATM2OCN={forecast/runSeq_ATM2OCN}
-runSeq_ATM={forecast/runSeq_ATM}
-runSeq_OCN={forecast/runSeq_OCN}
 ocean_tasks={forecast/ocean_tasks}
-ATM_petlist_bounds={forecast/ATM_petlist_bounds}
-OCN_petlist_bounds={forecast/OCN_petlist_bounds}
 ocean_start_dtg={forecast/ocean_start_dtg}
 merge_import={forecast/merge_import}

parm/hafs_holdvars.txt

@@ -116,14 +116,7 @@ export run_ocean={run_ocean}
 export ocean_model={ocean_model}
 export cpl_dt={cpl_dt}
 export cpl_ocean={cpl_ocean}
-export cplflx={cplflx}
-export runSeq_OCN2ATM={runSeq_OCN2ATM}
-export runSeq_ATM2OCN={runSeq_ATM2OCN}
-export runSeq_ATM={runSeq_ATM}
-export runSeq_OCN={runSeq_OCN}
 export ocean_tasks={ocean_tasks}
-export ATM_petlist_bounds={ATM_petlist_bounds}
-export OCN_petlist_bounds={OCN_petlist_bounds}
 export ocean_start_dtg={ocean_start_dtg}
 export merge_import={merge_import}
 

parm/hafs_hycom.conf

@@ -24,40 +24,19 @@ nstf_n4=0
 nstf_n5=0
 
 write_groups=1
-write_tasks_per_group=48
-
-#ocean_tasks=120 ;; Number of PEs for the OCN component
-ocean_tasks=60 ;; Number of PEs for the OCN component
-cpl_dt=360  ;; coupling time step in seconds
-cplflx=auto
+write_tasks_per_group=24
 
+# Ocean coupling related options. Only useful when run_ocean=yes
 # When run_ocean=yes, then the following options are available for cpl_ocean
 # 0, run ocean model side by side (no coupling)
 # 1, direct coupling through the nearest point regridding method
 # 2, direct coupling through the bilinear regridding method
+# 3, CMEPS based coupling through the bilinear regridding method
 cpl_ocean=2
-
-## By default, the following itmes will be generated automatically
-## However, they can be overriden by the users if values (other than auto) are given.
-## if run_ocean=yes and cpl_ocean=0
-#cplflx=.false.
-#runSeq_OCN2ATM=""
-#runSeq_ATM2OCN=""
-## if run_ocean=yes and cpl_ocean=1
-#cplflx=.true.
-#runSeq_OCN2ATM="OCN -> ATM :remapMethod=nearest_stod:srcmaskvalues=0"
-#runSeq_ATM2OCN="ATM -> OCN :remapMethod=nearest_stod:srcmaskvalues=1:dstmaskvalues=0"
-## if run_ocean=yes and cpl_ocean=2 (auto option not implemented yet)
-#cplflx=.true.
-#runSeq_OCN2ATM="OCN -> ATM :remapMethod=bilinear:unmappedaction=ignore:zeroregion=select:srcmaskvalues=0"
-#runSeq_ATM2OCN="ATM -> OCN :remapMethod=bilinear:unmappedaction=ignore:zeroregion=select:srcmaskvalues=1:dstmaskvalues=0"
-#
-#runSeq_ATM=auto
-#runSeq_OCN=auto
-#ATM_petlist_bounds=auto ;; e.g., "ATM_petlist_bounds: 0000 1319"
-#OCN_petlist_bounds=auto ;; e.g., "ATM_petlist_bounds: 1320 1439"
-#ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31,0,0,0), e.g., 43340.00000 
-
+cpl_dt=360  ;; coupling time step in seconds
+#ocean_tasks=120 ;; Number of PEs for the OCN component
+ocean_tasks=60 ;; Number of PEs for the OCN component
+ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31,0,0,0), e.g., 43340.00000 
 merge_import=.true.
 
 [rocotostr]

rocoto/cronjob_hafs_hycom.sh

@@ -35,9 +35,18 @@ scrubopt="config.scrub_work=no config.scrub_com=no"
 
 #===============================================================================
 
+# Run regional hafs-hycom coupled configuration with the cmeps-based coupling
+ ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY \
+     config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_cplocean3 \
+     config.ictype=gfsnemsio config.bctype=gfsgrib2_0p25 \
+     ${scrubopt} \
+     ../parm/hafs_regional_static.conf \
+     ../parm/hafs_hycom.conf \
+     forecast.cpl_ocean=3
+
 # Run regional hafs-hycom coupled configuration with the bilinear regridding method
  ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY \
-     config.EXPT=${EXPT} config.SUBEXPT=${EXPT} \
+     config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_cplocean2 \
      config.ictype=gfsnemsio config.bctype=gfsgrib2_0p25 \
      ${scrubopt} \
      ../parm/hafs_regional_static.conf \
@@ -46,7 +55,7 @@ scrubopt="config.scrub_work=no config.scrub_com=no"
 
 # Run regional hafs-hycom coupled configuration with the nearest point regridding method
 #${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY \
-#    config.EXPT=${EXPT} config.SUBEXPT=${EXPT} \
+#    config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_cplocean1 \
 #    config.ictype=gfsnemsio config.bctype=gfsgrib2_0p25 \
 #    ${scrubopt} \
 #    ../parm/hafs_regional_static.conf \
@@ -55,7 +64,7 @@ scrubopt="config.scrub_work=no config.scrub_com=no"
 
 # Run regional hafs-hycom coupled configuration in side-by-side model (no coupling)
 #${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY \
-#    config.EXPT=${EXPT} config.SUBEXPT=${EXPT} \
+#    config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_cplocean0 \
 #    config.ictype=gfsnemsio config.bctype=gfsgrib2_0p25 \
 #    ${scrubopt} \
 #    ../parm/hafs_regional_static.conf \

rocoto/cronjob_hafs_rt.sh

@@ -18,14 +18,14 @@ date
 #PYTHON3=/apps/intel/intelpython3/bin/python3
 
 # MSU Orion
-#HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS
-#dev="-s sites/orion.ent -f"
-#PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3
+ HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS
+ dev="-s sites/orion.ent -f"
+ PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3
 
 # NOAA RDHPCS Hera
- HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS
- dev="-s sites/hera.ent -f"
- PYTHON3=/apps/intel/intelpython3/bin/python3
+#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS
+#dev="-s sites/hera.ent -f"
+#PYTHON3=/apps/intel/intelpython3/bin/python3
 
 cd ${HOMEhafs}/rocoto
 
@@ -40,11 +40,23 @@ scrubopt="config.scrub_work=no config.scrub_com=no"
      config.ictype=gfsnetcdf config.bctype=gfsgrib2ab_0p25 \
      config.halo_blend=10 \
      config.GFSVER=PROD2021 \
-     dir.COMgfs=/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFSv16 \
+     dir.COMgfs=/work/noaa/hwrf/noscrub/hafs-input/COMGFSv16 \
      config.NHRS=12 ${scrubopt} \
      ../parm/hafs_regional_static.conf \
      ../parm/hafs_hycom.conf
 
+# Regional static NATL basin-focused and cmeps-based ocean-coupled configuration with GFSv16 netcdf format IC and grib2ab format BC
+ ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY \
+     config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_rt_regional_coupled_netcdf_cmeps \
+     config.ictype=gfsnetcdf config.bctype=gfsgrib2ab_0p25 \
+     config.halo_blend=10 \
+     config.GFSVER=PROD2021 \
+     dir.COMgfs=/work/noaa/hwrf/noscrub/hafs-input/COMGFSv16 \
+     config.NHRS=12 ${scrubopt} \
+     ../parm/hafs_regional_static.conf \
+     ../parm/hafs_hycom.conf \
+     forecast.cpl_ocean=3
+
 # Regional static NATL basin-focused configuration with GFS nemsio format IC/BC
 #${PYTHON3} ./run_hafs.py -t ${dev} 2019091600 09L HISTORY \
 #    config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_rt_regional_static \

rocoto/sites/hera.ent

@@ -42,9 +42,9 @@
 
   <!-- FORECAST_RESOURCES for regional HYCOM coupling -->
   <!ENTITY FORECAST_RESOURCES_regional_40x30io3x40_ocn120_omp2 "<nodes>72:ppn=20:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>1440</value></envar><envar><name>NCTSK</name><value>20</value></envar>&FORECAST_EXTRA;">
-  <!ENTITY FORECAST_RESOURCES_regional_40x30io2x40_ocn60_omp2 "<nodes>69:ppn=20:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>1380</value></envar><envar><name>NCTSK</name><value>20</value></envar>&FORECAST_EXTRA;">
+  <!ENTITY FORECAST_RESOURCES_regional_40x30io2x40_ocn60_omp2 "<nodes>67:ppn=20:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>1340</value></envar><envar><name>NCTSK</name><value>20</value></envar>&FORECAST_EXTRA;">
   <!ENTITY FORECAST_RESOURCES_regional_40x30io1x40_ocn60_omp2 "<nodes>65:ppn=20:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>1300</value></envar><envar><name>NCTSK</name><value>20</value></envar>&FORECAST_EXTRA;">
-  <!ENTITY FORECAST_RESOURCES_regional_40x30io1x80_ocn60_omp2 "<nodes>69:ppn=20:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>1380</value></envar><envar><name>NCTSK</name><value>20</value></envar>&FORECAST_EXTRA;">
+  <!ENTITY FORECAST_RESOURCES_regional_40x30io1x80_ocn60_omp2 "<nodes>67:ppn=20:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>1340</value></envar><envar><name>NCTSK</name><value>20</value></envar>&FORECAST_EXTRA;">
 
   <!ENTITY FORECAST_RESOURCES_regional_48x40io1x40_ocn60_omp2 "<nodes>101:ppn=20:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>2020</value></envar><envar><name>NCTSK</name><value>20</value></envar>&FORECAST_EXTRA;">
   <!ENTITY FORECAST_RESOURCES_regional_48x40io2x40_ocn60_omp2 "<nodes>103:ppn=20:tpp=2</nodes><envar><name>TOTAL_TASKS</name><value>2060</value></envar><envar><name>NCTSK</name><value>20</value></envar>&FORECAST_EXTRA;">