Update regional workflow for wcoss2 RRFS_A v0.6.1

scripts/exregional_clean.ksh

@@ -78,6 +78,10 @@ for onetime in ${XX[*]};do
         rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi/pe0*_setup
         rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi/obs_input.*
         rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi/diag*
+        rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi_spinup/pe0*.nc4
+        rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi_spinup/pe0*_setup
+        rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi_spinup/obs_input.*
+        rm -f ${CYCLE_BASEDIR}/${onetime}/anal_conv_gsi_spinup/diag*
 
         rm -rf ${CYCLE_BASEDIR}/${onetime}/fcst_fv3lam/RESTART
         echo "Deleted netCDF files in ${CYCLE_BASEDIR}/${onetime}/fcst_fv3lam"

scripts/exregional_run_analysis.sh

@@ -84,7 +84,7 @@ case $MACHINE in
   ulimit -s unlimited
   ulimit -a
   export FI_OFI_RXM_SAR_LIMIT=3145728
-  export OMP_STACKSIZE=1G
+  export OMP_STACKSIZE=500M
   export OMP_NUM_THREADS=${TPP_RUN_ANAL}
   ncores=$(( NNODES_RUN_ANAL*PPN_RUN_ANAL))
   APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_ANAL} --cpu-bind core --depth ${OMP_NUM_THREADS}"
@@ -375,7 +375,7 @@ if [ ${regional_ensemble_option:-1} -eq 5 ]  && [ ${BKTYPE} != 1  ]; then
   print_info_msg "$VERBOSE" " Cycle ${YYYYMMDDHH}: GSI hybrid uses FV3LAM ensemble with n_ens=${nummem}" 
   echo " ${YYYYMMDDHH}(${cycle_type}): GSI hybrid uses FV3LAM ensemble with n_ens=${nummem}" >> ${EXPTDIR}/log.cycles
   grid_ratio_ens="1"
-  ens_fast_read=.true.
+  ens_fast_read=.false.
 else    
   nummem_gfs=$(more filelist03 | wc -l)
   nummem_gfs=$((nummem_gfs - 3 ))
@@ -674,6 +674,7 @@ if [[ ${gsi_type} == "ANALYSIS" && ${anav_type} == "radardbz" ]]; then
 fi
 if [[ ${gsi_type} == "ANALYSIS" && ${anav_type} == "conv_dbz" ]]; then
   ANAVINFO=${FIX_GSI}/${ANAVINFO_CONV_DBZ_FN}
+  beta1_inv=0.0
   if_model_dbz=.true.
 fi
 naensloc=`expr ${nsclgrp} \* ${ngvarloc} + ${nsclgrp} - 1`

scripts/exregional_run_enkf.sh

@@ -72,10 +72,13 @@ case $MACHINE in
   ulimit -s unlimited
   ulimit -a
   export FI_OFI_RXM_SAR_LIMIT=3145728
-  export OMP_STACKSIZE=1G
+  export OMP_STACKSIZE=2G
   export OMP_NUM_THREADS=16
+  export OMP_PROC_BIND=close
+  export OMP_PLACES=threads
+  export MPICH_RANK_REORDER_METHOD=0
   ncores=$(( NNODES_RUN_ENKF*PPN_RUN_ENKF ))
-  APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_ENKF} --cpu-bind core --depth ${OMP_NUM_THREADS}"
+  APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_ENKF} --label --line-buffer --cpu-bind core --depth ${OMP_NUM_THREADS}"
   ;;
 #
 "THEIA")
@@ -489,6 +492,7 @@ fi
 countdiag=$(ls diag*conv* | wc -l)
 if [ $countdiag -gt $nens ]; then
 
+    if [ ${ob_type} == "conv" ]; then
 ${APRUN}  $enkfworkdir/enkf.x < enkf.nml 1>${stdout_name} 2>${stderr_name} || print_err_msg_exit "\
 Call to executable to run EnKF returned with nonzero exit code."
 
@@ -500,6 +504,11 @@ if [ ! -d ${NWGES_DIR}/../enkf_diag ]; then
 fi
 cp_vrfy ${stdout_name} ${NWGES_DIR}/../enkf_diag/${stdout_name}.$vlddate
 cp_vrfy ${stderr_name} ${NWGES_DIR}/../enkf_diag/${stderr_name}.$vlddate
+    else
+${APRUN}  $enkfworkdir/enkf.x < enkf.nml 1>${stdout_name} 2>${stderr_name} || print_err_msg_exit "\
+Call to executable to run EnKF returned with nonzero exit code."
+       echo "Warning: EnKF dbz analysis due to lack of ${ob_type} obs for cycle $vlddate !!!"
+    fi
 
 else
   echo "Warning: EnKF not running due to lack of ${ob_type} obs for cycle $vlddate !!!"

scripts/exregional_run_prdgen.sh

@@ -410,6 +410,8 @@ fi
 fi  # block for parallel or series wgrib2 runs.
 
 rm_vrfy -rf ${fhr_dir}
+rm -fr $DATAprdgen
+rm -f $DATA/*.t${cyc}z.*.f${fhr}.*.grib2
 #
 #-----------------------------------------------------------------------
 #

scripts/exregional_run_prepstart.sh

@@ -499,10 +499,10 @@ if [ ${HH} -eq ${SST_update_hour} ] && [ ${cycle_type} == "prod" ] ; then
       cp ${SST_ROOT}/latest.SST .
    elif [ -r "${SST_ROOT}/${YYJJJ00000000}" ]; then
       cp ${SST_ROOT}/${YYJJJ00000000} latest.SST
-   elif [ -r "${SST_ROOT}/sst.$YYYYMMDD/rtgssthr_grb_0.083.grib2" ]; then 
-      cp ${SST_ROOT}/sst.$YYYYMMDD/rtgssthr_grb_0.083.grib2 latest.SST
-   elif [ -r "${SST_ROOT}/sst.$YYYYMMDDm1/rtgssthr_grb_0.083.grib2" ]; then 
-      cp ${SST_ROOT}/sst.$YYYYMMDDm1/rtgssthr_grb_0.083.grib2 latest.SST
+   elif [ -r "${SST_ROOT}/nsst.$YYYYMMDD/rtgssthr_grb_0.083.grib2" ]; then 
+      cp ${SST_ROOT}/nsst.$YYYYMMDD/rtgssthr_grb_0.083.grib2 latest.SST
+   elif [ -r "${SST_ROOT}/nsst.$YYYYMMDDm1/rtgssthr_grb_0.083.grib2" ]; then 
+      cp ${SST_ROOT}/nsst.$YYYYMMDDm1/rtgssthr_grb_0.083.grib2 latest.SST
    else
      echo "${SST_ROOT} data does not exist!!"
      echo "ERROR: No SST update at ${HH}!!!!"

ush/config.sh_rrfs_a_enkf_n3

@@ -1,11 +1,13 @@
 MACHINE="wcoss2"
-version="v0.5.3"
+MACHINETYPE="backup"
+version="v0.6.1"
 ACCOUNT="RRFS_DEV"
 #RESERVATION="rrfsdet"
 EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/${version}"
 EXPT_SUBDIR="rrfs_na_enkf"
 
 PREDEF_GRID_NAME="RRFS_NA_3km"
+
 . set_rrfs_config_general.sh
 
 DO_ENSEMBLE="TRUE"
@@ -28,9 +30,11 @@ DO_SMOKE_DUST="FALSE"
 #DO_BUFRSND="TRUE"
 #USE_FVCOM="TRUE"
 #PREP_FVCOM="TRUE"
+DO_PARALLEL_PRDGEN="TRUE"
+DO_GSIDIAG_OFFLINE="FALSE"
 
 if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
-  EXPT_SUBDIR="rrfs_conus_enfcst"
+  EXPT_SUBDIR="rrfs_na_enfcst"
   DO_SPINUP="FALSE"
   DO_SAVE_DA_OUTPUT="FALSE"
   DO_NONVAR_CLDANAL="FALSE"
@@ -45,8 +49,8 @@ BOUNDARY_PROC_GROUP_NUM="12"
 
 # avaialble retro period:
 # 20210511-20210531; 20210718-20210801
-DATE_FIRST_CYCL="20230601"
-DATE_LAST_CYCL="20230630"
+DATE_FIRST_CYCL="20230701"
+DATE_LAST_CYCL="20230830"
 CYCL_HRS=( "00" "12" )
 CYCL_HRS=( "18" )
 CYCL_HRS_SPINSTART=("06" "18")
@@ -55,7 +59,7 @@ if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
   CYCL_HRS_STOCH=("00" "06" "12" "18")
 fi
 #CYCL_HRS_RECENTER=("19")
-CYCLEMONTH="06"
+CYCLEMONTH="07,08"
 CYCLEDAY="*"
 
 STARTYEAR=${DATE_FIRST_CYCL:0:4}
@@ -76,7 +80,7 @@ PRODLONG_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
 #ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1400 ${DATE_LAST_CYCL}2300 24:00:00"
 if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
   BOUNDARY_LEN_HRS="60"
-  LBC_SPEC_INTVL_HRS="1"
+  LBC_SPEC_INTVL_HRS="3"
   DO_SPINUP="FALSE"
   INITIAL_CYCLEDEF="00 01 01 01 2100 *"
   BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 06:00:00"
@@ -116,7 +120,9 @@ binary_diag=.false.
 
 WTIME_RUN_FCST="00:30:00"
 WTIME_RUN_FCST_LONG="04:45:00"
+PPN_RUN_ANAL="16"
 NNODES_RUN_ANAL="1"
+TPP_RUN_ANAL="8"
 
 EXTRN_MDL_NAME_ICS="GEFS"
 EXTRN_MDL_NAME_LBCS="GEFS"
@@ -157,16 +163,16 @@ if [[ ${DO_ENSEMBLE}  == "TRUE" ]]; then
    NNODES_RUN_FCST="13"
    WRTCMP_write_tasks_per_group="7"
 
-   NUM_ENS_MEMBERS_FCST=9
+   NUM_ENS_MEMBERS_FCST=5
    if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
      NUM_ENS_MEMBERS=${NUM_ENS_MEMBERS_FCST}
      WTIME_RUN_FCST="04:45:00"
      WTIME_MAKE_LBCS="01:30:00"
 
-     LAYOUT_X="15"
-     LAYOUT_Y="52"
-     NNODES_RUN_FCST="13"
-     WRTCMP_write_tasks_per_group="52"
+     LAYOUT_X="50"
+     LAYOUT_Y="64"
+     NNODES_RUN_FCST="51"
+     WRTCMP_write_tasks_per_group="64"
 
      DO_SPP="TRUE"
      DO_SPPT="FALSE"
@@ -184,21 +190,22 @@ fi
 
 RUN_ensctrl="rrfs"
 RUN="enkfrrfs"
-TAG="n3enkf53"
+TAG="n3enkf61"
 if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
   RUN="refs"
-  TAG="n3enfcst53"
+  TAG="n3enfcst61"
 fi
 COMINgfs=""
 
 . set_rrfs_config.sh
 
 STMP="/lfs/f2/t2o/ptmp/emc/stmp/emc.lam/rrfs/${version}"  # Path to directory STMP that mostly contains input files.
+PTMP="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}"  # Path to directory STMP that mostly contains input files.
+NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges"  # Path to directory NWGES that save boundary, cold initial, restart files
 if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
   STMP="/lfs/f2/t2o/ptmp/emc/stmp/emc.lam/rrfs/${version}/enfcst"  # Path to directory STMP that mostly contains input files.
+  NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges/enfcst"  # Path to directory NWGES that save boundary, cold initial, restart files
 fi
-PTMP="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}"  # Path to directory STMP that mostly contains input files.
-NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges"  # Path to directory NWGES that save boundary, cold initial, restart files
 ENSCTRL_STMP="/lfs/f2/t2o/ptmp/emc/stmp/emc.lam/rrfs/${version}"  # Path to directory STMP that mostly contains control input files for ensemble recentering.
 ENSCTRL_PTMP="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}"  # Path to directory STMP that mostly contains control input files for ensemble recentering.
 ENSCTRL_NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges"  # Path to directory STMP that mostly contains control input files for ensemble recentering.

ush/config.sh_rrfs_a_n3

@@ -1,5 +1,6 @@
 MACHINE="wcoss2"
-version="v0.5.3"
+MACHINETYPE="backup"
+version="v0.6.1"
 ACCOUNT="RRFS_DEV"
 #RESERVATION="rrfsdet"
 EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/${version}"
@@ -41,12 +42,12 @@ BOUNDARY_PROC_GROUP_NUM="72"
 
 # avaialble retro period:
 # 20210511-20210531; 20210718-20210801
-DATE_FIRST_CYCL="20230501"
-DATE_LAST_CYCL="20230630"
+DATE_FIRST_CYCL="20230701"
+DATE_LAST_CYCL="20230830"
 CYCL_HRS=( "00" "12" )
 CYCL_HRS_SPINSTART=("03" "15")
 CYCL_HRS_PRODSTART=("09" "21")
-CYCLEMONTH="05,06"
+CYCLEMONTH="07,08"
 CYCLEDAY="*"
 
 STARTYEAR=${DATE_FIRST_CYCL:0:4}
@@ -91,7 +92,7 @@ CYCL_HRS_HYB_FV3LAM_ENS=("00" "01" "02" "03" "04" "05" "06" "07" "08" "09" "10"
 
 SST_update_hour=01
 GVF_update_hour=04
-SNOWICE_update_hour=00
+SNOWICE_update_hour=01
 netcdf_diag=.true.
 binary_diag=.false.
 
@@ -103,7 +104,7 @@ EXTRN_MDL_NAME_LBCS="FV3GFS"
 envir="para"
 
 NET="rrfs_a"
-TAG="n3v53"
+TAG="n3v61"
 
 ARCHIVEDIR="/NCEPDEV/emc-meso/1year/emc.lam/${TAG}"
 NCL_REGION="conus"
@@ -112,11 +113,11 @@ RUN="rrfs"
 
 . set_rrfs_config.sh
 
-STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}"  # Path to directory STMP that mostly contains input files.
-PTMP="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}"  # Path to directory STMP that mostly contains input files.
-NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges"  # Path to directory NWGES that save boundary, cold initial, restart files
+STMP="/lfs/f2/t2o/ptmp/emc/stmp/emc.lam/rrfs/${version}"  # Path to directory STMP that mostly contains input files.
+PTMP="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}"  # Path to directory STMP that mostly contains input files.
+NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges"  # Path to directory NWGES that save boundary, cold initial, restart files
 if [[ ${regional_ensemble_option} == "5" ]]; then
-  RRFSE_NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges"  # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid.
+  RRFSE_NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges"  # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid.
   NUM_ENS_MEMBERS=30     # FV3LAM ensemble size for GSI hybrid analysis
   CYCL_HRS_PRODSTART_ENS=( "07" "19" )
   DO_ENVAR_RADAR_REF="TRUE"

ush/config_defaults.sh

@@ -46,6 +46,9 @@ version="0.1.0"
 # MACHINE:
 # Machine on which the workflow will run.
 #
+# MACHINETYPE:
+# decide Machine type for wcoss2 (backup or primary)
+#
 # ACCOUNT:
 # The account under which to submit jobs to the queue.
 #
@@ -145,6 +148,7 @@ version="0.1.0"
 #-----------------------------------------------------------------------
 #
 MACHINE="BIG_COMPUTER"
+MACHINETYPE="backup"
 ACCOUNT="project_name"
 SERVICE_ACCOUNT=""
 HPSS_ACCOUNT=""

ush/set_rrfs_config.sh

@@ -37,7 +37,7 @@ if [[ $MACHINE == "wcoss2" ]] ; then
   SST_ROOT=/lfs/h1/ops/prod/com/gfs/v16.3
   GVF_ROOT=/lfs/h1/ops/prod/dcom/viirs
   IMSSNOW_ROOT=/lfs/h1/ops/prod/com/obsproc/v1.1
-  FIRE_RAVE_DIR=/lfs/h2/emc/physics/noscrub/partha.bhattacharjee/RAVE_rawdata/RAVE_NA
+  FIRE_RAVE_DIR=/lfs/h2/emc/lam/noscrub/emc.lam/RAVE_rawdata/RAVE_NA
   FVCOM_DIR="/lfs/h1/ops/prod/com/nosofs/v3.5"
   FVCOM_FILE="fvcom"
 fi

ush/set_rrfs_config_general.sh

@@ -56,6 +56,13 @@ if [[ $MACHINE == "wcoss2" ]] ; then
   QUEUE_HPSS="dev_transfer"
   QUEUE_PRDGEN="pridev"
   QUEUE_GRAPHICS="pridev"
+  if [[ $MACHINETYPE == "backup" ]] ; then
+    QUEUE_DEFAULT="devhigh"
+    QUEUE_FCST="devmax"
+    QUEUE_ANALYSIS="devmax"
+    QUEUE_PRDGEN="devhigh"
+    QUEUE_POST="devhigh"
+  fi
 fi
 
 VERBOSE="TRUE"
@@ -195,6 +202,7 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then
     MEMO_RUN_PROCESSBUFR="80G"
     PPN_RUN_NONVARCLDANL="64"
     NNODES_RUN_NONVARCLDANL="10"
+    PPN_RUN_GSIDIAG=128
 
     MAXTRIES_GET_EXTRN_ICS="1"
     MAXTRIES_GET_EXTRN_LBCS="1"
@@ -277,6 +285,7 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_13km" ]] ; then
     NNODES_RUN_ENKF="2"
     PPN_RUN_NONVARCLDANL="128"
     NNODES_RUN_NONVARCLDANL="1"
+    PPN_RUN_GSIDIAG=128
 
     WTIME_GET_EXTRN_ICS="00:05:00"
     WTIME_GET_EXTRN_LBCS="00:05:00"
@@ -398,9 +407,9 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_NA_3km" ]] ; then
     TPP_MAKE_LBCS="2"
     NNODES_MAKE_ICS="2"
     NNODES_MAKE_LBCS="1"
-    PPN_RUN_ANAL="16"
-    NNODES_RUN_ANAL="22"
-    TPP_RUN_ANAL="8"
+    PPN_RUN_ANAL="8"
+    NNODES_RUN_ANAL="60"
+    TPP_RUN_ANAL="16"
     PPN_RUN_FCST="64"
     NNODES_RUN_FCST="51"
     TPP_RUN_FCST="2"
@@ -412,10 +421,11 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_NA_3km" ]] ; then
     PPN_RUN_ENKF="8"
     NNODES_RUN_ENKF="10"
     TPP_RUN_ENKF="16"
-    PPN_RUN_NONVARCLDANL="64"
-    NNODES_RUN_NONVARCLDANL="5"
+    PPN_RUN_NONVARCLDANL="128"
+    NNODES_RUN_NONVARCLDANL="1"
     PPN_RUN_PRDGEN="128"
     PPN_RUN_POSTANAL="8"
+    PPN_RUN_GSIDIAG=128
 
     MAXTRIES_MAKE_ICS="1"
     MAXTRIES_MAKE_LBCS="1"
@@ -445,13 +455,13 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_NA_3km" ]] ; then
     FV3GFS_FILE_FMT_LBCS="netcdf"
 
     WTIME_RUN_FCST="02:30:00"
-    WTIME_RUN_FCST_LONG="04:30:00"
+    WTIME_RUN_FCST_LONG="05:30:00"
     WTIME_MAKE_LBCS="02:30:00"
 
-    CLEAN_OLDPROD_HRS="72"
-    CLEAN_OLDLOG_HRS="72"
+    CLEAN_OLDPROD_HRS="120"
+    CLEAN_OLDLOG_HRS="120"
     CLEAN_OLDRUN_HRS="18"
-    CLEAN_OLDFCST_HRS="12"
+    CLEAN_OLDFCST_HRS="18"
     CLEAN_OLDSTMPPOST_HRS="12"
     CLEAN_NWGES_HRS="18"
 

ush/templates/FV3LAM_wflow.xml

@@ -359,7 +359,7 @@ define resources used for each tasks
 {%- if do_retro %}
 <workflow realtime="F" scheduler="&SCHED;" cyclethrottle="5">
 {% else %}
-<workflow realtime="T" scheduler="&SCHED;" cyclethrottle="24" cyclelifespan="01:00:00:00">
+<workflow realtime="T" scheduler="&SCHED;" cyclethrottle="26" cyclelifespan="01:00:00:00">
 {%- endif %}
 {# Double quotes are required inside the strftime! Expect an error from reading the template if using single quotes. #}
   <cycledef group="at_start">{{ at_start_cycledef }} </cycledef>
@@ -749,7 +749,7 @@ MODULES_RUN_TASK_FP script.
 
   </task>
 
-{%- if extrn_mdl_name_lbcs in ["GEFS"] and bc_update_interval==1 %}
+{%- if extrn_mdl_name_lbcs in ["GEFS"] %}
 <!--
 ************************************************************************
 ************************************************************************
@@ -781,7 +781,7 @@ MODULES_RUN_TASK_FP script.
        <and>
       {%- if machine in ["WCOSS2"]  %}
          {%- for h in range(extrn_mdl_lbcs_offset_hrs, boundary_len_hrs+extrn_mdl_lbcs_offset_hrs+1, 3) %}
-         <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_ics }}/gefs.@Y@m@d/@H/atmos/pgrb2bp5/#subdirGE#.t@Hz.pgrb2b.0p50.f{{ "%03d" % extrn_mdl_ics_offset_hrs }}</cyclestr></datadep>
+         <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_lbcs_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_ics }}/gefs.@Y@m@d/@H/atmos/pgrb2bp5/#subdirGE#.t@Hz.pgrb2b.0p50.f{{ "%03d" % h }}</cyclestr></datadep>
          {%- endfor %}
       {% else %}
          {%- for h in range(extrn_mdl_lbcs_offset_hrs, boundary_len_hrs+extrn_mdl_lbcs_offset_hrs+1, 3) %}