NOAA-GSL · christinaholtNOAA · Feb 7, 2022 · Feb 1, 2022 · Feb 1, 2022 · Feb 4, 2022
@@ -24,7 +24,7 @@ Add any links to external PRs. For example:
 If this PR is contributing new capabilities that need to be documented, please also include updates to the RST files in the ufs-srweather-app repository (docs/UsersGuide/source) as supporting material.
 
 ## ISSUE (optional): 
-If this PR is resolving or referencing one or more issues, in this repository or elewhere, list them here. For example, "Fixes issue mentioned in #123" or "Related to bug in https://github.com/NOAA-EMC/other_repository/pull/63"
+If this PR is resolving or referencing one or more issues, in this repository or elewhere, list them here. For example, "Fixes issue mentioned in #123" or "Related to bug in https://github.com/ufs-community/other_repository/pull/63"
 
 ## CONTRIBUTORS (optional): 
 If others have contributed to this work aside from the PR author, list them here

@@ -5,7 +5,7 @@ Configuring the FV3SAR Workflow
 ***************************************
 
 The following steps describe how to create a user-specific configuration
-file in order to un your experiment in a given environment.
+file in order to run your experiment in a given environment.
 
 1. Create a user-specific configuration file named ``config.sh`` in the subdirectory
    ``ush`` under the ``$BASEDIR/regional_workflow`` directory containing appropriate
@@ -46,7 +46,7 @@ file in order to un your experiment in a given environment.
    #
    BASEDIR="/path/to/directory/of/regional_workflow/and/NEMSfv3gfs/clones"
    TMPDIR="/path/to/temporary/work/directories"
-   UPPDIR="/scratch3/BMC/det/beck/FV3-CAM/EMC_post/sorc/ncep_post.fd"
+   UPPDIR="/path/to/UPP/executable/directory"
    CCPP="false"
    #
    CDATE="2018060400"

@@ -92,7 +92,7 @@ case "$MACHINE" in
     module list
     { restore_shell_opts; } > /dev/null 2>&1
     export NODES=1
-    export APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth"
+    export RUN_CMD_SERIAL="aprun -n 1 -N 1 -j 1 -d 1 -cc depth"
     export KMP_AFFINITY=disabled
     ulimit -s unlimited
     ulimit -a
@@ -102,56 +102,25 @@ case "$MACHINE" in
     { save_shell_opts; set +x; } > /dev/null 2>&1
     module list
     { restore_shell_opts; } > /dev/null 2>&1
-    export APRUN="mpirun"
+    export RUN_CMD_SERIAL="mpirun"
     ulimit -s unlimited
     ;;
 
-  "HERA")
-    APRUN="time"
-    ;;
-
-  "ORION")
-    ulimit -s unlimited
-    ulimit -a
-    APRUN="time"
-    ;;
-
-  "JET")
-    APRUN="time"
-    ulimit -a
-    ;;
-
-  "ODIN")
-    export APRUN="srun -n 1"
-    ulimit -s unlimited
-    ulimit -a
-    ;;
-
-  "CHEYENNE")
-    APRUN="time"
-    ;;
-
-  "STAMPEDE")
-    export APRUN="time"
-    ulimit -s unlimited
-    ulimit -a
-    ;;
-
-  "MACOS")
-    APRUN=time
-    ;;
-
-  "LINUX")
-    APRUN=time
-    ;;
-
   *)
-    print_err_msg_exit "\
-Run command has not been specified for this machine:
-  MACHINE = \"$MACHINE\""
+    source ${MACHINE_FILE}
     ;;
 
 esac
+
+if [ -z ${RUN_CMD_SERIAL:-} ] ; then
+  print_err_msg_exit " \
+  Run command was not set in machine file. \
+  Please set RUN_CMD_SERIAL for your platform"
+else
+  RUN_CMD_SERIAL=$(eval echo ${RUN_CMD_SERIAL})
+  print_info_msg "$VERBOSE" "
+  All executables will be submitted with command \'${RUN_CMD_SERIAL}\'."
+fi
 #
 #-----------------------------------------------------------------------
 #
@@ -311,7 +280,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then
 # for the 6 global tiles.  However, after this call we will only need the
 # regional grid file.
 #
-  $APRUN ${exec_fp} \
+  $RUN_CMD_SERIAL ${exec_fp} \
     --grid_type gnomonic_ed \
     --nlon ${nx_t6sg} \
     --grid_name ${grid_name} \
@@ -385,7 +354,7 @@ $settings"
 #
 # Call the executable that generates the grid file.
 #
-  $APRUN ${exec_fp} ${rgnl_grid_nml_fp} || \
+  $RUN_CMD_SERIAL ${exec_fp} ${rgnl_grid_nml_fp} || \
     print_err_msg_exit "\
 Call to executable (exec_fp) that generates a ESGgrid-type regional grid
 returned with nonzero exit code:
@@ -424,7 +393,7 @@ cubed-sphere grid equivalent resolution does not exist:
 Please ensure that you've built this executable."
 fi
 
-$APRUN ${exec_fp} "${grid_fp}" || \
+$RUN_CMD_SERIAL ${exec_fp} "${grid_fp}" || \
 print_err_msg_exit "\
 Call to executable (exec_fp) that calculates the regional grid's global
 uniform cubed-sphere grid equivalent resolution returned with nonzero exit
@@ -519,7 +488,7 @@ fi
 exec_fn="shave"
 exec_fp="$EXECDIR/${exec_fn}"
 if [ ! -f "${exec_fp}" ]; then
-  print_err_msg_exit "\
+  print_err_msg_exit " \
 The executable (exec_fp) for \"shaving\" down the halo in the grid file
 does not exist:
   exec_fp = \"${exec_fp}\"
@@ -553,7 +522,7 @@ printf "%s %s %s %s %s\n" \
   $NX $NY ${NH3} \"${unshaved_fp}\" \"${shaved_fp}\" \
   > ${nml_fn}
 
-$APRUN ${exec_fp} < ${nml_fn} || \
+$RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} || \
 print_err_msg_exit "\
 Call to executable (exec_fp) to generate a grid file with a ${NH3}-cell-wide
 halo from the grid file with a ${NHW}-cell-wide halo returned with nonzero
@@ -579,7 +548,7 @@ printf "%s %s %s %s %s\n" \
   $NX $NY ${NH4} \"${unshaved_fp}\" \"${shaved_fp}\" \
   > ${nml_fn}
 
-$APRUN ${exec_fp} < ${nml_fn} || \
+$RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} || \
 print_err_msg_exit "\
 Call to executable (exec_fp) to generate a grid file with a ${NH4}-cell-wide
 halo from the grid file with a ${NHW}-cell-wide halo returned with nonzero
@@ -603,7 +572,8 @@ cd_vrfy -
 make_grid_mosaic_file \
   grid_dir="${GRID_DIR}" \
   grid_fn="${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NHW}.nc" \
-  mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NHW}.nc" || \
+  mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NHW}.nc" \
+  run_cmd="${RUN_CMD_SERIAL}" || \
   print_err_msg_exit "\
 Call to function to generate the mosaic file for a grid with a ${NHW}-cell-wide
 halo failed."
@@ -617,7 +587,8 @@ halo failed."
 make_grid_mosaic_file \
   grid_dir="${GRID_DIR}" \
   grid_fn="${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH3}.nc" \
-  mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NH3}.nc" || \
+  mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NH3}.nc" \
+  run_cmd="${RUN_CMD_SERIAL}" || \
   print_err_msg_exit "\
 Call to function to generate the mosaic file for a grid with a ${NH3}-cell-wide
 halo failed."
@@ -631,7 +602,8 @@ halo failed."
 make_grid_mosaic_file \
   grid_dir="${GRID_DIR}" \
   grid_fn="${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH4}.nc" \
-  mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" || \
+  mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" \
+  run_cmd="${RUN_CMD_SERIAL}" || \
   print_err_msg_exit "\
 Call to function to generate the mosaic file for a grid with a ${NH4}-cell-wide
 halo failed."

@@ -90,59 +90,33 @@ case "$MACHINE" in
 
   "WCOSS_CRAY")
     ulimit -s unlimited
-    APRUN="aprun -b -j1 -n48 -N12 -d1 -cc depth"
+    RUN_CMD_UTILS="aprun -b -j1 -n48 -N12 -d1 -cc depth"
     ;;
 
   "WCOSS_DELL_P3")
     ulimit -s unlimited
-    APRUN="mpirun"
+    RUN_CMD_UTILS="mpirun"
     ;;
 
-  "HERA")
-    ulimit -s unlimited
-    APRUN="srun"
-    ;;
-
-  "ORION")
-    ulimit -s unlimited
-    ulimit -a
-    APRUN="srun"
-    ;;
-
-  "JET")
-    ulimit -s unlimited
-    APRUN="srun"
-    ;;
-
-  "ODIN")
-    APRUN="srun"
-    ;;
-
-  "CHEYENNE")
-    nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS ))
-    APRUN="mpirun -np $nprocs"
+  *)
+    source ${MACHINE_FILE}
     ;;
 
-  "STAMPEDE")
-    APRUN="ibrun"
-    ;;
+esac
 
-  "MACOS")
-    APRUN=$RUN_CMD_UTILS
-    ;;
+nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS ))
 
-  "LINUX")
-    ulimit -s unlimited
-    APRUN=$RUN_CMD_UTILS
-    ;;
+if [ -z ${RUN_CMD_UTILS:-} ] ; then
+  print_err_msg_exit "\
+  Run command was not set in machine file. \
+  Please set RUN_CMD_UTILS for your platform"
+else
+  RUN_CMD_UTILS=$(eval echo ${RUN_CMD_UTILS})
+  print_info_msg "$VERBOSE" "
+  All executables will be submitted with command \'${RUN_CMD_UTILS}\'."
+fi
 
-  *)
-    print_err_msg_exit "\
-Run command has not been specified for this machine:
-  MACHINE = \"$MACHINE\""
-    ;;
 
-esac
 #
 #-----------------------------------------------------------------------
 #
@@ -618,14 +592,14 @@ $settings"
 #-----------------------------------------------------------------------
 #
 # NOTE:
-# Often when the chgres_cube.exe run fails, it still returns a zero re-
-# turn code, so the failure isn't picked up the the logical OR (||) be-
-# low.  That should be fixed.  This might be due to the APRUN command -
-# maybe that is returning a zero exit code even though the exit code
-# of chgres_cube is nonzero.
-# A similar thing happens in the forecast task.
-#
-${APRUN} ${exec_fp} || \
+# Often when the chgres_cube.exe run fails, it still returns a zero
+# return code, so the failure isn't picked up the the logical OR (||)
+# below.  That should be fixed.  This might be due to the RUN_CMD_UTILS
+# command - maybe that is returning a zero exit code even though the
+# exit code of chgres_cube is nonzero.  A similar thing happens in the
+# forecast task.
+#
+${RUN_CMD_UTILS} ${exec_fp} || \
   print_err_msg_exit "\
 Call to executable (exec_fp) to generate surface and initial conditions
 (ICs) files for the FV3-LAM failed:
@@ -685,7 +659,7 @@ Please check the following user defined variables:
 
   cp_vrfy ${fvcom_data_fp} ${ics_dir}/fvcom.nc
   cd_vrfy ${ics_dir}
-  ${APRUN} ${fvcom_exec_fn} sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time}|| \
+  ${RUN_CMD_UTILS} ${fvcom_exec_fn} sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time}|| \
   print_err_msg_exit "\
 Call to executable (fvcom_exe) to modify sfc fields for FV3-LAM failed:
   fvcom_exe = \"${fvcom_exe}\"

@@ -90,58 +90,31 @@ case "$MACHINE" in
 
   "WCOSS_CRAY")
     ulimit -s unlimited
-    APRUN="aprun -b -j1 -n48 -N12 -d1 -cc depth"
+    RUN_CMD_UTILS="aprun -b -j1 -n48 -N12 -d1 -cc depth"
     ;;
 
   "WCOSS_DELL_P3")
     ulimit -s unlimited
-    APRUN="mpirun"
-    ;;
-
-  "HERA")
-    ulimit -s unlimited
-    APRUN="srun"
-    ;;
-
-  "ORION")
-    ulimit -s unlimited
-    APRUN="srun"
-    ;;
-
-  "JET")
-    ulimit -s unlimited
-    APRUN="srun"
-    ;;
-
-  "ODIN")
-    APRUN="srun"
-    ;;
-
-  "CHEYENNE")
-    nprocs=$(( NNODES_MAKE_LBCS*PPN_MAKE_LBCS ))
-    APRUN="mpirun -np $nprocs"
-    ;;
-
-  "STAMPEDE")
-    APRUN="ibrun"
-    ;;
-
-  "MACOS")
-    APRUN=$RUN_CMD_UTILS
-    ;;
-
-  "LINUX")
-    ulimit -s unlimited
-    APRUN=$RUN_CMD_UTILS
+    RUN_CMD_UTILS="mpirun"
     ;;
 
   *)
-    print_err_msg_exit "\
-Run command has not been specified for this machine:
-  MACHINE = \"$MACHINE\""
+    source ${MACHINE_FILE}
     ;;
 
 esac
+
+nprocs=$(( NNODES_MAKE_LBCS*PPN_MAKE_LBCS ))
+
+if [ -z ${RUN_CMD_UTILS:-} ] ; then
+  print_err_msg_exit "\
+  Run command was not set in machine file. \
+  Please set RUN_CMD_UTILS for your platform"
+else
+  RUN_CMD_UTILS=$(eval echo ${RUN_CMD_UTILS})
+  print_info_msg "$VERBOSE" "
+  All executables will be submitted with command \'${RUN_CMD_UTILS}\'."
+fi
 #
 #-----------------------------------------------------------------------
 #
@@ -532,14 +505,14 @@ $settings"
 #-----------------------------------------------------------------------
 #
 # NOTE:
-# Often when the chgres_cube.exe run fails, it still returns a zero re-
-# turn code, so the failure isn't picked up the the logical OR (||) be-
-# low.  That should be fixed.  This might be due to the APRUN command -
-# maybe that is returning a zero exit code even though the exit code
-# of chgres_cube is nonzero.
-# A similar thing happens in the forecast task.
-#
-  ${APRUN} ${exec_fp} || \
+# Often when the chgres_cube.exe run fails, it still returns a zero
+# return code, so the failure isn't picked up the the logical OR (||)
+# below.  That should be fixed.  This might be due to the RUN_CMD_UTILS
+# command - maybe that is returning a zero exit code even though the
+# exit code of chgres_cube is nonzero.  A similar thing happens in the
+# forecast task.
+#
+  ${RUN_CMD_UTILS} ${exec_fp} || \
     print_err_msg_exit "\
 Call to executable (exec_fp) to generate lateral boundary conditions (LBCs)
 file for the FV3-LAM for forecast hour fhr failed: