Changes towards making SRW app NCO compliant

.github/workflows/build.yml

@@ -12,10 +12,5 @@ jobs:
       - run: |
              cd ufs-srweather-app
              ./manage_externals/checkout_externals
-             module use modulefiles
-             module load build_cheyenne_intel
-             mdkir build
-             cd build
-             cmake -DCMAKE_INSTALL_PREFIX=.. ..
-             make -j 4
+             ./devbuild.sh -p=cheyenne -c=intel
       - run: echo "🍏 This job's status is ${{ job.status }}."

.github/workflows/python_func_tests.yaml

@@ -3,7 +3,7 @@ on:
   push:
     paths:
       - ush/*retrieve_data.py
-      - ush/templates/data_locations.yml
+      - parm/data_locations.yml
   pull_request:
 
 env:

.github/workflows/python_unittests.yaml

@@ -1,5 +1,8 @@
 name: Python unittests
 on: [push, pull_request, workflow_dispatch]
+defaults:
+  run:
+    shell: bash
 jobs:
 
   python_unittests:
@@ -15,13 +18,17 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install python3 python3-pip netcdf-bin
-          sudo pip3 install pyyaml jinja2 f90nml
+          sudo pip3 install pyyaml jinja2==2.11 f90nml
           sudo pip3 install numpy matplotlib basemap
 
       # Run python unittests
       - name: Run python unittests
         run: |
+          ./manage_externals/checkout_externals ufs-weather-model
           cd ush
           python3 -m unittest -b python_utils/test_python_utils.py
-          python3 -m unittest -b *.py
+          # exclude test_retrieve_data that is tested in functional test
+          files=$(find . -maxdepth 1 -name '*.py' -a ! -name 'test_retrieve_data.py' -exec basename {} \;)
+          files=$(echo $(echo "${files[@]}"))
+          python3 -m unittest -b ${files[@]}
 

devbuild.sh

@@ -38,7 +38,7 @@ OPTIONS
   --install-dir=INSTALL_DIR
       installation prefix
   --bin-dir=BIN_DIR
-      installation binary directory name ("bin" by default; any name is available)
+      installation binary directory name ("exec" by default; any name is available)
   --build-type=BUILD_TYPE
       build type; defaults to RELEASE
       (e.g. DEBUG | RELEASE | RELWITHDEBINFO)
@@ -102,7 +102,7 @@ SRW_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P)
 MACHINE_SETUP=${SRW_DIR}/src/UFS_UTILS/sorc/machine-setup.sh
 BUILD_DIR="${SRW_DIR}/build"
 INSTALL_DIR=${SRW_DIR}
-BIN_DIR="bin" #change this to "exec" for NCO mode later
+BIN_DIR="exec"
 COMPILER=""
 APPLICATION=""
 CCPP_SUITES=""

docs/UsersGuide/source/ConfigWorkflow.rst

@@ -117,7 +117,7 @@ Cron is a job scheduler accessed through the command-line on UNIX-like operating
 Directory Parameters
 ====================
 ``EXPT_BASEDIR``: (Default: "")
-   The full path to the base directory inside of which the experiment directory (``EXPT_SUBDIR``) will be created. If this is not specified or if it is set to an empty string, it will default to ``${HOMErrfs}/../../expt_dirs``, where ``${HOMErrfs}`` contains the full path to the ``regional_workflow`` directory.
+   The full path to the base directory inside of which the experiment directory (``EXPT_SUBDIR``) will be created. If this is not specified or if it is set to an empty string, it will default to ``${HOMEdir}/../../expt_dirs``, where ``${HOMEdir}`` contains the full path to the ``regional_workflow`` directory.
 
 ``EXPT_SUBDIR``: (Default: "")
    A descriptive name of the user's choice for the experiment directory (*not* its full path). The full path to the experiment directory, which will be contained in the variable ``EXPTDIR``, will be:

docs/UsersGuide/source/Graphics.rst

@@ -196,36 +196,36 @@ At a minimum, the account should be set appropriately prior to job submission:
 Depending on the platform, users may also need to adjust the settings to use the correct Python environment and path to the shapefiles.
 
 When working with these batch scripts, several environment variables must be set prior to submission.
-If plotting output from a single cycle, the variables to set are ``$HOMErrfs`` and ``$EXPTDIR``.
+If plotting output from a single cycle, the variables to set are ``$HOMEdir`` and ``$EXPTDIR``.
 If the user's login shell is bash, these variables can be set as follows:
 
 .. code-block:: console
 
-   export HOMErrfs=/path-to/ufs-srweather-app/regional_workflow
+   export HOMEdir=/path-to/ufs-srweather-app/regional_workflow
    export EXPTDIR=/path-to/experiment/directory
 
 If the user's login shell is csh/tcsh, they can be set as follows:
 
 .. code-block:: console
 
-   setenv HOMErrfs /path-to/ufs-srweather-app/regional_workflow
+   setenv HOMEdir /path-to/ufs-srweather-app/regional_workflow
    setenv EXPTDIR /path-to/experiment/directory
 
 If plotting the difference between the same cycle from two different experiments, the variables 
-to set are ``$HOMErrfs``, ``$EXPTDIR1``, and ``$EXPTDIR2``. If the user's login shell 
+to set are ``$HOMEdir``, ``$EXPTDIR1``, and ``$EXPTDIR2``. If the user's login shell 
 is bash, these variables can be set as follows:
 
 .. code-block:: console
 
-   export HOMErrfs=/path-to/ufs-srweather-app/regional_workflow
+   export HOMEdir=/path-to/ufs-srweather-app/regional_workflow
    export EXPTDIR1=/path-to/experiment/directory1
    export EXPTDIR2=/path-to/experiment/directory2
 
 If the user's login shell is csh/tcsh, they can be set as follows:
 
 .. code-block:: console
 
-   setenv HOMErrfs /path-to/ufs-srweather-app/regional_workflow
+   setenv HOMEdir /path-to/ufs-srweather-app/regional_workflow
    setenv EXPTDIR1 /path-to/experiment/directory1
    setenv EXPTDIR2 /path-to/experiment/directory2
 

jobs/JREGIONAL_GET_EXTRN_MDL_FILES

@@ -29,7 +29,8 @@
 #-----------------------------------------------------------------------
 #
 . ${GLOBAL_VAR_DEFNS_FP}
-. $USHDIR/source_util_funcs.sh
+. $USHdir/source_util_funcs.sh
+. $USHdir/job_preamble.sh
 #
 #-----------------------------------------------------------------------
 #
@@ -38,7 +39,7 @@
 #
 #-----------------------------------------------------------------------
 #
-{ save_shell_opts; set -u +x; } > /dev/null 2>&1
+{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1
 #
 #-----------------------------------------------------------------------
 #
@@ -67,8 +68,6 @@ This is the J-job script for the task that copies or fetches external
 model files from disk, HPSS, or URL, and stages them for downstream use
 to generate initial or lateral boundary conditions for the FV3 model.
 ========================================================================"
-
-
 #
 #-----------------------------------------------------------------------
 #
@@ -92,12 +91,12 @@ check_var_valid_value "ICS_OR_LBCS" "valid_vals_ICS_OR_LBCS"
 #-----------------------------------------------------------------------
 #
 if [ "${ICS_OR_LBCS}" = "ICS" ]; then
-  time_offset_hrs=${EXTRN_MDL_ICS_OFFSET_HRS:-0}
-  extrn_mdl_name=${EXTRN_MDL_NAME_ICS}
+  export TIME_OFFSET_HRS=${EXTRN_MDL_ICS_OFFSET_HRS:-0}
+  export EXTRN_MDL_NAME=${EXTRN_MDL_NAME_ICS}
 
 elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then
-  time_offset_hrs=${EXTRN_MDL_LBCS_OFFSET_HRS:-0}
-  extrn_mdl_name=${EXTRN_MDL_NAME_LBCS}
+  export TIME_OFFSET_HRS=${EXTRN_MDL_LBCS_OFFSET_HRS:-0}
+  export EXTRN_MDL_NAME=${EXTRN_MDL_NAME_LBCS}
 fi
 
 #
@@ -108,16 +107,16 @@ fi
 #-----------------------------------------------------------------------
 #
 
-hh=${CDATE:8:2}
-yyyymmdd=${CDATE:0:8}
-extrn_mdl_cdate=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${time_offset_hrs} hours" "+%Y%m%d%H" )
+yyyymmdd=${PDY}
+hh=${cyc}
+export EXTRN_MDL_CDATE=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${TIME_OFFSET_HRS} hours" "+%Y%m%d%H" )
 
 #
 #-----------------------------------------------------------------------
 #
 # Check whether output files from the specified external model
-# (extrn_mdl_name) are available on the specified cycle date and time
-# (extrn_mdl_cdate).
+# (EXTRN_MDL_NAME) are available on the specified cycle date and time
+# (EXTRN_MDL_CDATE).
 #
 #-----------------------------------------------------------------------
 #
@@ -133,60 +132,60 @@ function data_unavailable() {
 
   if [ ${min_max} = max ]; then
     msg="\
-Output from the specified external model (extrn_mdl_name) is not availa-
-ble for the specified cycle date and time (extrn_mdl_cdate) because the latter is
+Output from the specified external model (EXTRN_MDL_NAME) is not availa-
+ble for the specified cycle date and time (EXTRN_MDL_CDATE) because the latter is
 later than the last forecast date and time (cdate_max) with this model:
-  extrn_mdl_name = \"${name}\"
+  EXTRN_MDL_NAME = \"${name}\"
   CDATE_max = \"${end_date}\"
-  extrn_mdl_cdate = \"${cdate}\""
+  EXTRN_MDL_CDATE = \"${cdate}\""
 
   elif [ ${min_max} = min ]; then
     msg="\
-Output from the specified external model (extrn_mdl_name) is not availa-
-ble for the specified cycle date and time (extrn_mdl_cdate) because the latter is
+Output from the specified external model (EXTRN_MDL_NAME) is not availa-
+ble for the specified cycle date and time (EXTRN_MDL_CDATE) because the latter is
 earlier than the implementation date of this model:
-  extrn_mdl_name = \"${name}\"
+  EXTRN_MDL_NAME = \"${name}\"
   CDATE_min = \"${end_date}\"
-  extrn_mdl_cdate = \"${cdate}\""
+  EXTRN_MDL_CDATE = \"${cdate}\""
   fi
 
   echo ${msg}
 }
 
 
-case ${extrn_mdl_name} in
+case ${EXTRN_MDL_NAME} in
 
 "GSMGFS")
 # The transition date from the GSMGFS to the FV3GFS was 2019061212, i.e.
 # this was the first official forecast with the FV3GFS.  So we set the
 # last CDATE for the GSMGFS to the one 6 hours before this.
   cdate_max="2019061206"
-  if [ "$extrn_mdl_cdate" -gt "$cdate_max" ]; then
+  if [ "$EXTRN_MDL_CDATE" -gt "$cdate_max" ]; then
     print_err_msg_exit "\
-      $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_max max)"
+      $(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $cdate_max max)"
   fi
   ;;
 
 "FV3GFS")
 # The transition date from the GSMGFS to the FV3GFS was 2019061212, i.e.
 # this was the first official forecast with the FV3GFS.  However, paral-
 # lel runs with the FV3GFS go back to 2018121500.  So we set the first 
-# extrn_mdl_cdate for the FV3GFS to this date and time.
+# EXTRN_MDL_CDATE for the FV3GFS to this date and time.
 #  CDATE_min="2019061212"
   CDATE_min="2018121500"
-  if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then
+  if [ "$EXTRN_MDL_CDATE" -lt "$CDATE_min" ]; then
     print_err_msg_exit "\
-      $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)"
+      $(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $cdate_min min)"
   fi
   ;;
 
 "RAP")
 # Examination of the HPSS archives shows that the RAPX data goes back to
 # July 01, 2015.
   CDATE_min="2015070100"
-  if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then
+  if [ "$EXTRN_MDL_CDATE" -lt "$CDATE_min" ]; then
     print_err_msg_exit "\
-      $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)"
+      $(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $cdate_min min)"
   fi
   ;;
 
@@ -195,9 +194,9 @@ case ${extrn_mdl_name} in
 # implementation of the first version of the operational HRRR was 
 # September 30, 2014.
   CDATE_min="2014103000"
-  if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then
+  if [ "$EXTRN_MDL_CDATE" -lt "$CDATE_min" ]; then
     print_err_msg_exit "\
-      $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)"
+      $(data_unavailable $EXTRN_MDL_NAME $EXTRN_MDL_CDATE $cdate_min min)"
   fi
   ;;
 
@@ -209,35 +208,32 @@ esac
 #
 #-----------------------------------------------------------------------
 #
-extrn_mdl_staging_dir="${CYCLE_DIR}/${extrn_mdl_name}/for_${ICS_OR_LBCS}"
-mkdir_vrfy -p "${extrn_mdl_staging_dir}"
-cd_vrfy "${extrn_mdl_staging_dir}"
+if [ $RUN_ENVIR = "nco" ]; then
+    export EXTRN_MDL_STAGING_DIR="${COMIN}"
+    mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}"
+else
+    export EXTRN_MDL_STAGING_DIR="${COMIN}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}"
+    mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}"
+    cd_vrfy "${EXTRN_MDL_STAGING_DIR}"
+fi
 #
 #-----------------------------------------------------------------------
 #
 # Call the ex-script for this J-job and pass to it the necessary variables.
 #
 #-----------------------------------------------------------------------
 #
-$SCRIPTSDIR/exregional_get_extrn_mdl_files.sh \
-  extrn_mdl_cdate="${extrn_mdl_cdate}" \
-  extrn_mdl_name="${extrn_mdl_name}" \
-  extrn_mdl_staging_dir="${extrn_mdl_staging_dir}" \
-  time_offset_hrs=${time_offset_hrs} ||
+$SCRIPTSdir/exregional_get_extrn_mdl_files.sh || \
 print_err_msg_exit "\
 Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
 #
 #-----------------------------------------------------------------------
 #
-# Print exit message.
+# Run job postamble.
 #
 #-----------------------------------------------------------------------
 #
-print_info_msg "
-========================================================================
-Exiting script:  \"${scrfunc_fn}\"
-In directory:    \"${scrfunc_dir}\"
-========================================================================"
+job_postamble
 #
 #-----------------------------------------------------------------------
 #

jobs/JREGIONAL_GET_OBS_CCPA

@@ -20,7 +20,8 @@
 #-----------------------------------------------------------------------
 #
 . ${GLOBAL_VAR_DEFNS_FP}
-. $USHDIR/source_util_funcs.sh
+. $USHdir/source_util_funcs.sh
+. $USHdir/job_preamble.sh
 #
 #-----------------------------------------------------------------------
 #
@@ -29,7 +30,7 @@
 #
 #-----------------------------------------------------------------------
 #
-{ save_shell_opts; set -u +x; } > /dev/null 2>&1
+{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1
 #
 #-----------------------------------------------------------------------
 #
@@ -67,21 +68,17 @@ for verification purposes.
 #
 #-----------------------------------------------------------------------
 #
-$SCRIPTSDIR/exregional_get_ccpa_files.sh || \
+$SCRIPTSdir/exregional_get_obs_ccpa.sh || \
 print_err_msg_exit "\
 Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
 #
 #-----------------------------------------------------------------------
 #
-# Print exit message.
+# Run job postamble.
 #
 #-----------------------------------------------------------------------
 #
-print_info_msg "
-========================================================================
-Exiting script:  \"${scrfunc_fn}\"
-In directory:    \"${scrfunc_dir}\"
-========================================================================"
+job_postamble
 #
 #-----------------------------------------------------------------------
 #