[develop] Integrate UW CLI tool for templater and remove external dependency.

.cicd/scripts/srw_ftest.sh

@@ -88,8 +88,6 @@ module load wflow_${platform,,}
 conda activate srw_app
 set -e -u
 
-export PYTHONPATH=${workspace}/ush/python_utils/workflow-tools:${workspace}/ush/python_utils/workflow-tools/src
-
 # Adjust for strict limitation of stack size 
 sed "s|ulimit -s unlimited;|ulimit -S -s unlimited;|" -i ${workspace}/ush/machine/hera.yaml
 

.github/workflows/python_tests.yaml

@@ -32,12 +32,12 @@ jobs:
 
       - name: Checkout externals
         run: |
-          ./manage_externals/checkout_externals ufs-weather-model workflow-tools
+          ./manage_externals/checkout_externals ufs-weather-model
 
       - name: Lint the python code
         run: |
           micromamba activate srw_app
-          export PYTHONPATH=$(pwd)/ush:$(pwd)/ush/python_utils/workflow-tools:$(pwd)/ush/python_utils/workflow-tools/src
+          export PYTHONPATH=$(pwd)/ush
           pylint --ignore-imports=yes tests/test_python/
           pylint ush/create_*.py
           pylint ush/generate_FV3LAM_wflow.py
@@ -47,7 +47,7 @@ jobs:
           # exclude test_retrieve_data that is tested in functional test
           micromamba activate srw_app
           export UNIT_TEST=True
-          export PYTHONPATH=$(pwd)/ush:$(pwd)/ush/python_utils/workflow-tools:$(pwd)/ush/python_utils/workflow-tools/src
+          export PYTHONPATH=$(pwd)/ush
           python -m unittest -b tests/test_python/*.py
 
       - name: Run python functional tests

.pylintrc

@@ -182,6 +182,7 @@ function-naming-style=snake_case
 good-names=i,
            j,
            k,
+           e,
            ex,
            Run,
            _
@@ -422,7 +423,9 @@ disable=raw-checker-failed,
         useless-suppression,
         deprecated-pragma,
         use-symbolic-message-instead,
-        logging-fstring-interpolation
+        logging-fstring-interpolation,
+        too-many-locals,
+        similarities
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

Externals.cfg

@@ -43,14 +43,5 @@ hash = 694a139
 local_path = sorc/AQM-utils
 required = True
 
-[workflow-tools]
-protocol = git
-repo_url = https://github.com/ufs-community/workflow-tools
-# Specify either a branch name or a hash but not both.
-# branch = develop
-hash = e1b3b6f
-local_path = ush/python_utils/workflow-tools
-required = True
-
 [externals_description]
 schema_version = 1.0.0

docs/UsersGuide/source/BackgroundInfo/Components.rst

@@ -89,7 +89,7 @@ For more information on NEXUS, visit the GitHub repository at https://github.com
 Unified Workflow Tools
 ========================
 
-The Unified Workflow (UW) is a set of tools intended to unify the workflow for various UFS applications under one framework. The UW toolkit currently includes templater and config tools, which have been incorporated into the SRW App workflow and will soon be incorporated into other UFS repositories. Additional tools are under development. More details about the UW can be found in the `workflow-tools <https://github.com/ufs-community/workflow-tools>`__ GitHub repository and in the `UW Documentation <https://unified-workflow.readthedocs.io/en/latest/>`__.
+The Unified Workflow (UW) is a set of tools intended to unify the workflow for various UFS applications under one framework. The UW toolkit currently includes template and config tools, which have been incorporated into the SRW App workflow and will soon be incorporated into other UFS repositories. Additional tools are under development. More details about the UW can be found in the `workflow-tools <https://github.com/ufs-community/workflow-tools>`__ GitHub repository and in the `UW Documentation <https://unified-workflow.readthedocs.io/en/latest/>`__.
 
 Build System and Workflow
 =========================

modulefiles/wflow_cheyenne.lua

@@ -13,7 +13,7 @@ load("rocoto")
 unload("python")
 
 load("conda")
-load("set_pythonpath")
+
 
 if mode() == "load" then
    LmodMsgRaw([===[Please do the following to activate conda:

modulefiles/wflow_derecho.lua

@@ -13,7 +13,7 @@ load("rocoto")
 unload("python")
 
 load("conda")
-load("set_pythonpath")
+
 
 if mode() == "load" then
    LmodMsgRaw([===[Please do the following to activate conda:

modulefiles/wflow_gaea-c5.lua

@@ -6,7 +6,7 @@ the NOAA RDHPC machine Gaea C5
 whatis([===[Loads libraries needed for running the UFS SRW App on gaea ]===])
 
 unload("python")
-load("set_pythonpath")
+
 load("conda")
 prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/C5/rocoto/modulefiles")
 load("rocoto")

modulefiles/wflow_gaea.lua

@@ -6,7 +6,7 @@ the NOAA RDHPC machine Gaea
 whatis([===[Loads libraries needed for running the UFS SRW App on gaea ]===])
 
 unload("python")
-load("set_pythonpath")
+
 load("conda")
 prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/rocoto/modulefiles")
 load("rocoto")

modulefiles/wflow_hera.lua

@@ -6,7 +6,7 @@ the NOAA RDHPC machine Hera
 whatis([===[Loads libraries needed for running the UFS SRW App on Hera ]===])
 
 load("rocoto")
-load("set_pythonpath")
+
 load("conda")
 
 if mode() == "load" then

modulefiles/wflow_hercules.lua

@@ -7,7 +7,7 @@ whatis([===[Loads libraries needed for running SRW on Hercules ]===])
 
 load("contrib")
 load("rocoto")
-load("set_pythonpath")
+
 
 unload("python")
 load("conda")

modulefiles/wflow_jet.lua

@@ -6,7 +6,7 @@ the NOAA RDHPC machine Jet
 whatis([===[Loads libraries needed for running the UFS SRW App on Jet ]===])
 
 load("rocoto")
-load("set_pythonpath")
+
 
 load("conda")
 

modulefiles/wflow_linux.lua

@@ -16,8 +16,6 @@ prepend_path("PATH", pathJoin(rocoto_path,"bin"))
 local srw_path="/home/username/ufs-srweather-app"
 prepend_path("PATH", pathJoin(srw_path, "ush/rocoto_fake_slurm"))
 
--- set python path
-load("set_pythonpath")
 
 -- display conda activation message
 if mode() == "load" then

modulefiles/wflow_macos.lua

@@ -16,8 +16,6 @@ prepend_path("PATH", pathJoin(rocoto_path,"bin"))
 local srw_path="/Users/username/ufs-srweather-app"
 prepend_path("PATH", pathJoin(srw_path, "ush/rocoto_fake_slurm"))
 
--- set python path
-load("set_pythonpath")
 
 -- display conda activation message
 if mode() == "load" then

modulefiles/wflow_noaacloud.lua

@@ -7,7 +7,7 @@ whatis([===[Loads libraries needed for running the UFS SRW App on NOAA cloud ]==
 
 prepend_path("MODULEPATH","/apps/modules/modulefiles")
 load("rocoto")
-load("set_pythonpath")
+
 
 
 load("conda")

modulefiles/wflow_odin.lua

@@ -5,7 +5,7 @@ the NSSL machine Odin
 
 whatis([===[Loads libraries needed for running the UFS SRW App on Odin ]===])
 
-load("set_pythonpath")
+
 
 if mode() == "load" then
   -- >>> conda initialize >>>

modulefiles/wflow_orion.lua

@@ -8,7 +8,7 @@ whatis([===[Loads libraries needed for running SRW on Orion ]===])
 load("contrib")
 load("rocoto")
 load("wget")
-load("set_pythonpath")
+
 
 unload("python")
 load("conda")

modulefiles/wflow_singularity.lua

@@ -4,7 +4,7 @@ a singularity container
 ]])
 
 whatis([===[Loads libraries needed for running the UFS SRW App in a singularity container]===])
-load("set_pythonpath")
+
 
 load("conda")
 

modulefiles/wflow_wcoss2.lua

@@ -7,7 +7,7 @@ whatis([===[Loads libraries needed for running the UFS SRW App on WCOSS2 ]===])
 
 load(pathJoin("intel", os.getenv("intel_ver")))
 load(pathJoin("python", os.getenv("python_ver")))
-load("set_pythonpath")
+
 
 prepend_path("MODULEPATH","/apps/ops/test/nco/modulefiles")
 load(pathJoin("core/rocoto", os.getenv("rocoto_ver")))

scripts/exregional_run_met_genensprod_or_ensemblestat.sh

@@ -388,31 +388,30 @@ settings="\
   'field_thresholds': '${FIELD_THRESHOLDS:-}'
 "
 
-# Store the settings in a temporary file
+# Render the template to create a METplus configuration file
 tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)")
 cat > $tmpfile << EOF
 $settings
 EOF
-#
-# Call the python script to generate the METplus configuration file from
-# the jinja template.
-#
-python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \
-  -c "${tmpfile}" \
+
+uw template render \
   -i ${metplus_config_tmpl_fp} \
-  -o ${metplus_config_fp} || \
-print_err_msg_exit "\
-Call to workflow-tools templater to generate a METplus
-configuration file from a jinja template failed.  Parameters passed
-to this script are:
-  Full path to template METplus configuration file:
-    metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\"
-  Full path to output METplus configuration file:
-    metplus_config_fp = \"${metplus_config_fp}\"
-  Full path to configuration file:
-    ${tmpfile}
-"
+  -o ${metplus_config_fp} \
+  -v \
+  --values-file "${tmpfile}"
+
+err=$?
 rm $tmpfile
+if [ $err -ne 0 ]; then
+  message_txt="Error rendering template for METplus config.
+     Contents of input are:
+$settings"
+  if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then
+    err_exit "${message_txt}"
+  else
+    print_err_msg_exit "${message_txt}"
+  fi
+fi
 #
 #-----------------------------------------------------------------------
 #

scripts/exregional_run_met_gridstat_or_pointstat_vx.sh

@@ -389,31 +389,31 @@ settings="\
   'field_thresholds': '${FIELD_THRESHOLDS:-}'
 "
 
-# Store the settings in a temporary file
+# Render the template to create a METplus configuration file
 tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)")
 cat > $tmpfile << EOF
 $settings
 EOF
-#
-# Call the python script to generate the METplus configuration file from
-# the jinja template.
-#
-python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \
-  -c "${tmpfile}" \
+
+uw template render \
   -i ${metplus_config_tmpl_fp} \
-  -o ${metplus_config_fp} || \
-print_err_msg_exit "\
-Call to workflow-tools templater to generate a METplus
-configuration file from a jinja template failed.  Parameters passed
-to this script are:
-  Full path to template METplus configuration file:
-    metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\"
-  Full path to output METplus configuration file:
-    metplus_config_fp = \"${metplus_config_fp}\"
-  Full path to configuration file:
-    ${tmpfile}
-"
+  -o ${metplus_config_fp} \
+  -v \
+  --values-file "${tmpfile}"
+
+err=$?
 rm $tmpfile
+if [ $err -ne 0 ]; then
+  message_txt="Error rendering template for METplus config.
+     Contents of input are:
+$settings"
+  if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then
+    err_exit "${message_txt}"
+  else
+    print_err_msg_exit "${message_txt}"
+  fi
+fi
+
 #
 #-----------------------------------------------------------------------
 #

scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh

@@ -351,32 +351,31 @@ settings="\
   'field_thresholds': '${FIELD_THRESHOLDS:-}'
 "
 
-# Store the settings in a temporary file
+# Render the template to create a METplus configuration file
 tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)")
 cat > $tmpfile << EOF
 $settings
 EOF
-#
-# Call the python script to generate the METplus configuration file from
-# the jinja template.
-#
 
-python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \
-  -c "${tmpfile}" \
+uw template render \
   -i ${metplus_config_tmpl_fp} \
-  -o ${metplus_config_fp} || \
-print_err_msg_exit "\
-Call to workflow-tools templater to generate a METplus
-configuration file from a jinja template failed.  Parameters passed
-to this script are:
-  Full path to template METplus configuration file:
-    metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\"
-  Full path to output METplus configuration file:
-    metplus_config_fp = \"${metplus_config_fp}\"
-  Full path to configuration file:
-    ${tmpfile}
-"
+  -o ${metplus_config_fp} \
+  -v \
+  --values-file "${tmpfile}"
+
+err=$?
 rm $tmpfile
+if [ $err -ne 0 ]; then
+  message_txt="Error rendering template for METplus config.
+     Contents of input are:
+$settings"
+  if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then
+    err_exit "${message_txt}"
+  else
+    print_err_msg_exit "${message_txt}"
+  fi
+fi
+
 #
 #-----------------------------------------------------------------------
 #