diff --git a/.cicd/scripts/srw_ftest.sh b/.cicd/scripts/srw_ftest.sh index 4b8b672905..5479e8b46d 100755 --- a/.cicd/scripts/srw_ftest.sh +++ b/.cicd/scripts/srw_ftest.sh @@ -88,8 +88,6 @@ module load wflow_${platform,,} conda activate srw_app set -e -u -export PYTHONPATH=${workspace}/ush/python_utils/workflow-tools:${workspace}/ush/python_utils/workflow-tools/src - # Adjust for strict limitation of stack size sed "s|ulimit -s unlimited;|ulimit -S -s unlimited;|" -i ${workspace}/ush/machine/hera.yaml diff --git a/.github/workflows/python_tests.yaml b/.github/workflows/python_tests.yaml index 1bab329c0c..0e71f8d72d 100644 --- a/.github/workflows/python_tests.yaml +++ b/.github/workflows/python_tests.yaml @@ -32,12 +32,12 @@ jobs: - name: Checkout externals run: | - ./manage_externals/checkout_externals ufs-weather-model workflow-tools + ./manage_externals/checkout_externals ufs-weather-model - name: Lint the python code run: | micromamba activate srw_app - export PYTHONPATH=$(pwd)/ush:$(pwd)/ush/python_utils/workflow-tools:$(pwd)/ush/python_utils/workflow-tools/src + export PYTHONPATH=$(pwd)/ush pylint --ignore-imports=yes tests/test_python/ pylint ush/create_*.py pylint ush/generate_FV3LAM_wflow.py @@ -47,7 +47,7 @@ jobs: # exclude test_retrieve_data that is tested in functional test micromamba activate srw_app export UNIT_TEST=True - export PYTHONPATH=$(pwd)/ush:$(pwd)/ush/python_utils/workflow-tools:$(pwd)/ush/python_utils/workflow-tools/src + export PYTHONPATH=$(pwd)/ush python -m unittest -b tests/test_python/*.py - name: Run python functional tests diff --git a/.pylintrc b/.pylintrc index 0cb488e3d5..1d441d141f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -182,6 +182,7 @@ function-naming-style=snake_case good-names=i, j, k, + e, ex, Run, _ @@ -422,7 +423,9 @@ disable=raw-checker-failed, useless-suppression, deprecated-pragma, use-symbolic-message-instead, - logging-fstring-interpolation + logging-fstring-interpolation, + too-many-locals, + similarities # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/Externals.cfg b/Externals.cfg index b4a5840568..15301c921f 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -43,14 +43,5 @@ hash = 694a139 local_path = sorc/AQM-utils required = True -[workflow-tools] -protocol = git -repo_url = https://github.com/ufs-community/workflow-tools -# Specify either a branch name or a hash but not both. -# branch = develop -hash = e1b3b6f -local_path = ush/python_utils/workflow-tools -required = True - [externals_description] schema_version = 1.0.0 diff --git a/docs/UsersGuide/source/BackgroundInfo/Components.rst b/docs/UsersGuide/source/BackgroundInfo/Components.rst index fd362ec691..70711ed315 100644 --- a/docs/UsersGuide/source/BackgroundInfo/Components.rst +++ b/docs/UsersGuide/source/BackgroundInfo/Components.rst @@ -89,7 +89,7 @@ For more information on NEXUS, visit the GitHub repository at https://github.com Unified Workflow Tools ======================== -The Unified Workflow (UW) is a set of tools intended to unify the workflow for various UFS applications under one framework. The UW toolkit currently includes templater and config tools, which have been incorporated into the SRW App workflow and will soon be incorporated into other UFS repositories. Additional tools are under development. More details about the UW can be found in the `workflow-tools `__ GitHub repository and in the `UW Documentation `__. +The Unified Workflow (UW) is a set of tools intended to unify the workflow for various UFS applications under one framework. The UW toolkit currently includes template and config tools, which have been incorporated into the SRW App workflow and will soon be incorporated into other UFS repositories. Additional tools are under development. More details about the UW can be found in the `workflow-tools `__ GitHub repository and in the `UW Documentation `__. Build System and Workflow ========================= diff --git a/modulefiles/set_pythonpath.lua b/modulefiles/set_pythonpath.lua deleted file mode 100644 index e816ed6cfc..0000000000 --- a/modulefiles/set_pythonpath.lua +++ /dev/null @@ -1,13 +0,0 @@ -help([[ -This module sets the PYTHONPATH in the user environment to allow the -workflow tools to be imported -]]) - -whatis([===[Sets paths for using workflow-tools with SRW]===]) - -local mod_path, mod_file = splitFileName(myFileName()) -local uwtools_scripts_path = pathJoin(mod_path, "/../ush/python_utils/workflow-tools") -local uwtools_package_path = pathJoin(mod_path, "/../ush/python_utils/workflow-tools/src/") - -prepend_path("PYTHONPATH", uwtools_scripts_path) -prepend_path("PYTHONPATH", uwtools_package_path) diff --git a/modulefiles/wflow_cheyenne.lua b/modulefiles/wflow_cheyenne.lua index f3fec407c7..fd4bc3eae5 100644 --- a/modulefiles/wflow_cheyenne.lua +++ b/modulefiles/wflow_cheyenne.lua @@ -13,7 +13,7 @@ load("rocoto") unload("python") load("conda") -load("set_pythonpath") + if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: diff --git a/modulefiles/wflow_derecho.lua b/modulefiles/wflow_derecho.lua index f32cfdc298..d9a3e24e2f 100644 --- a/modulefiles/wflow_derecho.lua +++ b/modulefiles/wflow_derecho.lua @@ -13,7 +13,7 @@ load("rocoto") unload("python") load("conda") -load("set_pythonpath") + if mode() == "load" then LmodMsgRaw([===[Please do the following to activate conda: diff --git a/modulefiles/wflow_gaea-c5.lua b/modulefiles/wflow_gaea-c5.lua index 5f9d93e58f..7cfd1b4529 100644 --- a/modulefiles/wflow_gaea-c5.lua +++ b/modulefiles/wflow_gaea-c5.lua @@ -6,7 +6,7 @@ the NOAA RDHPC machine Gaea C5 whatis([===[Loads libraries needed for running the UFS SRW App on gaea ]===]) unload("python") -load("set_pythonpath") + load("conda") prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/C5/rocoto/modulefiles") load("rocoto") diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua index 7e30a20649..83c83d5e92 100644 --- a/modulefiles/wflow_gaea.lua +++ b/modulefiles/wflow_gaea.lua @@ -6,7 +6,7 @@ the NOAA RDHPC machine Gaea whatis([===[Loads libraries needed for running the UFS SRW App on gaea ]===]) unload("python") -load("set_pythonpath") + load("conda") prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/rocoto/modulefiles") load("rocoto") diff --git a/modulefiles/wflow_hera.lua b/modulefiles/wflow_hera.lua index 832f40bb73..56ef72ca9a 100644 --- a/modulefiles/wflow_hera.lua +++ b/modulefiles/wflow_hera.lua @@ -6,7 +6,7 @@ the NOAA RDHPC machine Hera whatis([===[Loads libraries needed for running the UFS SRW App on Hera ]===]) load("rocoto") -load("set_pythonpath") + load("conda") if mode() == "load" then diff --git a/modulefiles/wflow_hercules.lua b/modulefiles/wflow_hercules.lua index 978ef14711..4afe59c4d6 100644 --- a/modulefiles/wflow_hercules.lua +++ b/modulefiles/wflow_hercules.lua @@ -7,7 +7,7 @@ whatis([===[Loads libraries needed for running SRW on Hercules ]===]) load("contrib") load("rocoto") -load("set_pythonpath") + unload("python") load("conda") diff --git a/modulefiles/wflow_jet.lua b/modulefiles/wflow_jet.lua index c3e740537a..fcb46c0c07 100644 --- a/modulefiles/wflow_jet.lua +++ b/modulefiles/wflow_jet.lua @@ -6,7 +6,7 @@ the NOAA RDHPC machine Jet whatis([===[Loads libraries needed for running the UFS SRW App on Jet ]===]) load("rocoto") -load("set_pythonpath") + load("conda") diff --git a/modulefiles/wflow_linux.lua b/modulefiles/wflow_linux.lua index 4c2305f047..53a19059b0 100644 --- a/modulefiles/wflow_linux.lua +++ b/modulefiles/wflow_linux.lua @@ -16,8 +16,6 @@ prepend_path("PATH", pathJoin(rocoto_path,"bin")) local srw_path="/home/username/ufs-srweather-app" prepend_path("PATH", pathJoin(srw_path, "ush/rocoto_fake_slurm")) --- set python path -load("set_pythonpath") -- display conda activation message if mode() == "load" then diff --git a/modulefiles/wflow_macos.lua b/modulefiles/wflow_macos.lua index 987adcdd52..f515130b86 100644 --- a/modulefiles/wflow_macos.lua +++ b/modulefiles/wflow_macos.lua @@ -16,8 +16,6 @@ prepend_path("PATH", pathJoin(rocoto_path,"bin")) local srw_path="/Users/username/ufs-srweather-app" prepend_path("PATH", pathJoin(srw_path, "ush/rocoto_fake_slurm")) --- set python path -load("set_pythonpath") -- display conda activation message if mode() == "load" then diff --git a/modulefiles/wflow_noaacloud.lua b/modulefiles/wflow_noaacloud.lua index e4a355f0b2..ebf907545b 100644 --- a/modulefiles/wflow_noaacloud.lua +++ b/modulefiles/wflow_noaacloud.lua @@ -7,7 +7,7 @@ whatis([===[Loads libraries needed for running the UFS SRW App on NOAA cloud ]== prepend_path("MODULEPATH","/apps/modules/modulefiles") load("rocoto") -load("set_pythonpath") + load("conda") diff --git a/modulefiles/wflow_odin.lua b/modulefiles/wflow_odin.lua index f042b00894..be3f9607e8 100644 --- a/modulefiles/wflow_odin.lua +++ b/modulefiles/wflow_odin.lua @@ -5,7 +5,7 @@ the NSSL machine Odin whatis([===[Loads libraries needed for running the UFS SRW App on Odin ]===]) -load("set_pythonpath") + if mode() == "load" then -- >>> conda initialize >>> diff --git a/modulefiles/wflow_orion.lua b/modulefiles/wflow_orion.lua index 0a9ef26fce..711991bb09 100644 --- a/modulefiles/wflow_orion.lua +++ b/modulefiles/wflow_orion.lua @@ -8,7 +8,7 @@ whatis([===[Loads libraries needed for running SRW on Orion ]===]) load("contrib") load("rocoto") load("wget") -load("set_pythonpath") + unload("python") load("conda") diff --git a/modulefiles/wflow_singularity.lua b/modulefiles/wflow_singularity.lua index 3e1de292e1..7f627f2ee9 100644 --- a/modulefiles/wflow_singularity.lua +++ b/modulefiles/wflow_singularity.lua @@ -4,7 +4,7 @@ a singularity container ]]) whatis([===[Loads libraries needed for running the UFS SRW App in a singularity container]===]) -load("set_pythonpath") + load("conda") diff --git a/modulefiles/wflow_wcoss2.lua b/modulefiles/wflow_wcoss2.lua index 4212f17493..1af7e2d0f0 100644 --- a/modulefiles/wflow_wcoss2.lua +++ b/modulefiles/wflow_wcoss2.lua @@ -7,7 +7,7 @@ whatis([===[Loads libraries needed for running the UFS SRW App on WCOSS2 ]===]) load(pathJoin("intel", os.getenv("intel_ver"))) load(pathJoin("python", os.getenv("python_ver"))) -load("set_pythonpath") + prepend_path("MODULEPATH","/apps/ops/test/nco/modulefiles") load(pathJoin("core/rocoto", os.getenv("rocoto_ver"))) diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 9ccb2d1b25..cf72c723a4 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -388,31 +388,30 @@ settings="\ 'field_thresholds': '${FIELD_THRESHOLDS:-}' " -# Store the settings in a temporary file +# Render the template to create a METplus configuration file tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") cat > $tmpfile << EOF $settings EOF -# -# Call the python script to generate the METplus configuration file from -# the jinja template. -# -python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ - -c "${tmpfile}" \ + +uw template render \ -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ -print_err_msg_exit "\ -Call to workflow-tools templater to generate a METplus -configuration file from a jinja template failed. Parameters passed -to this script are: - Full path to template METplus configuration file: - metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" - Full path to output METplus configuration file: - metplus_config_fp = \"${metplus_config_fp}\" - Full path to configuration file: - ${tmpfile} -" + -o ${metplus_config_fp} \ + -v \ + --values-file "${tmpfile}" + +err=$? rm $tmpfile +if [ $err -ne 0 ]; then + message_txt="Error rendering template for METplus config. + Contents of input are: +$settings" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index 46c93ee299..1dd7adf973 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -389,31 +389,31 @@ settings="\ 'field_thresholds': '${FIELD_THRESHOLDS:-}' " -# Store the settings in a temporary file +# Render the template to create a METplus configuration file tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") cat > $tmpfile << EOF $settings EOF -# -# Call the python script to generate the METplus configuration file from -# the jinja template. -# -python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ - -c "${tmpfile}" \ + +uw template render \ -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ -print_err_msg_exit "\ -Call to workflow-tools templater to generate a METplus -configuration file from a jinja template failed. Parameters passed -to this script are: - Full path to template METplus configuration file: - metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" - Full path to output METplus configuration file: - metplus_config_fp = \"${metplus_config_fp}\" - Full path to configuration file: - ${tmpfile} -" + -o ${metplus_config_fp} \ + -v \ + --values-file "${tmpfile}" + +err=$? rm $tmpfile +if [ $err -ne 0 ]; then + message_txt="Error rendering template for METplus config. + Contents of input are: +$settings" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index bf3a8280a1..b6c1d9ab07 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -351,32 +351,31 @@ settings="\ 'field_thresholds': '${FIELD_THRESHOLDS:-}' " -# Store the settings in a temporary file +# Render the template to create a METplus configuration file tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") cat > $tmpfile << EOF $settings EOF -# -# Call the python script to generate the METplus configuration file from -# the jinja template. -# -python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ - -c "${tmpfile}" \ +uw template render \ -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ -print_err_msg_exit "\ -Call to workflow-tools templater to generate a METplus -configuration file from a jinja template failed. Parameters passed -to this script are: - Full path to template METplus configuration file: - metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" - Full path to output METplus configuration file: - metplus_config_fp = \"${metplus_config_fp}\" - Full path to configuration file: - ${tmpfile} -" + -o ${metplus_config_fp} \ + -v \ + --values-file "${tmpfile}" + +err=$? rm $tmpfile +if [ $err -ne 0 ]; then + message_txt="Error rendering template for METplus config. + Contents of input are: +$settings" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index 89c128eeed..ea9a7fe475 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -309,31 +309,33 @@ settings="\ 'accum_no_pad': '${ACCUM_NO_PAD:-}' 'field_thresholds': '${FIELD_THRESHOLDS:-}' " -# Store the settings in a temporary file + +# Render the template to create a METplus configuration file tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") cat > $tmpfile << EOF $settings EOF -# -# Call the python script to generate the METplus configuration file from -# the jinja template. -# -python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ - -c "${tmpfile}" \ + +uw template render \ -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ -print_err_msg_exit "\ -Call to workflow-tools templater to generate a METplus -configuration file from a jinja template failed. Parameters passed -to this script are: - Full path to template METplus configuration file: - metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" - Full path to output METplus configuration file: - metplus_config_fp = \"${metplus_config_fp}\" - Full path to configuration file: - ${tmpfile} -" + -o ${metplus_config_fp} \ + -v \ + --values-file "${tmpfile}" + +err=$? rm $tmpfile +if [ $err -ne 0 ]; then + message_txt="Error rendering template for METplus config. + Contents of input are: +$settings" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi + + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index be0f0a2b91..c5305643b5 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -267,31 +267,32 @@ settings="\ 'accum_no_pad': '${ACCUM_NO_PAD:-}' 'field_thresholds': '${FIELD_THRESHOLDS:-}' " -# Store the settings in a temporary file + +# Render the template to create a METplus configuration file tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") cat > $tmpfile << EOF $settings EOF -# -# Call the python script to generate the METplus configuration file from -# the jinja template. -# -python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ - -c ${tmpfile} \ + +uw template render \ -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ -print_err_msg_exit "\ -Call to workflow-tools templater.py to generate a METplus -configuration file from a jinja template failed. Parameters passed -to this script are: - Full path to template METplus configuration file: - metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" - Full path to output METplus configuration file: - metplus_config_fp = \"${metplus_config_fp}\" - Full path to configuration file: - ${tmpfile} -" + -o ${metplus_config_fp} \ + -v \ + --values-file "${tmpfile}" + +err=$? rm $tmpfile +if [ $err -ne 0 ]; then + message_txt="Error rendering template for METplus config. + Contents of input are: +$settings" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index cb047163cb..11438fd907 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -343,32 +343,30 @@ settings="\ 'accum_no_pad': '${ACCUM_NO_PAD:-}' 'field_thresholds': '${FIELD_THRESHOLDS:-}' " -# Store the settings in a temporary file +# Render the template to create a METplus configuration file tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") cat > $tmpfile << EOF $settings EOF -# -# Call the python script to generate the METplus configuration file from -# the jinja template. -# -python3 $USHdir/python_utils/workflow-tools/scripts/templater.py \ - -c ${tmpfile} \ +uw template render \ -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} || \ -print_err_msg_exit "\ -Call to workflow-tools templater.py to generate a METplus -configuration file from a jinja template failed. Parameters passed -to this script are: - Full path to template METplus configuration file: - metplus_config_tmpl_fp = \"${metplus_config_tmpl_fp}\" - Full path to output METplus configuration file: - metplus_config_fp = \"${metplus_config_fp}\" - Full path to configuration file: - ${tmpfile} -" + -o ${metplus_config_fp} \ + -v \ + --values-file "${tmpfile}" + +err=$? rm $tmpfile +if [ $err -ne 0 ]; then + message_txt="Error rendering template for METplus config. + Contents of input are: +$settings" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi +fi # #----------------------------------------------------------------------- # diff --git a/ush/create_aqm_rc_file.py b/ush/create_aqm_rc_file.py index acf675a3b9..5608e4cbf2 100644 --- a/ush/create_aqm_rc_file.py +++ b/ush/create_aqm_rc_file.py @@ -3,25 +3,23 @@ Function that creates the config file for running AQM. """ +import argparse import os import sys -import argparse -from textwrap import dedent import tempfile +from subprocess import STDOUT, CalledProcessError, check_output +from textwrap import dedent from python_utils import ( + cfg_to_yaml_str, + flatten_dict, import_vars, + load_shell_config, + print_info_msg, print_input_args, str_to_type, - print_info_msg, - cfg_to_yaml_str, - load_shell_config, - flatten_dict ) -# These come from ush/python_utils/workflow-tools -from scripts.templater import set_template - def create_aqm_rc_file(cdate, run_dir, init_concentrations): """ Creates an aqm.rc file in the specified run directory @@ -136,17 +134,29 @@ def create_aqm_rc_file(cdate, run_dir, init_concentrations): suffix=".yaml") as tmpfile: tmpfile.write(settings_str) tmpfile.seek(0) - set_template( - [ - "-q", - "-c", - tmpfile.name, + cmd = " ".join(["uw template render", "-i", AQM_RC_TMPL_FP, "-o", aqm_rc_fp, + "-v", + "--values-file", + tmpfile.name, ] ) + indent = " " + output = "" + try: + output = check_output(cmd, encoding="utf=8", shell=True, + stderr=STDOUT, text=True) + except CalledProcessError as e: + output = e.output + print(f"Failed with status: {e.returncode}") + sys.exit(1) + finally: + print("Output:") + for line in output.split("\n"): + print(f"{indent * 2}{line}") return True def parse_args(argv): diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index 3ef9ec3901..40f5e0deee 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -4,25 +4,22 @@ Function to create a diag_table file for the FV3 model using a template. """ +import argparse import os import sys -import argparse -from textwrap import dedent import tempfile - +from subprocess import STDOUT, CalledProcessError, check_output +from textwrap import dedent from python_utils import ( - import_vars, - print_input_args, - print_info_msg, cfg_to_yaml_str, - load_shell_config, flatten_dict, + import_vars, + load_shell_config, + print_info_msg, + print_input_args, ) -# These come from ush/python_utils/workflow-tools -from scripts.templater import set_template - def create_diag_table_file(run_dir): """Creates a diagnostic table file for each cycle to be run @@ -83,10 +80,26 @@ def create_diag_table_file(run_dir): suffix=".yaml") as tmpfile: tmpfile.write(settings_str) tmpfile.seek(0) - # set_template does its own error handling - set_template( - ["-c", tmpfile.name, "-i", DIAG_TABLE_TMPL_FP, "-o", diag_table_fp] + cmd = " ".join(["uw template render", + "-i", DIAG_TABLE_TMPL_FP, + "-o", diag_table_fp, + "-v", + "--values-file", tmpfile.name, + ] ) + indent = " " + output = "" + try: + output = check_output(cmd, encoding="utf=8", shell=True, + stderr=STDOUT, text=True) + except CalledProcessError as e: + output = e.output + print(f"Failed with status: {e.returncode}") + sys.exit(1) + finally: + print("Output:") + for line in output.split("\n"): + print(f"{indent * 2}{line}") return True diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 68d68b724e..79994aa695 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -3,26 +3,24 @@ Create a model_configure file for the FV3 forecast model from a template. """ +import argparse import os import sys -import argparse -from textwrap import dedent import tempfile +from textwrap import dedent +from subprocess import STDOUT, CalledProcessError, check_output from python_utils import ( + cfg_to_yaml_str, + flatten_dict, import_vars, + load_shell_config, + lowercase, + print_info_msg, print_input_args, str_to_type, - print_info_msg, - lowercase, - cfg_to_yaml_str, - load_shell_config, - flatten_dict, ) -# These come from ush/python_utils/workflow-tools -from scripts.templater import set_template - def create_model_configure_file( cdate, fcst_len_hrs, fhrot, run_dir, sub_hourly_post, dt_subhourly_post_mnts, dt_atmos @@ -136,10 +134,7 @@ def create_model_configure_file( ) # # If not using the write-component (aka quilting), set those variables - # needed for quilting in the jinja template for the model configuration - # file (MODEL_CONFIG_TMPL_FP) to "None". This is necessary because - # otherwise, the run_fcst task will fail in the call to set_template() - # below with a "variables are not provided" message. + # needed for quilting to None so that it gets rendered in the template appropriately. # else: settings.update( @@ -228,18 +223,26 @@ def create_model_configure_file( prefix="model_config_settings.") as tmpfile: tmpfile.write(settings_str) tmpfile.seek(0) - # set_template does its own error handling - set_template( - [ - "-c", - tmpfile.name, - "-i", - MODEL_CONFIG_TMPL_FP, - "-o", - model_config_fp, + cmd = " ".join(["uw template render", + "-i", MODEL_CONFIG_TMPL_FP, + "-o", model_config_fp, + "-v", + "--values-file", tmpfile.name, ] ) - + indent = " " + output = "" + try: + output = check_output(cmd, encoding="utf=8", shell=True, + stderr=STDOUT, text=True) + except CalledProcessError as e: + output = e.output + print(f"Failed with status: {e.returncode}") + sys.exit(1) + finally: + print("Output:") + for line in output.split("\n"): + print(f"{indent * 2}{line}") return True diff --git a/ush/create_nems_configure_file.py b/ush/create_nems_configure_file.py index 7a2ef0723f..a6ba1cbd6b 100644 --- a/ush/create_nems_configure_file.py +++ b/ush/create_nems_configure_file.py @@ -5,24 +5,22 @@ model(s) from a template. """ +import argparse import os import sys -import argparse import tempfile +from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent from python_utils import ( - import_vars, - print_input_args, - print_info_msg, cfg_to_yaml_str, - load_shell_config, flatten_dict, + import_vars, + load_shell_config, + print_info_msg, + print_input_args, ) -# These come from ush/python_utils/workflow-tools -from scripts.templater import set_template - def create_nems_configure_file(run_dir): """ Creates a nems configuration file in the specified run directory @@ -100,7 +98,27 @@ def create_nems_configure_file(run_dir): tmpfile.write(settings_str) tmpfile.seek(0) - set_template(["-c", tmpfile.name, "-i", NEMS_CONFIG_TMPL_FP, "-o", nems_config_fp]) + cmd = " ".join(["uw template render", + "-i", NEMS_CONFIG_TMPL_FP, + "-o", nems_config_fp, + "-v", + "--values-file", tmpfile.name, + ] + ) + + indent = " " + output = "" + try: + output = check_output(cmd, encoding="utf=8", shell=True, + stderr=STDOUT, text=True) + except CalledProcessError as e: + output = e.output + print(f"Failed with status: {e.returncode}") + sys.exit(1) + finally: + print("Output:") + for line in output.split("\n"): + print(f"{indent * 2}{line}") return True def parse_args(argv): diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 516cbf94cb..dfd42ab489 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -8,10 +8,11 @@ # pylint: disable=invalid-name import argparse -import os import logging -from textwrap import dedent +import os import sys +from subprocess import STDOUT, CalledProcessError, check_output +from textwrap import dedent from python_utils import ( log_info, @@ -34,9 +35,6 @@ from set_namelist import set_namelist from check_python_version import check_python_version -# These come from ush/python_utils/workflow-tools -from scripts.templater import set_template - # pylint: disable=too-many-locals,too-many-branches, too-many-statements def generate_FV3LAM_wflow( ushdir, @@ -113,13 +111,29 @@ def generate_FV3LAM_wflow( # Call the python script to generate the experiment's XML file # rocoto_yaml_fp = expt_config["workflow"]["ROCOTO_YAML_FP"] - args = ["-o", wflow_xml_fp, - "-i", template_xml_fp, - "-c", rocoto_yaml_fp, - ] - if not debug: - args.append("-q") - set_template(args) + cmd = " ".join(["uw template render", + "-i", template_xml_fp, + "-o", wflow_xml_fp, + "-v", + "--values-file", rocoto_yaml_fp, + ] + ) + + indent = " " + output = "" + logfunc = logging.info + try: + output = check_output(cmd, encoding="utf=8", shell=True, + stderr=STDOUT, text=True) + except CalledProcessError as e: + logfunc = logging.error + output = e.output + logging.exception(("Failed with status: %s", e.returncode)) + raise + finally: + logfunc("Output:") + for line in output.split("\n"): + logfunc("%s%s", indent * 2, line) # # ----------------------------------------------------------------------- # diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 8765c3f698..89f3addf41 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -113,8 +113,6 @@ for the workflow task specified by task_name failed: BUILD_MOD_FN = \"${BUILD_MOD_FN}\"" fi -module load set_pythonpath || print_err_msg_exit "\ - Loading the module to set PYTHONPATH for workflow-tools failed." # #----------------------------------------------------------------------- #