From 7dfbe3631db6028d52de08125abd8e28c4252e22 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 30 Jan 2025 11:55:16 -0700 Subject: [PATCH 01/11] very preliminary updates in template.cupid for cupid workflow --- machines/template.cupid | 82 +++++++++++++++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/machines/template.cupid b/machines/template.cupid index 8b1b94e2fd..ec9ae233c9 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -11,15 +11,71 @@ source .env_mach_specific.sh # by setting $PYTHONPATH; since this is conda-based we # want an empty PYTHONPATH environment variable MACH=`./xmlquery --value MACH` -unset PYTHONPATH -conda activate cupid-infrastructure # Set variables that should eventually come from environment file CUPID_EXAMPLE=key_metrics -RUN_CUPID_ANALYSIS=TRUE -RUN_CUPID_TIMESERIES=TRUE +RUN_POSTPROCESSING=TRUE +CUPID_GEN_TIMESERIES=TRUE +CUPID_GEN_DIAGNOSTICS=TRUE +CUPID_GEN_HTML=TRUE +CUPID_BASELINE_CASE="" +CUPID_BASELINE_ROOT=DOUT_S_ROOT +CUPID_START_YEAR=1 +CUPID_END_YEAR=100 +CUPID_BASE_START_YEAR=1 +CUPID_BASE_END_YEAR=100 +CUPID_RUN_ALL=TRUE +CUPID_RUN_ATM=FALSE +CUPID_RUN_OCN=FALSE +CUPID_RUN_LND=FALSE +CUPID_RUN_ICE=FALSE +CUPID_RUN_ROF=FALSE +CUPID_INFRASTRUCTURE_ENV=cupid-infrastructure +CUPID_ANALYSIS_ENV=cupid-analysis + +# CUPID_EXAMPLE=`./xmlquery --value CUPID_EXAMPLE` +# RUN_POSTPROCESSING=`./xmlquery --value RUN_POSTPROCESSING` +# CUPID_GEN_TIMESERIES=`./xmlquery --value CUPID_GEN_TIMESERIES` +# CUPID_GEN_DIAGNOSTICS=`./xmlquery --value CUPID_GEN_DIAGNOSTICS` +# CUPID_GEN_HTML=`./xmlquery --value CUPID_GEN_HTML` +# CUPID_BASELINE_CASE=`./xmlquery --value CUPID_BASELINE_CASE` +# CUPID_BASELINE_ROOT=`./xmlquery --value CUPID_BASELINE_ROOT` +# CUPID_START_YEAR=`./xmlquery --value CUPID_START_YEAR` +# CUPID_END_YEAR=`./xmlquery --value CUPID_END_YEAR` +# CUPID_BASE_START_YEAR=`./xmlquery --value CUPID_BASE_START_YEAR` +# CUPID_BASE_END_YEAR=`./xmlquery --value CUPID_BASE_END_YEAR` +# CUPID_RUN_ALL=`./xmlquery --value CUPID_RUN_ALL` +# CUPID_RUN_ATM=`./xmlquery --value CUPID_RUN_ATM` +# CUPID_RUN_OCN=`./xmlquery --value CUPID_RUN_OCN` +# CUPID_RUN_LND=`./xmlquery --value CUPID_RUN_LND` +# CUPID_RUN_ICE=`./xmlquery --value CUPID_RUN_ICE` +# CUPID_RUN_ROF=`./xmlquery --value CUPID_RUN_ROF` +# CUPID_INFRASTRUCTURE_ENV=`./xmlquery --value CUPID_INFRASTRUCTURE_ENV` +# CUPID_ANALYSIS_ENV=`./xmlquery --value CUPID_ANALYSIS_ENV` -if [ "${RUN_CUPID_ANALYSIS}" == "TRUE" ]; then +CUPID_FLAG_STRING="" +if [ "${CUPID_RUN_ALL}" == "FALSE" ]; then + if [ "${CUPID_RUN_ATM}" == "TRUE" ]; then + CUPID_FLAG_STRING=CUPID_FLAG_STRING+" --atm" + fi + if [ "${CUPID_RUN_OCN}" == "TRUE" ]; then + CUPID_FLAG_STRING=CUPID_FLAG_STRING+" --ocn" + fi + if [ "${CUPID_RUN_LND}" == "TRUE" ]; then + CUPID_FLAG_STRING=CUPID_FLAG_STRING+" --lnd" + fi + if [ "${CUPID_RUN_ICE}" == "TRUE" ]; then + CUPID_FLAG_STRING=CUPID_FLAG_STRING+" --ice" + fi + if [ "${CUPID_RUN_ROF}" == "TRUE" ]; then + CUPID_FLAG_STRING=CUPID_FLAG_STRING+" --rof" + fi +fi + +unset PYTHONPATH +conda activate $CUPID_INFRASTRUCTURE_ENV + +if [ "${RUN_POSTPROCESSING}" == "TRUE" ]; then # 1. Generate CUPiD config file {{ srcroot }}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ --cesm-root {{ SRCROOT }} @@ -32,16 +88,20 @@ if [ "${RUN_CUPID_ANALYSIS}" == "TRUE" ]; then --out-file adf_config.yml # 3. Generate timeseries files and run ADF - if [ "${RUN_CUPID_TIMESERIES}" == "TRUE" ]; then - {{ srcroot }}/tools/CUPiD/cupid/run_timeseries.py + if [ "${CUPID_GEN_TIMESERIES}" == "TRUE" ]; then + {{ srcroot }}/tools/CUPiD/cupid/run_timeseries.py $CUPID_FLAG_STRING fi conda deactivate - conda activate cupid-analysis + conda activate $CUPID_ANALYSIS_ENV {{ SRCROOT }}/tools/CUPiD/externals/ADF/run_adf_diag adf_config.yml # 4. Run CUPiD and build webpage conda deactivate - conda activate cupid-infrastructure - {{ srcroot }}/tools/CUPiD/cupid/run_diagnostics.py - {{ srcroot }}/tools/CUPiD/cupid/cupid_webpage.py + conda activate $CUPID_INFRASTRUCTURE_ENV + if [ "${CUPID_GEN_DIAGNOSTICS}" == "TRUE" ]; then + {{ srcroot }}/tools/CUPiD/cupid/run_diagnostics.py $CUPID_FLAG_STRING + fi + if [ "${CUPID_GEN_HTML}" == "TRUE" ]; then + {{ srcroot }}/tools/CUPiD/cupid/cupid_webpage.py + fi fi From 9739ebf13ff280c7d9a1157191a4b3f5b289e2ea Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 30 Jan 2025 11:59:36 -0700 Subject: [PATCH 02/11] Update template.cupid with bash += string format --- machines/template.cupid | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/machines/template.cupid b/machines/template.cupid index ec9ae233c9..4144fc7fb7 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -56,19 +56,19 @@ CUPID_ANALYSIS_ENV=cupid-analysis CUPID_FLAG_STRING="" if [ "${CUPID_RUN_ALL}" == "FALSE" ]; then if [ "${CUPID_RUN_ATM}" == "TRUE" ]; then - CUPID_FLAG_STRING=CUPID_FLAG_STRING+" --atm" + CUPID_FLAG_STRING+=" --atm" fi if [ "${CUPID_RUN_OCN}" == "TRUE" ]; then - CUPID_FLAG_STRING=CUPID_FLAG_STRING+" --ocn" + CUPID_FLAG_STRING+=" --ocn" fi if [ "${CUPID_RUN_LND}" == "TRUE" ]; then - CUPID_FLAG_STRING=CUPID_FLAG_STRING+" --lnd" + CUPID_FLAG_STRING+=" --lnd" fi if [ "${CUPID_RUN_ICE}" == "TRUE" ]; then - CUPID_FLAG_STRING=CUPID_FLAG_STRING+" --ice" + CUPID_FLAG_STRING+=" --ice" fi if [ "${CUPID_RUN_ROF}" == "TRUE" ]; then - CUPID_FLAG_STRING=CUPID_FLAG_STRING+" --rof" + CUPID_FLAG_STRING+=" --rof" fi fi From bca903e9c37b50e37de3a11e4a033587b6c8ac66 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 30 Jan 2025 13:50:42 -0700 Subject: [PATCH 03/11] Update template.cupid for glc --- machines/template.cupid | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/machines/template.cupid b/machines/template.cupid index 4144fc7fb7..a575d55221 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -30,6 +30,7 @@ CUPID_RUN_OCN=FALSE CUPID_RUN_LND=FALSE CUPID_RUN_ICE=FALSE CUPID_RUN_ROF=FALSE +CUPID_RUN_GLC=FALSE CUPID_INFRASTRUCTURE_ENV=cupid-infrastructure CUPID_ANALYSIS_ENV=cupid-analysis @@ -50,6 +51,7 @@ CUPID_ANALYSIS_ENV=cupid-analysis # CUPID_RUN_LND=`./xmlquery --value CUPID_RUN_LND` # CUPID_RUN_ICE=`./xmlquery --value CUPID_RUN_ICE` # CUPID_RUN_ROF=`./xmlquery --value CUPID_RUN_ROF` +# CUPID_RUN_GLC=`./xmlquery --value CUPID_RUN_GLC` # CUPID_INFRASTRUCTURE_ENV=`./xmlquery --value CUPID_INFRASTRUCTURE_ENV` # CUPID_ANALYSIS_ENV=`./xmlquery --value CUPID_ANALYSIS_ENV` @@ -70,6 +72,9 @@ if [ "${CUPID_RUN_ALL}" == "FALSE" ]; then if [ "${CUPID_RUN_ROF}" == "TRUE" ]; then CUPID_FLAG_STRING+=" --rof" fi + if [ "${CUPID_RUN_GLC}" == "TRUE" ]; then + CUPID_FLAG_STRING+=" --glc" + fi fi unset PYTHONPATH From 3308bd2f16e806d0e0cfa4a36e2a0fd35242246d Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Fri, 7 Feb 2025 14:02:24 -0700 Subject: [PATCH 04/11] Update template.cupid in response to Bill's review --- machines/template.cupid | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/machines/template.cupid b/machines/template.cupid index a575d55221..5cc1aa88ff 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -13,6 +13,11 @@ source .env_mach_specific.sh MACH=`./xmlquery --value MACH` # Set variables that should eventually come from environment file +RUN_POSTPROCESSING=TRUE +# RUN_POSTPROCESSING=`./xmlquery --value RUN_POSTPROCESSING` +if [ "${RUN_POSTPROCESSING}" == "FALSE" ]; then + exit 0 +fi CUPID_EXAMPLE=key_metrics RUN_POSTPROCESSING=TRUE CUPID_GEN_TIMESERIES=TRUE @@ -55,6 +60,8 @@ CUPID_ANALYSIS_ENV=cupid-analysis # CUPID_INFRASTRUCTURE_ENV=`./xmlquery --value CUPID_INFRASTRUCTURE_ENV` # CUPID_ANALYSIS_ENV=`./xmlquery --value CUPID_ANALYSIS_ENV` +# If CUPID_RUN_ALL is TRUE, we don't add any component flags. +# The lack of any component flags tells CUPiD to run all components. CUPID_FLAG_STRING="" if [ "${CUPID_RUN_ALL}" == "FALSE" ]; then if [ "${CUPID_RUN_ATM}" == "TRUE" ]; then @@ -76,6 +83,10 @@ if [ "${CUPID_RUN_ALL}" == "FALSE" ]; then CUPID_FLAG_STRING+=" --glc" fi fi +if [ "${CUPID_FLAG_STRING}" == "" ]; then + echo "Need to set at least one component" + exit 1 +fi unset PYTHONPATH conda activate $CUPID_INFRASTRUCTURE_ENV From 6cb38e190a0b9a4695c90bce946319a9e959fec8 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Fri, 7 Feb 2025 14:28:26 -0700 Subject: [PATCH 05/11] Update template.cupid to move if postprocessing --- machines/template.cupid | 51 +++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/machines/template.cupid b/machines/template.cupid index 5cc1aa88ff..423b815c66 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -40,7 +40,6 @@ CUPID_INFRASTRUCTURE_ENV=cupid-infrastructure CUPID_ANALYSIS_ENV=cupid-analysis # CUPID_EXAMPLE=`./xmlquery --value CUPID_EXAMPLE` -# RUN_POSTPROCESSING=`./xmlquery --value RUN_POSTPROCESSING` # CUPID_GEN_TIMESERIES=`./xmlquery --value CUPID_GEN_TIMESERIES` # CUPID_GEN_DIAGNOSTICS=`./xmlquery --value CUPID_GEN_DIAGNOSTICS` # CUPID_GEN_HTML=`./xmlquery --value CUPID_GEN_HTML` @@ -91,33 +90,31 @@ fi unset PYTHONPATH conda activate $CUPID_INFRASTRUCTURE_ENV -if [ "${RUN_POSTPROCESSING}" == "TRUE" ]; then - # 1. Generate CUPiD config file - {{ srcroot }}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ - --cesm-root {{ SRCROOT }} +# 1. Generate CUPiD config file +{{ srcroot }}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ + --cesm-root {{ SRCROOT }} - # 2. Generate ADF config file - {{ srcroot }}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ - --cesm-root {{ SRCROOT }} \ - --cupid-config-loc ${PWD} \ - --adf-template {{ SRCROOT }}/tools/CUPiD/externals/ADF/config_amwg_default_plots.yaml \ - --out-file adf_config.yml +# 2. Generate ADF config file +{{ srcroot }}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ + --cesm-root {{ SRCROOT }} \ + --cupid-config-loc ${PWD} \ + --adf-template {{ SRCROOT }}/tools/CUPiD/externals/ADF/config_amwg_default_plots.yaml \ + --out-file adf_config.yml - # 3. Generate timeseries files and run ADF - if [ "${CUPID_GEN_TIMESERIES}" == "TRUE" ]; then - {{ srcroot }}/tools/CUPiD/cupid/run_timeseries.py $CUPID_FLAG_STRING - fi - conda deactivate - conda activate $CUPID_ANALYSIS_ENV - {{ SRCROOT }}/tools/CUPiD/externals/ADF/run_adf_diag adf_config.yml +# 3. Generate timeseries files and run ADF +if [ "${CUPID_GEN_TIMESERIES}" == "TRUE" ]; then + {{ srcroot }}/tools/CUPiD/cupid/run_timeseries.py $CUPID_FLAG_STRING +fi +conda deactivate +conda activate $CUPID_ANALYSIS_ENV +{{ SRCROOT }}/tools/CUPiD/externals/ADF/run_adf_diag adf_config.yml - # 4. Run CUPiD and build webpage - conda deactivate - conda activate $CUPID_INFRASTRUCTURE_ENV - if [ "${CUPID_GEN_DIAGNOSTICS}" == "TRUE" ]; then - {{ srcroot }}/tools/CUPiD/cupid/run_diagnostics.py $CUPID_FLAG_STRING - fi - if [ "${CUPID_GEN_HTML}" == "TRUE" ]; then - {{ srcroot }}/tools/CUPiD/cupid/cupid_webpage.py - fi +# 4. Run CUPiD and build webpage +conda deactivate +conda activate $CUPID_INFRASTRUCTURE_ENV +if [ "${CUPID_GEN_DIAGNOSTICS}" == "TRUE" ]; then + {{ srcroot }}/tools/CUPiD/cupid/run_diagnostics.py $CUPID_FLAG_STRING +fi +if [ "${CUPID_GEN_HTML}" == "TRUE" ]; then + {{ srcroot }}/tools/CUPiD/cupid/cupid_webpage.py fi From f3f6f556b4e274c9e101a9eeb2a43483c052ec11 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:27:05 -0700 Subject: [PATCH 06/11] Update template.cupid - logic fix for all components flag --- machines/template.cupid | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/machines/template.cupid b/machines/template.cupid index 423b815c66..56f7d3eb43 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -81,10 +81,13 @@ if [ "${CUPID_RUN_ALL}" == "FALSE" ]; then if [ "${CUPID_RUN_GLC}" == "TRUE" ]; then CUPID_FLAG_STRING+=" --glc" fi + if [ "${CUPID_FLAG_STRING}" == "" ]; then + echo "If CUPID_RUN_ALL is False, user must set at least one component" + exit 1 + fi fi -if [ "${CUPID_FLAG_STRING}" == "" ]; then - echo "Need to set at least one component" - exit 1 +if [ "${CUPID_RUN_ALL}" == "TRUE" ]; then + echo "CUPID_RUN_ALL is True, running diagnostics for all components" fi unset PYTHONPATH From 615153dbe06a62494957ce8cf6eeaea0a9d6882c Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:38:55 -0700 Subject: [PATCH 07/11] Update template.cupid to include new arguments for generate_cupid_config_for_cesm_case.py --- machines/template.cupid | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/machines/template.cupid b/machines/template.cupid index 56f7d3eb43..fc28236903 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -95,7 +95,15 @@ conda activate $CUPID_INFRASTRUCTURE_ENV # 1. Generate CUPiD config file {{ srcroot }}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ - --cesm-root {{ SRCROOT }} + --cesm-root {{ SRCROOT }} \ + --case-root {{ CASEROOT }} \ + --cupid-example {{ CUPID_EXAMPLE }} \ + --cupid-baseline-case {{ CUPID_BASELINE_CASE }} \ + --cupid-baseline-root {{ CUPID_BASELINE_ROOT }} \ + --cupid-start-year {{ CUPID_START_YEAR }} \ + --cupid-end-year {{ CUPID_END_YEAR }} \ + --cupid-base-start-year {{ CUPID_BASE_START_YEAR }} \ + --cupid-base-end-year {{ CUPID_BASE_END_YEAR }} \ # 2. Generate ADF config file {{ srcroot }}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ From 9268ab18f5c662ee4acf9b0889d0397d139bbf88 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:47:18 -0700 Subject: [PATCH 08/11] Update template.cupid to use cupid example directory rather than PWD --- machines/template.cupid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machines/template.cupid b/machines/template.cupid index fc28236903..80b16a4e8b 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -108,9 +108,9 @@ conda activate $CUPID_INFRASTRUCTURE_ENV # 2. Generate ADF config file {{ srcroot }}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ --cesm-root {{ SRCROOT }} \ - --cupid-config-loc ${PWD} \ + --cupid-config-loc {{ srcroot }}/tools/CUPiD/examples/{{ CUPID_EXAMPLE }} \ --adf-template {{ SRCROOT }}/tools/CUPiD/externals/ADF/config_amwg_default_plots.yaml \ - --out-file adf_config.yml + --out-file {{ srcroot }}/tools/CUPiD/examples/{{ CUPID_EXAMPLE }}/adf_config.yml # 3. Generate timeseries files and run ADF if [ "${CUPID_GEN_TIMESERIES}" == "TRUE" ]; then From 1d0783aa58991128898671c92a766ce5797ba5a1 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Tue, 11 Feb 2025 11:32:14 -0700 Subject: [PATCH 09/11] Updates to CUPiD workflow Teagan and I added a bunch of xml variables to env_postprocessing.xml (in a CUPiD PR), so template.cupid now uses those variables to modify the CUPiD config. Also, we moved case.cupid into the default workflow (but RUN_POSTPROCESSING is FALSE, so it won't run unless the user changes that variable to TRUE) --- machines/config_workflow.xml | 43 +++++++------ machines/template.cupid | 113 +++++++++++++++++------------------ 2 files changed, 74 insertions(+), 82 deletions(-) diff --git a/machines/config_workflow.xml b/machines/config_workflow.xml index 7f6f2fc311..2902e74185 100644 --- a/machines/config_workflow.xml +++ b/machines/config_workflow.xml @@ -43,9 +43,20 @@ case.run or case.test $DOUT_S - 1 - 1 - 0:20:00 + 1 + 1 + 0:20:00 + + + + + false + case.st_archive + $RUN_POSTPROCESSING + + 1 + 1 + 0:20:00 @@ -56,28 +67,14 @@ case.st_archive 1 - 200 - 10 - 12:00:00 + 200 + 10 + 12:00:00 - 72 - 9 - 0:20:00 - - - - - - - - false - case.st_archive - 1 - - 1 - 1 - 0:20:00 + 72 + 9 + 0:20:00 diff --git a/machines/template.cupid b/machines/template.cupid index 80b16a4e8b..9583bf7190 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -1,5 +1,16 @@ #!/bin/bash -e +# Function to add some number of years to a string that +# is formatted as YYYY-MM-DD and print out the updated +# string in the same format +add_years() { + YEAR=`echo $1 | cut -d '-' -f 1` + MM=`echo $1 | cut -d '-' -f 2` + DD=`echo $1 | cut -d '-' -f 3` + NEW_YEAR=`printf '%04d' "$((YEAR + $2))"`-`printf '%02d' "${MM}"`-`printf '%02d' "${DD}"` + echo $NEW_YEAR +} + # Batch system directives {{ batchdirectives }} @@ -12,52 +23,32 @@ source .env_mach_specific.sh # want an empty PYTHONPATH environment variable MACH=`./xmlquery --value MACH` -# Set variables that should eventually come from environment file -RUN_POSTPROCESSING=TRUE -# RUN_POSTPROCESSING=`./xmlquery --value RUN_POSTPROCESSING` -if [ "${RUN_POSTPROCESSING}" == "FALSE" ]; then - exit 0 -fi -CUPID_EXAMPLE=key_metrics -RUN_POSTPROCESSING=TRUE -CUPID_GEN_TIMESERIES=TRUE -CUPID_GEN_DIAGNOSTICS=TRUE -CUPID_GEN_HTML=TRUE -CUPID_BASELINE_CASE="" -CUPID_BASELINE_ROOT=DOUT_S_ROOT -CUPID_START_YEAR=1 -CUPID_END_YEAR=100 -CUPID_BASE_START_YEAR=1 -CUPID_BASE_END_YEAR=100 -CUPID_RUN_ALL=TRUE -CUPID_RUN_ATM=FALSE -CUPID_RUN_OCN=FALSE -CUPID_RUN_LND=FALSE -CUPID_RUN_ICE=FALSE -CUPID_RUN_ROF=FALSE -CUPID_RUN_GLC=FALSE -CUPID_INFRASTRUCTURE_ENV=cupid-infrastructure -CUPID_ANALYSIS_ENV=cupid-analysis +# Set variables that come from environment file +CUPID_EXAMPLE=`./xmlquery --value CUPID_EXAMPLE` +CUPID_GEN_TIMESERIES=`./xmlquery --value CUPID_GEN_TIMESERIES` +CUPID_GEN_DIAGNOSTICS=`./xmlquery --value CUPID_GEN_DIAGNOSTICS` +CUPID_GEN_HTML=`./xmlquery --value CUPID_GEN_HTML` +CUPID_BASELINE_CASE=`./xmlquery --value CUPID_BASELINE_CASE` +CUPID_BASELINE_ROOT=`./xmlquery --value CUPID_BASELINE_ROOT` +CUPID_START_YEAR=`./xmlquery --value CUPID_START_YEAR` +CUPID_NYEARS=`./xmlquery --value CUPID_NYEARS` +CUPID_END_YEAR=`add_years $CUPID_START_YEAR $CUPID_NYEARS` +CUPID_BASE_START_YEAR=`./xmlquery --value CUPID_BASE_START_YEAR` +CUPID_BASE_NYEARS=`./xmlquery --value CUPID_BASE_NYEARS` +CUPID_BASE_END_YEAR=`add_years $CUPID_BASE_START_YEAR $CUPID_BASE_NYEARS` +CUPID_RUN_ALL=`./xmlquery --value CUPID_RUN_ALL` +CUPID_RUN_ATM=`./xmlquery --value CUPID_RUN_ATM` +CUPID_RUN_OCN=`./xmlquery --value CUPID_RUN_OCN` +CUPID_RUN_LND=`./xmlquery --value CUPID_RUN_LND` +CUPID_RUN_ICE=`./xmlquery --value CUPID_RUN_ICE` +CUPID_RUN_ROF=`./xmlquery --value CUPID_RUN_ROF` +CUPID_RUN_GLC=`./xmlquery --value CUPID_RUN_GLC` +CUPID_INFRASTRUCTURE_ENV=`./xmlquery --value CUPID_INFRASTRUCTURE_ENV` +CUPID_ANALYSIS_ENV=`./xmlquery --value CUPID_ANALYSIS_ENV` -# CUPID_EXAMPLE=`./xmlquery --value CUPID_EXAMPLE` -# CUPID_GEN_TIMESERIES=`./xmlquery --value CUPID_GEN_TIMESERIES` -# CUPID_GEN_DIAGNOSTICS=`./xmlquery --value CUPID_GEN_DIAGNOSTICS` -# CUPID_GEN_HTML=`./xmlquery --value CUPID_GEN_HTML` -# CUPID_BASELINE_CASE=`./xmlquery --value CUPID_BASELINE_CASE` -# CUPID_BASELINE_ROOT=`./xmlquery --value CUPID_BASELINE_ROOT` -# CUPID_START_YEAR=`./xmlquery --value CUPID_START_YEAR` -# CUPID_END_YEAR=`./xmlquery --value CUPID_END_YEAR` -# CUPID_BASE_START_YEAR=`./xmlquery --value CUPID_BASE_START_YEAR` -# CUPID_BASE_END_YEAR=`./xmlquery --value CUPID_BASE_END_YEAR` -# CUPID_RUN_ALL=`./xmlquery --value CUPID_RUN_ALL` -# CUPID_RUN_ATM=`./xmlquery --value CUPID_RUN_ATM` -# CUPID_RUN_OCN=`./xmlquery --value CUPID_RUN_OCN` -# CUPID_RUN_LND=`./xmlquery --value CUPID_RUN_LND` -# CUPID_RUN_ICE=`./xmlquery --value CUPID_RUN_ICE` -# CUPID_RUN_ROF=`./xmlquery --value CUPID_RUN_ROF` -# CUPID_RUN_GLC=`./xmlquery --value CUPID_RUN_GLC` -# CUPID_INFRASTRUCTURE_ENV=`./xmlquery --value CUPID_INFRASTRUCTURE_ENV` -# CUPID_ANALYSIS_ENV=`./xmlquery --value CUPID_ANALYSIS_ENV` +# Create directory for running CUPiD +mkdir -p cupid-postprocessing +cd cupid-postprocessing # If CUPID_RUN_ALL is TRUE, we don't add any component flags. # The lack of any component flags tells CUPiD to run all components. @@ -94,27 +85,31 @@ unset PYTHONPATH conda activate $CUPID_INFRASTRUCTURE_ENV # 1. Generate CUPiD config file -{{ srcroot }}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ +{{ SRCROOT }}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ --cesm-root {{ SRCROOT }} \ --case-root {{ CASEROOT }} \ - --cupid-example {{ CUPID_EXAMPLE }} \ - --cupid-baseline-case {{ CUPID_BASELINE_CASE }} \ - --cupid-baseline-root {{ CUPID_BASELINE_ROOT }} \ - --cupid-start-year {{ CUPID_START_YEAR }} \ - --cupid-end-year {{ CUPID_END_YEAR }} \ - --cupid-base-start-year {{ CUPID_BASE_START_YEAR }} \ - --cupid-base-end-year {{ CUPID_BASE_END_YEAR }} \ + --cupid-example $CUPID_EXAMPLE \ + --cupid-baseline-case $CUPID_BASELINE_CASE \ + --cupid-baseline-root $CUPID_BASELINE_ROOT \ + --cupid-start-year $CUPID_START_YEAR \ + --cupid-end-year $CUPID_END_YEAR \ + --cupid-base-start-year $CUPID_BASE_START_YEAR \ + --cupid-base-end-year $CUPID_BASE_END_YEAR \ # 2. Generate ADF config file -{{ srcroot }}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ +{{ SRCROOT }}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ --cesm-root {{ SRCROOT }} \ - --cupid-config-loc {{ srcroot }}/tools/CUPiD/examples/{{ CUPID_EXAMPLE }} \ + --cupid-config-loc {{ SRCROOT }}/tools/CUPiD/examples/{{ CUPID_EXAMPLE }} \ --adf-template {{ SRCROOT }}/tools/CUPiD/externals/ADF/config_amwg_default_plots.yaml \ - --out-file {{ srcroot }}/tools/CUPiD/examples/{{ CUPID_EXAMPLE }}/adf_config.yml + --out-file adf_config.yml + + +# TODO: remove this before committing +exit 0 # 3. Generate timeseries files and run ADF if [ "${CUPID_GEN_TIMESERIES}" == "TRUE" ]; then - {{ srcroot }}/tools/CUPiD/cupid/run_timeseries.py $CUPID_FLAG_STRING + {{ SRCROOT }}/tools/CUPiD/cupid/run_timeseries.py $CUPID_FLAG_STRING fi conda deactivate conda activate $CUPID_ANALYSIS_ENV @@ -124,8 +119,8 @@ conda activate $CUPID_ANALYSIS_ENV conda deactivate conda activate $CUPID_INFRASTRUCTURE_ENV if [ "${CUPID_GEN_DIAGNOSTICS}" == "TRUE" ]; then - {{ srcroot }}/tools/CUPiD/cupid/run_diagnostics.py $CUPID_FLAG_STRING + {{ SRCROOT }}/tools/CUPiD/cupid/run_diagnostics.py $CUPID_FLAG_STRING fi if [ "${CUPID_GEN_HTML}" == "TRUE" ]; then - {{ srcroot }}/tools/CUPiD/cupid/cupid_webpage.py + {{ SRCROOT }}/tools/CUPiD/cupid/cupid_webpage.py fi From cc9b543c912189156d3463c7890409ae0cb0880e Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Tue, 11 Feb 2025 12:08:33 -0700 Subject: [PATCH 10/11] Update template.cupid to include ADF xml var --- machines/template.cupid | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/machines/template.cupid b/machines/template.cupid index 9583bf7190..dd536bee1a 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -43,6 +43,7 @@ CUPID_RUN_LND=`./xmlquery --value CUPID_RUN_LND` CUPID_RUN_ICE=`./xmlquery --value CUPID_RUN_ICE` CUPID_RUN_ROF=`./xmlquery --value CUPID_RUN_ROF` CUPID_RUN_GLC=`./xmlquery --value CUPID_RUN_GLC` +CUPID_RUN_ADF=`./xmlquery --value CUPID_RUN_ADF` CUPID_INFRASTRUCTURE_ENV=`./xmlquery --value CUPID_INFRASTRUCTURE_ENV` CUPID_ANALYSIS_ENV=`./xmlquery --value CUPID_ANALYSIS_ENV` @@ -97,25 +98,27 @@ conda activate $CUPID_INFRASTRUCTURE_ENV --cupid-base-end-year $CUPID_BASE_END_YEAR \ # 2. Generate ADF config file -{{ SRCROOT }}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ - --cesm-root {{ SRCROOT }} \ - --cupid-config-loc {{ SRCROOT }}/tools/CUPiD/examples/{{ CUPID_EXAMPLE }} \ - --adf-template {{ SRCROOT }}/tools/CUPiD/externals/ADF/config_amwg_default_plots.yaml \ - --out-file adf_config.yml - - -# TODO: remove this before committing -exit 0 +if [ "${CUPID_RUN_ADF}" == "TRUE" ]; then + {{ SRCROOT }}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ + --cesm-root {{ SRCROOT }} \ + --cupid-config-loc {{ SRCROOT }}/tools/CUPiD/examples/{{ CUPID_EXAMPLE }} \ + --adf-template {{ SRCROOT }}/tools/CUPiD/externals/ADF/config_amwg_default_plots.yaml \ + --out-file adf_config.yml +fi -# 3. Generate timeseries files and run ADF +# 3. Generate timeseries files if [ "${CUPID_GEN_TIMESERIES}" == "TRUE" ]; then {{ SRCROOT }}/tools/CUPiD/cupid/run_timeseries.py $CUPID_FLAG_STRING fi -conda deactivate -conda activate $CUPID_ANALYSIS_ENV -{{ SRCROOT }}/tools/CUPiD/externals/ADF/run_adf_diag adf_config.yml -# 4. Run CUPiD and build webpage +#4. Run ADF +if [ "${CUPID_RUN_ADF}" == "TRUE" ]; then + conda deactivate + conda activate $CUPID_ANALYSIS_ENV + {{ SRCROOT }}/tools/CUPiD/externals/ADF/run_adf_diag adf_config.yml +fi + +# 5. Run CUPiD and build webpage conda deactivate conda activate $CUPID_INFRASTRUCTURE_ENV if [ "${CUPID_GEN_DIAGNOSTICS}" == "TRUE" ]; then From c99f0445b61a7f053e0498e7429c17ed6a1ae14d Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Tue, 11 Feb 2025 16:28:09 -0700 Subject: [PATCH 11/11] Update template.cupid in response to Bill's review --- machines/template.cupid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/template.cupid b/machines/template.cupid index dd536bee1a..d5399dea7f 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -101,7 +101,7 @@ conda activate $CUPID_INFRASTRUCTURE_ENV if [ "${CUPID_RUN_ADF}" == "TRUE" ]; then {{ SRCROOT }}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ --cesm-root {{ SRCROOT }} \ - --cupid-config-loc {{ SRCROOT }}/tools/CUPiD/examples/{{ CUPID_EXAMPLE }} \ + --cupid-config-loc {{ SRCROOT }}/tools/CUPiD/examples/${CUPID_EXAMPLE} \ --adf-template {{ SRCROOT }}/tools/CUPiD/externals/ADF/config_amwg_default_plots.yaml \ --out-file adf_config.yml fi